sys/dev/nxge/if_nxge.c

   1 /*-
   2  * Copyright (c) 2002-2007 Neterion, Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  *
  26  * $FreeBSD$
  27  */
  28
  29 #include <dev/nxge/if_nxge.h>
  30 #include <dev/nxge/xge-osdep.h>
  31 #include <net/if_arp.h>
  32 #include <sys/types.h>
  33 #include <net/if.h>
  34 #include <net/if_var.h>
  35 #include <net/if_vlan_var.h>
  36
  37 int       copyright_print       = 0;
  38 int       hal_driver_init_count = 0;
  39 size_t    size                  = sizeof(int);
  40
  41 static void inline xge_flush_txds(xge_hal_channel_h);
  42
  43 /**
  44  * xge_probe
  45  * Probes for Xframe devices
  46  *
  47  * @dev Device handle
  48  *
  49  * Returns
  50  * BUS_PROBE_DEFAULT if device is supported
  51  * ENXIO if device is not supported
  52  */
  53 int
  54 xge_probe(device_t dev)
  55 {
  56         int  devid    = pci_get_device(dev);
  57         int  vendorid = pci_get_vendor(dev);
  58         int  retValue = ENXIO;
  59
  60         if(vendorid == XGE_PCI_VENDOR_ID) {
  61             if((devid == XGE_PCI_DEVICE_ID_XENA_2) ||
  62                 (devid == XGE_PCI_DEVICE_ID_HERC_2)) {
  63                 if(!copyright_print) {
  64                     xge_os_printf(XGE_COPYRIGHT);
  65                     copyright_print = 1;
  66                 }
  67                 device_set_desc_copy(dev,
  68                     "Neterion Xframe 10 Gigabit Ethernet Adapter");
  69                 retValue = BUS_PROBE_DEFAULT;
  70             }
  71         }
  72
  73         return retValue;
  74 }
  75
  76 /**
  77  * xge_init_params
  78  * Sets HAL parameter values (from kenv).
  79  *
  80  * @dconfig Device Configuration
  81  * @dev Device Handle
  82  */
  83 void
  84 xge_init_params(xge_hal_device_config_t *dconfig, device_t dev)
  85 {
  86         int qindex, tindex, revision;
  87         device_t checkdev;
  88         xge_lldev_t *lldev = (xge_lldev_t *)device_get_softc(dev);
  89
  90         dconfig->mtu                   = XGE_DEFAULT_INITIAL_MTU;
  91         dconfig->pci_freq_mherz        = XGE_DEFAULT_USER_HARDCODED;
  92         dconfig->device_poll_millis    = XGE_HAL_DEFAULT_DEVICE_POLL_MILLIS;
  93         dconfig->link_stability_period = XGE_HAL_DEFAULT_LINK_STABILITY_PERIOD;
  94         dconfig->mac.rmac_bcast_en     = XGE_DEFAULT_MAC_RMAC_BCAST_EN;
  95         dconfig->fifo.alignment_size   = XGE_DEFAULT_FIFO_ALIGNMENT_SIZE;
  96
  97         XGE_GET_PARAM("hw.xge.enable_tso", (*lldev), enabled_tso,
  98             XGE_DEFAULT_ENABLED_TSO);
  99         XGE_GET_PARAM("hw.xge.enable_lro", (*lldev), enabled_lro,
 100             XGE_DEFAULT_ENABLED_LRO);
 101         XGE_GET_PARAM("hw.xge.enable_msi", (*lldev), enabled_msi,
 102             XGE_DEFAULT_ENABLED_MSI);
 103
 104         XGE_GET_PARAM("hw.xge.latency_timer", (*dconfig), latency_timer,
 105             XGE_DEFAULT_LATENCY_TIMER);
 106         XGE_GET_PARAM("hw.xge.max_splits_trans", (*dconfig), max_splits_trans,
 107             XGE_DEFAULT_MAX_SPLITS_TRANS);
 108         XGE_GET_PARAM("hw.xge.mmrb_count", (*dconfig), mmrb_count,
 109             XGE_DEFAULT_MMRB_COUNT);
 110         XGE_GET_PARAM("hw.xge.shared_splits", (*dconfig), shared_splits,
 111             XGE_DEFAULT_SHARED_SPLITS);
 112         XGE_GET_PARAM("hw.xge.isr_polling_cnt", (*dconfig), isr_polling_cnt,
 113             XGE_DEFAULT_ISR_POLLING_CNT);
 114         XGE_GET_PARAM("hw.xge.stats_refresh_time_sec", (*dconfig),
 115             stats_refresh_time_sec, XGE_DEFAULT_STATS_REFRESH_TIME_SEC);
 116
 117         XGE_GET_PARAM_MAC("hw.xge.mac_tmac_util_period", tmac_util_period,
 118             XGE_DEFAULT_MAC_TMAC_UTIL_PERIOD);
 119         XGE_GET_PARAM_MAC("hw.xge.mac_rmac_util_period", rmac_util_period,
 120             XGE_DEFAULT_MAC_RMAC_UTIL_PERIOD);
 121         XGE_GET_PARAM_MAC("hw.xge.mac_rmac_pause_gen_en", rmac_pause_gen_en,
 122             XGE_DEFAULT_MAC_RMAC_PAUSE_GEN_EN);
 123         XGE_GET_PARAM_MAC("hw.xge.mac_rmac_pause_rcv_en", rmac_pause_rcv_en,
 124             XGE_DEFAULT_MAC_RMAC_PAUSE_RCV_EN);
 125         XGE_GET_PARAM_MAC("hw.xge.mac_rmac_pause_time", rmac_pause_time,
 126             XGE_DEFAULT_MAC_RMAC_PAUSE_TIME);
 127         XGE_GET_PARAM_MAC("hw.xge.mac_mc_pause_threshold_q0q3",
 128             mc_pause_threshold_q0q3, XGE_DEFAULT_MAC_MC_PAUSE_THRESHOLD_Q0Q3);
 129         XGE_GET_PARAM_MAC("hw.xge.mac_mc_pause_threshold_q4q7",
 130             mc_pause_threshold_q4q7, XGE_DEFAULT_MAC_MC_PAUSE_THRESHOLD_Q4Q7);
 131
 132         XGE_GET_PARAM_FIFO("hw.xge.fifo_memblock_size", memblock_size,
 133             XGE_DEFAULT_FIFO_MEMBLOCK_SIZE);
 134         XGE_GET_PARAM_FIFO("hw.xge.fifo_reserve_threshold", reserve_threshold,
 135             XGE_DEFAULT_FIFO_RESERVE_THRESHOLD);
 136         XGE_GET_PARAM_FIFO("hw.xge.fifo_max_frags", max_frags,
 137             XGE_DEFAULT_FIFO_MAX_FRAGS);
 138
 139         for(qindex = 0; qindex < XGE_FIFO_COUNT; qindex++) {
 140             XGE_GET_PARAM_FIFO_QUEUE("hw.xge.fifo_queue_intr", intr, qindex,
 141                 XGE_DEFAULT_FIFO_QUEUE_INTR);
 142             XGE_GET_PARAM_FIFO_QUEUE("hw.xge.fifo_queue_max", max, qindex,
 143                 XGE_DEFAULT_FIFO_QUEUE_MAX);
 144             XGE_GET_PARAM_FIFO_QUEUE("hw.xge.fifo_queue_initial", initial,
 145                 qindex, XGE_DEFAULT_FIFO_QUEUE_INITIAL);
 146
 147             for (tindex = 0; tindex < XGE_HAL_MAX_FIFO_TTI_NUM; tindex++) {
 148                 dconfig->fifo.queue[qindex].tti[tindex].enabled  = 1;
 149                 dconfig->fifo.queue[qindex].configured = 1;
 150
 151                 XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_urange_a",
 152                     urange_a, qindex, tindex,
 153                     XGE_DEFAULT_FIFO_QUEUE_TTI_URANGE_A);
 154                 XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_urange_b",
 155                     urange_b, qindex, tindex,
 156                     XGE_DEFAULT_FIFO_QUEUE_TTI_URANGE_B);
 157                 XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_urange_c",
 158                     urange_c, qindex, tindex,
 159                     XGE_DEFAULT_FIFO_QUEUE_TTI_URANGE_C);
 160                 XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_ufc_a",
 161                     ufc_a, qindex, tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_UFC_A);
 162                 XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_ufc_b",
 163                     ufc_b, qindex, tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_UFC_B);
 164                 XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_ufc_c",
 165                     ufc_c, qindex, tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_UFC_C);
 166                 XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_ufc_d",
 167                     ufc_d, qindex, tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_UFC_D);
 168                 XGE_GET_PARAM_FIFO_QUEUE_TTI(
 169                     "hw.xge.fifo_queue_tti_timer_ci_en", timer_ci_en, qindex,
 170                     tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_TIMER_CI_EN);
 171                 XGE_GET_PARAM_FIFO_QUEUE_TTI(
 172                     "hw.xge.fifo_queue_tti_timer_ac_en", timer_ac_en, qindex,
 173                     tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_TIMER_AC_EN);
 174                 XGE_GET_PARAM_FIFO_QUEUE_TTI(
 175                     "hw.xge.fifo_queue_tti_timer_val_us", timer_val_us, qindex,
 176                     tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_TIMER_VAL_US);
 177             }
 178         }
 179
 180         XGE_GET_PARAM_RING("hw.xge.ring_memblock_size", memblock_size,
 181             XGE_DEFAULT_RING_MEMBLOCK_SIZE);
 182
 183         XGE_GET_PARAM_RING("hw.xge.ring_strip_vlan_tag", strip_vlan_tag,
 184             XGE_DEFAULT_RING_STRIP_VLAN_TAG);
 185
 186         XGE_GET_PARAM("hw.xge.buffer_mode", (*lldev), buffer_mode,
 187             XGE_DEFAULT_BUFFER_MODE);
 188         if((lldev->buffer_mode < XGE_HAL_RING_QUEUE_BUFFER_MODE_1) ||
 189             (lldev->buffer_mode > XGE_HAL_RING_QUEUE_BUFFER_MODE_2)) {
 190             xge_trace(XGE_ERR, "Supported buffer modes are 1 and 2");
 191             lldev->buffer_mode = XGE_HAL_RING_QUEUE_BUFFER_MODE_1;
 192         }
 193
 194         for (qindex = 0; qindex < XGE_RING_COUNT; qindex++) {
 195             dconfig->ring.queue[qindex].max_frm_len  = XGE_HAL_RING_USE_MTU;
 196             dconfig->ring.queue[qindex].priority     = 0;
 197             dconfig->ring.queue[qindex].configured   = 1;
 198             dconfig->ring.queue[qindex].buffer_mode  =
 199                 (lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_2) ?
 200                 XGE_HAL_RING_QUEUE_BUFFER_MODE_3 : lldev->buffer_mode;
 201
 202             XGE_GET_PARAM_RING_QUEUE("hw.xge.ring_queue_max", max, qindex,
 203                 XGE_DEFAULT_RING_QUEUE_MAX);
 204             XGE_GET_PARAM_RING_QUEUE("hw.xge.ring_queue_initial", initial,
 205                 qindex, XGE_DEFAULT_RING_QUEUE_INITIAL);
 206             XGE_GET_PARAM_RING_QUEUE("hw.xge.ring_queue_dram_size_mb",
 207                 dram_size_mb, qindex, XGE_DEFAULT_RING_QUEUE_DRAM_SIZE_MB);
 208             XGE_GET_PARAM_RING_QUEUE("hw.xge.ring_queue_indicate_max_pkts",
 209                 indicate_max_pkts, qindex,
 210                 XGE_DEFAULT_RING_QUEUE_INDICATE_MAX_PKTS);
 211             XGE_GET_PARAM_RING_QUEUE("hw.xge.ring_queue_backoff_interval_us",
 212                 backoff_interval_us, qindex,
 213                 XGE_DEFAULT_RING_QUEUE_BACKOFF_INTERVAL_US);
 214
 215             XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_ufc_a", ufc_a,
 216                 qindex, XGE_DEFAULT_RING_QUEUE_RTI_UFC_A);
 217             XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_ufc_b", ufc_b,
 218                 qindex, XGE_DEFAULT_RING_QUEUE_RTI_UFC_B);
 219             XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_ufc_c", ufc_c,
 220                 qindex, XGE_DEFAULT_RING_QUEUE_RTI_UFC_C);
 221             XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_ufc_d", ufc_d,
 222                 qindex, XGE_DEFAULT_RING_QUEUE_RTI_UFC_D);
 223             XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_timer_ac_en",
 224                 timer_ac_en, qindex, XGE_DEFAULT_RING_QUEUE_RTI_TIMER_AC_EN);
 225             XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_timer_val_us",
 226                 timer_val_us, qindex, XGE_DEFAULT_RING_QUEUE_RTI_TIMER_VAL_US);
 227             XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_urange_a",
 228                 urange_a, qindex, XGE_DEFAULT_RING_QUEUE_RTI_URANGE_A);
 229             XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_urange_b",
 230                 urange_b, qindex, XGE_DEFAULT_RING_QUEUE_RTI_URANGE_B);
 231             XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_urange_c",
 232                 urange_c, qindex, XGE_DEFAULT_RING_QUEUE_RTI_URANGE_C);
 233         }
 234
 235         if(dconfig->fifo.max_frags > (PAGE_SIZE/32)) {
 236             xge_os_printf("fifo_max_frags = %d", dconfig->fifo.max_frags)
 237             xge_os_printf("fifo_max_frags should be <= (PAGE_SIZE / 32) = %d",
 238                 (int)(PAGE_SIZE / 32))
 239             xge_os_printf("Using fifo_max_frags = %d", (int)(PAGE_SIZE / 32))
 240             dconfig->fifo.max_frags = (PAGE_SIZE / 32);
 241         }
 242
 243         checkdev = pci_find_device(VENDOR_ID_AMD, DEVICE_ID_8131_PCI_BRIDGE);
 244         if(checkdev != NULL) {
 245             /* Check Revision for 0x12 */
 246             revision = pci_read_config(checkdev,
 247                 xge_offsetof(xge_hal_pci_config_t, revision), 1);
 248             if(revision <= 0x12) {
 249                 /* Set mmrb_count to 1k and max splits = 2 */
 250                 dconfig->mmrb_count       = 1;
 251                 dconfig->max_splits_trans = XGE_HAL_THREE_SPLIT_TRANSACTION;
 252             }
 253         }
 254 }
 255
 256 /**
 257  * xge_buffer_sizes_set
 258  * Set buffer sizes based on Rx buffer mode
 259  *
 260  * @lldev Per-adapter Data
 261  * @buffer_mode Rx Buffer Mode
 262  */
 263 void
 264 xge_rx_buffer_sizes_set(xge_lldev_t *lldev, int buffer_mode, int mtu)
 265 {
 266         int index = 0;
 267         int frame_header = XGE_HAL_MAC_HEADER_MAX_SIZE;
 268         int buffer_size = mtu + frame_header;
 269
 270         xge_os_memzero(lldev->rxd_mbuf_len, sizeof(lldev->rxd_mbuf_len));
 271
 272         if(buffer_mode != XGE_HAL_RING_QUEUE_BUFFER_MODE_5)
 273             lldev->rxd_mbuf_len[buffer_mode - 1] = mtu;
 274
 275         lldev->rxd_mbuf_len[0] = (buffer_mode == 1) ? buffer_size:frame_header;
 276
 277         if(buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5)
 278             lldev->rxd_mbuf_len[1] = XGE_HAL_TCPIP_HEADER_MAX_SIZE;
 279
 280         if(buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5) {
 281             index = 2;
 282             buffer_size -= XGE_HAL_TCPIP_HEADER_MAX_SIZE;
 283             while(buffer_size > MJUMPAGESIZE) {
 284                 lldev->rxd_mbuf_len[index++] = MJUMPAGESIZE;
 285                 buffer_size -= MJUMPAGESIZE;
 286             }
 287             XGE_ALIGN_TO(buffer_size, 128);
 288             lldev->rxd_mbuf_len[index] = buffer_size;
 289             lldev->rxd_mbuf_cnt = index + 1;
 290         }
 291
 292         for(index = 0; index < buffer_mode; index++)
 293             xge_trace(XGE_TRACE, "Buffer[%d] %d\n", index,
 294                 lldev->rxd_mbuf_len[index]);
 295 }
 296
 297 /**
 298  * xge_buffer_mode_init
 299  * Init Rx buffer mode
 300  *
 301  * @lldev Per-adapter Data
 302  * @mtu Interface MTU
 303  */
 304 void
 305 xge_buffer_mode_init(xge_lldev_t *lldev, int mtu)
 306 {
 307         int index = 0, buffer_size = 0;
 308         xge_hal_ring_config_t *ring_config = &((lldev->devh)->config.ring);
 309
 310         buffer_size = mtu + XGE_HAL_MAC_HEADER_MAX_SIZE;
 311
 312         if(lldev->enabled_lro)
 313             (lldev->ifnetp)->if_capenable |= IFCAP_LRO;
 314         else
 315             (lldev->ifnetp)->if_capenable &= ~IFCAP_LRO;
 316
 317         lldev->rxd_mbuf_cnt = lldev->buffer_mode;
 318         if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_2) {
 319             XGE_SET_BUFFER_MODE_IN_RINGS(XGE_HAL_RING_QUEUE_BUFFER_MODE_3);
 320             ring_config->scatter_mode = XGE_HAL_RING_QUEUE_SCATTER_MODE_B;
 321         }
 322         else {
 323             XGE_SET_BUFFER_MODE_IN_RINGS(lldev->buffer_mode);
 324             ring_config->scatter_mode = XGE_HAL_RING_QUEUE_SCATTER_MODE_A;
 325         }
 326         xge_rx_buffer_sizes_set(lldev, lldev->buffer_mode, mtu);
 327
 328         xge_os_printf("%s: TSO %s", device_get_nameunit(lldev->device),
 329             ((lldev->enabled_tso) ? "Enabled":"Disabled"));
 330         xge_os_printf("%s: LRO %s", device_get_nameunit(lldev->device),
 331             ((lldev->ifnetp)->if_capenable & IFCAP_LRO) ? "Enabled":"Disabled");
 332         xge_os_printf("%s: Rx %d Buffer Mode Enabled",
 333             device_get_nameunit(lldev->device), lldev->buffer_mode);
 334 }
 335
 336 /**
 337  * xge_driver_initialize
 338  * Initializes HAL driver (common for all devices)
 339  *
 340  * Returns
 341  * XGE_HAL_OK if success
 342  * XGE_HAL_ERR_BAD_DRIVER_CONFIG if driver configuration parameters are invalid
 343  */
 344 int
 345 xge_driver_initialize(void)
 346 {
 347         xge_hal_uld_cbs_t       uld_callbacks;
 348         xge_hal_driver_config_t driver_config;
 349         xge_hal_status_e        status = XGE_HAL_OK;
 350
 351         /* Initialize HAL driver */
 352         if(!hal_driver_init_count) {
 353             xge_os_memzero(&uld_callbacks, sizeof(xge_hal_uld_cbs_t));
 354             xge_os_memzero(&driver_config, sizeof(xge_hal_driver_config_t));
 355
 356             /*
 357              * Initial and maximum size of the queue used to store the events
 358              * like Link up/down (xge_hal_event_e)
 359              */
 360             driver_config.queue_size_initial = XGE_HAL_MIN_QUEUE_SIZE_INITIAL;
 361             driver_config.queue_size_max     = XGE_HAL_MAX_QUEUE_SIZE_MAX;
 362
 363             uld_callbacks.link_up   = xge_callback_link_up;
 364             uld_callbacks.link_down = xge_callback_link_down;
 365             uld_callbacks.crit_err  = xge_callback_crit_err;
 366             uld_callbacks.event     = xge_callback_event;
 367
 368             status = xge_hal_driver_initialize(&driver_config, &uld_callbacks);
 369             if(status != XGE_HAL_OK) {
 370                 XGE_EXIT_ON_ERR("xgeX: Initialization of HAL driver failed",
 371                     xdi_out, status);
 372             }
 373         }
 374         hal_driver_init_count = hal_driver_init_count + 1;
 375
 376         xge_hal_driver_debug_module_mask_set(0xffffffff);
 377         xge_hal_driver_debug_level_set(XGE_TRACE);
 378
 379 xdi_out:
 380         return status;
 381 }
 382
 383 /**
 384  * xge_media_init
 385  * Initializes, adds and sets media
 386  *
 387  * @devc Device Handle
 388  */
 389 void
 390 xge_media_init(device_t devc)
 391 {
 392         xge_lldev_t *lldev = (xge_lldev_t *)device_get_softc(devc);
 393
 394         /* Initialize Media */
 395         ifmedia_init(&lldev->media, IFM_IMASK, xge_ifmedia_change,
 396             xge_ifmedia_status);
 397
 398         /* Add supported media */
 399         ifmedia_add(&lldev->media, IFM_ETHER | IFM_1000_SX | IFM_FDX, 0, NULL);
 400         ifmedia_add(&lldev->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
 401         ifmedia_add(&lldev->media, IFM_ETHER | IFM_AUTO,    0, NULL);
 402         ifmedia_add(&lldev->media, IFM_ETHER | IFM_10G_SR,  0, NULL);
 403         ifmedia_add(&lldev->media, IFM_ETHER | IFM_10G_LR,  0, NULL);
 404
 405         /* Set media */
 406         ifmedia_set(&lldev->media, IFM_ETHER | IFM_AUTO);
 407 }
 408
 409 /**
 410  * xge_pci_space_save
 411  * Save PCI configuration space
 412  *
 413  * @dev Device Handle
 414  */
 415 void
 416 xge_pci_space_save(device_t dev)
 417 {
 418         struct pci_devinfo *dinfo = NULL;
 419
 420         dinfo = device_get_ivars(dev);
 421         xge_trace(XGE_TRACE, "Saving PCI configuration space");
 422         pci_cfg_save(dev, dinfo, 0);
 423 }
 424
 425 /**
 426  * xge_pci_space_restore
 427  * Restore saved PCI configuration space
 428  *
 429  * @dev Device Handle
 430  */
 431 void
 432 xge_pci_space_restore(device_t dev)
 433 {
 434         struct pci_devinfo *dinfo = NULL;
 435
 436         dinfo = device_get_ivars(dev);
 437         xge_trace(XGE_TRACE, "Restoring PCI configuration space");
 438         pci_cfg_restore(dev, dinfo);
 439 }
 440
 441 /**
 442  * xge_msi_info_save
 443  * Save MSI info
 444  *
 445  * @lldev Per-adapter Data
 446  */
 447 void
 448 xge_msi_info_save(xge_lldev_t * lldev)
 449 {
 450         xge_os_pci_read16(lldev->pdev, NULL,
 451             xge_offsetof(xge_hal_pci_config_le_t, msi_control),
 452             &lldev->msi_info.msi_control);
 453         xge_os_pci_read32(lldev->pdev, NULL,
 454             xge_offsetof(xge_hal_pci_config_le_t, msi_lower_address),
 455             &lldev->msi_info.msi_lower_address);
 456         xge_os_pci_read32(lldev->pdev, NULL,
 457             xge_offsetof(xge_hal_pci_config_le_t, msi_higher_address),
 458             &lldev->msi_info.msi_higher_address);
 459         xge_os_pci_read16(lldev->pdev, NULL,
 460             xge_offsetof(xge_hal_pci_config_le_t, msi_data),
 461             &lldev->msi_info.msi_data);
 462 }
 463
 464 /**
 465  * xge_msi_info_restore
 466  * Restore saved MSI info
 467  *
 468  * @dev Device Handle
 469  */
 470 void
 471 xge_msi_info_restore(xge_lldev_t *lldev)
 472 {
 473         /*
 474          * If interface is made down and up, traffic fails. It was observed that
 475          * MSI information were getting reset on down. Restoring them.
 476          */
 477         xge_os_pci_write16(lldev->pdev, NULL,
 478             xge_offsetof(xge_hal_pci_config_le_t, msi_control),
 479             lldev->msi_info.msi_control);
 480
 481         xge_os_pci_write32(lldev->pdev, NULL,
 482             xge_offsetof(xge_hal_pci_config_le_t, msi_lower_address),
 483             lldev->msi_info.msi_lower_address);
 484
 485         xge_os_pci_write32(lldev->pdev, NULL,
 486             xge_offsetof(xge_hal_pci_config_le_t, msi_higher_address),
 487             lldev->msi_info.msi_higher_address);
 488
 489         xge_os_pci_write16(lldev->pdev, NULL,
 490             xge_offsetof(xge_hal_pci_config_le_t, msi_data),
 491             lldev->msi_info.msi_data);
 492 }
 493
 494 /**
 495  * xge_init_mutex
 496  * Initializes mutexes used in driver
 497  *
 498  * @lldev  Per-adapter Data
 499  */
 500 void
 501 xge_mutex_init(xge_lldev_t *lldev)
 502 {
 503         int qindex;
 504
 505         sprintf(lldev->mtx_name_drv, "%s_drv",
 506             device_get_nameunit(lldev->device));
 507         mtx_init(&lldev->mtx_drv, lldev->mtx_name_drv, MTX_NETWORK_LOCK,
 508             MTX_DEF);
 509
 510         for(qindex = 0; qindex < XGE_FIFO_COUNT; qindex++) {
 511             sprintf(lldev->mtx_name_tx[qindex], "%s_tx_%d",
 512                 device_get_nameunit(lldev->device), qindex);
 513             mtx_init(&lldev->mtx_tx[qindex], lldev->mtx_name_tx[qindex], NULL,
 514                 MTX_DEF);
 515         }
 516 }
 517
 518 /**
 519  * xge_mutex_destroy
 520  * Destroys mutexes used in driver
 521  *
 522  * @lldev Per-adapter Data
 523  */
 524 void
 525 xge_mutex_destroy(xge_lldev_t *lldev)
 526 {
 527         int qindex;
 528
 529         for(qindex = 0; qindex < XGE_FIFO_COUNT; qindex++)
 530             mtx_destroy(&lldev->mtx_tx[qindex]);
 531         mtx_destroy(&lldev->mtx_drv);
 532 }
 533
 534 /**
 535  * xge_print_info
 536  * Print device and driver information
 537  *
 538  * @lldev Per-adapter Data
 539  */
 540 void
 541 xge_print_info(xge_lldev_t *lldev)
 542 {
 543         device_t dev = lldev->device;
 544         xge_hal_device_t *hldev = lldev->devh;
 545         xge_hal_status_e status = XGE_HAL_OK;
 546         u64 val64 = 0;
 547         const char *xge_pci_bus_speeds[17] = {
 548             "PCI 33MHz Bus",
 549             "PCI 66MHz Bus",
 550             "PCIX(M1) 66MHz Bus",
 551             "PCIX(M1) 100MHz Bus",
 552             "PCIX(M1) 133MHz Bus",
 553             "PCIX(M2) 133MHz Bus",
 554             "PCIX(M2) 200MHz Bus",
 555             "PCIX(M2) 266MHz Bus",
 556             "PCIX(M1) Reserved",
 557             "PCIX(M1) 66MHz Bus (Not Supported)",
 558             "PCIX(M1) 100MHz Bus (Not Supported)",
 559             "PCIX(M1) 133MHz Bus (Not Supported)",
 560             "PCIX(M2) Reserved",
 561             "PCIX 533 Reserved",
 562             "PCI Basic Mode",
 563             "PCIX Basic Mode",
 564             "PCI Invalid Mode"
 565         };
 566
 567         xge_os_printf("%s: Xframe%s %s Revision %d Driver v%s",
 568             device_get_nameunit(dev),
 569             ((hldev->device_id == XGE_PCI_DEVICE_ID_XENA_2) ? "I" : "II"),
 570             hldev->vpd_data.product_name, hldev->revision, XGE_DRIVER_VERSION);
 571         xge_os_printf("%s: Serial Number %s",
 572             device_get_nameunit(dev), hldev->vpd_data.serial_num);
 573
 574         if(pci_get_device(dev) == XGE_PCI_DEVICE_ID_HERC_2) {
 575             status = xge_hal_mgmt_reg_read(hldev, 0,
 576                 xge_offsetof(xge_hal_pci_bar0_t, pci_info), &val64);
 577             if(status != XGE_HAL_OK)
 578                 xge_trace(XGE_ERR, "Error for getting bus speed");
 579
 580             xge_os_printf("%s: Adapter is on %s bit %s",
 581                 device_get_nameunit(dev), ((val64 & BIT(8)) ? "32":"64"),
 582                 (xge_pci_bus_speeds[((val64 & XGE_HAL_PCI_INFO) >> 60)]));
 583         }
 584
 585         xge_os_printf("%s: Using %s Interrupts",
 586             device_get_nameunit(dev),
 587             (lldev->enabled_msi == XGE_HAL_INTR_MODE_MSI) ? "MSI":"Line");
 588 }
 589
 590 /**
 591  * xge_create_dma_tags
 592  * Creates DMA tags for both Tx and Rx
 593  *
 594  * @dev Device Handle
 595  *
 596  * Returns XGE_HAL_OK or XGE_HAL_FAIL (if errors)
 597  */
 598 xge_hal_status_e
 599 xge_create_dma_tags(device_t dev)
 600 {
 601         xge_lldev_t *lldev = (xge_lldev_t *)device_get_softc(dev);
 602         xge_hal_status_e status = XGE_HAL_FAIL;
 603         int mtu = (lldev->ifnetp)->if_mtu, maxsize;
 604
 605         /* DMA tag for Tx */
 606         status = bus_dma_tag_create(
 607             bus_get_dma_tag(dev),                /* Parent                    */
 608             PAGE_SIZE,                           /* Alignment                 */
 609             0,                                   /* Bounds                    */
 610             BUS_SPACE_MAXADDR,                   /* Low Address               */
 611             BUS_SPACE_MAXADDR,                   /* High Address              */
 612             NULL,                                /* Filter Function           */
 613             NULL,                                /* Filter Function Arguments */
 614             MCLBYTES * XGE_MAX_SEGS,             /* Maximum Size              */
 615             XGE_MAX_SEGS,                        /* Number of Segments        */
 616             MCLBYTES,                            /* Maximum Segment Size      */
 617             BUS_DMA_ALLOCNOW,                    /* Flags                     */
 618             NULL,                                /* Lock Function             */
 619             NULL,                                /* Lock Function Arguments   */
 620             (&lldev->dma_tag_tx));               /* DMA Tag                   */
 621         if(status != 0)
 622             goto _exit;
 623
 624         maxsize = mtu + XGE_HAL_MAC_HEADER_MAX_SIZE;
 625         if(maxsize <= MCLBYTES) {
 626             maxsize = MCLBYTES;
 627         }
 628         else {
 629             if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5)
 630                 maxsize = MJUMPAGESIZE;
 631             else
 632                 maxsize = (maxsize <= MJUMPAGESIZE) ? MJUMPAGESIZE : MJUM9BYTES;
 633         }
 634
 635         /* DMA tag for Rx */
 636         status = bus_dma_tag_create(
 637             bus_get_dma_tag(dev),                /* Parent                    */
 638             PAGE_SIZE,                           /* Alignment                 */
 639             0,                                   /* Bounds                    */
 640             BUS_SPACE_MAXADDR,                   /* Low Address               */
 641             BUS_SPACE_MAXADDR,                   /* High Address              */
 642             NULL,                                /* Filter Function           */
 643             NULL,                                /* Filter Function Arguments */
 644             maxsize,                             /* Maximum Size              */
 645             1,                                   /* Number of Segments        */
 646             maxsize,                             /* Maximum Segment Size      */
 647             BUS_DMA_ALLOCNOW,                    /* Flags                     */
 648             NULL,                                /* Lock Function             */
 649             NULL,                                /* Lock Function Arguments   */
 650             (&lldev->dma_tag_rx));               /* DMA Tag                   */
 651         if(status != 0)
 652             goto _exit1;
 653
 654         status = bus_dmamap_create(lldev->dma_tag_rx, BUS_DMA_NOWAIT,
 655             &lldev->extra_dma_map);
 656         if(status != 0)
 657             goto _exit2;
 658
 659         status = XGE_HAL_OK;
 660         goto _exit;
 661
 662 _exit2:
 663         status = bus_dma_tag_destroy(lldev->dma_tag_rx);
 664         if(status != 0)
 665             xge_trace(XGE_ERR, "Rx DMA tag destroy failed");
 666 _exit1:
 667         status = bus_dma_tag_destroy(lldev->dma_tag_tx);
 668         if(status != 0)
 669             xge_trace(XGE_ERR, "Tx DMA tag destroy failed");
 670         status = XGE_HAL_FAIL;
 671 _exit:
 672         return status;
 673 }
 674
 675 /**
 676  * xge_confirm_changes
 677  * Disables and Enables interface to apply requested change
 678  *
 679  * @lldev Per-adapter Data
 680  * @mtu_set Is it called for changing MTU? (Yes: 1, No: 0)
 681  *
 682  * Returns 0 or Error Number
 683  */
 684 void
 685 xge_confirm_changes(xge_lldev_t *lldev, xge_option_e option)
 686 {
 687         if(lldev->initialized == 0) goto _exit1;
 688
 689         mtx_lock(&lldev->mtx_drv);
 690         if_down(lldev->ifnetp);
 691         xge_device_stop(lldev, XGE_HAL_CHANNEL_OC_NORMAL);
 692
 693         if(option == XGE_SET_MTU)
 694             (lldev->ifnetp)->if_mtu = lldev->mtu;
 695         else
 696             xge_buffer_mode_init(lldev, lldev->mtu);
 697
 698         xge_device_init(lldev, XGE_HAL_CHANNEL_OC_NORMAL);
 699         if_up(lldev->ifnetp);
 700         mtx_unlock(&lldev->mtx_drv);
 701         goto _exit;
 702
 703 _exit1:
 704         /* Request was to change MTU and device not initialized */
 705         if(option == XGE_SET_MTU) {
 706             (lldev->ifnetp)->if_mtu = lldev->mtu;
 707             xge_buffer_mode_init(lldev, lldev->mtu);
 708         }
 709 _exit:
 710         return;
 711 }
 712
 713 /**
 714  * xge_change_lro_status
 715  * Enable/Disable LRO feature
 716  *
 717  * @SYSCTL_HANDLER_ARGS sysctl_oid structure with arguments
 718  *
 719  * Returns 0 or error number.
 720  */
 721 static int
 722 xge_change_lro_status(SYSCTL_HANDLER_ARGS)
 723 {
 724         xge_lldev_t *lldev = (xge_lldev_t *)arg1;
 725         int request = lldev->enabled_lro, status = XGE_HAL_OK;
 726
 727         status = sysctl_handle_int(oidp, &request, arg2, req);
 728         if((status != XGE_HAL_OK) || (!req->newptr))
 729             goto _exit;
 730
 731         if((request < 0) || (request > 1)) {
 732             status = EINVAL;
 733             goto _exit;
 734         }
 735
 736         /* Return if current and requested states are same */
 737         if(request == lldev->enabled_lro){
 738             xge_trace(XGE_ERR, "LRO is already %s",
 739                 ((request) ? "enabled" : "disabled"));
 740             goto _exit;
 741         }
 742
 743         lldev->enabled_lro = request;
 744         xge_confirm_changes(lldev, XGE_CHANGE_LRO);
 745         arg2 = lldev->enabled_lro;
 746
 747 _exit:
 748         return status;
 749 }
 750
 751 /**
 752  * xge_add_sysctl_handlers
 753  * Registers sysctl parameter value update handlers
 754  *
 755  * @lldev Per-adapter data
 756  */
 757 void
 758 xge_add_sysctl_handlers(xge_lldev_t *lldev)
 759 {
 760         struct sysctl_ctx_list *context_list =
 761             device_get_sysctl_ctx(lldev->device);
 762         struct sysctl_oid *oid = device_get_sysctl_tree(lldev->device);
 763
 764         SYSCTL_ADD_PROC(context_list, SYSCTL_CHILDREN(oid), OID_AUTO,
 765             "enable_lro", CTLTYPE_INT | CTLFLAG_RW, lldev, 0,
 766             xge_change_lro_status, "I", "Enable or disable LRO feature");
 767 }
 768
 769 /**
 770  * xge_attach
 771  * Connects driver to the system if probe was success
 772  *
 773  * @dev Device Handle
 774  */
 775 int
 776 xge_attach(device_t dev)
 777 {
 778         xge_hal_device_config_t *device_config;
 779         xge_hal_device_attr_t   attr;
 780         xge_lldev_t             *lldev;
 781         xge_hal_device_t        *hldev;
 782         xge_pci_info_t          *pci_info;
 783         struct ifnet            *ifnetp;
 784         int                     rid, rid0, rid1, error;
 785         int                     msi_count = 0, status = XGE_HAL_OK;
 786         int                     enable_msi = XGE_HAL_INTR_MODE_IRQLINE;
 787
 788         device_config = xge_os_malloc(NULL, sizeof(xge_hal_device_config_t));
 789         if(!device_config) {
 790             XGE_EXIT_ON_ERR("Memory allocation for device configuration failed",
 791                 attach_out_config, ENOMEM);
 792         }
 793
 794         lldev = (xge_lldev_t *) device_get_softc(dev);
 795         if(!lldev) {
 796             XGE_EXIT_ON_ERR("Adapter softc is NULL", attach_out, ENOMEM);
 797         }
 798         lldev->device = dev;
 799
 800         xge_mutex_init(lldev);
 801
 802         error = xge_driver_initialize();
 803         if(error != XGE_HAL_OK) {
 804             xge_resources_free(dev, xge_free_mutex);
 805             XGE_EXIT_ON_ERR("Initializing driver failed", attach_out, ENXIO);
 806         }
 807
 808         /* HAL device */
 809         hldev =
 810             (xge_hal_device_t *)xge_os_malloc(NULL, sizeof(xge_hal_device_t));
 811         if(!hldev) {
 812             xge_resources_free(dev, xge_free_terminate_hal_driver);
 813             XGE_EXIT_ON_ERR("Memory allocation for HAL device failed",
 814                 attach_out, ENOMEM);
 815         }
 816         lldev->devh = hldev;
 817
 818         /* Our private structure */
 819         pci_info =
 820             (xge_pci_info_t*) xge_os_malloc(NULL, sizeof(xge_pci_info_t));
 821         if(!pci_info) {
 822             xge_resources_free(dev, xge_free_hal_device);
 823             XGE_EXIT_ON_ERR("Memory allocation for PCI info. failed",
 824                 attach_out, ENOMEM);
 825         }
 826         lldev->pdev      = pci_info;
 827         pci_info->device = dev;
 828
 829         /* Set bus master */
 830         pci_enable_busmaster(dev);
 831
 832         /* Get virtual address for BAR0 */
 833         rid0 = PCIR_BAR(0);
 834         pci_info->regmap0 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid0,
 835             RF_ACTIVE);
 836         if(pci_info->regmap0 == NULL) {
 837             xge_resources_free(dev, xge_free_pci_info);
 838             XGE_EXIT_ON_ERR("Bus resource allocation for BAR0 failed",
 839                 attach_out, ENOMEM);
 840         }
 841         attr.bar0 = (char *)pci_info->regmap0;
 842
 843         pci_info->bar0resource = (xge_bus_resource_t*)
 844             xge_os_malloc(NULL, sizeof(xge_bus_resource_t));
 845         if(pci_info->bar0resource == NULL) {
 846             xge_resources_free(dev, xge_free_bar0);
 847             XGE_EXIT_ON_ERR("Memory allocation for BAR0 Resources failed",
 848                 attach_out, ENOMEM);
 849         }
 850         ((xge_bus_resource_t *)(pci_info->bar0resource))->bus_tag =
 851             rman_get_bustag(pci_info->regmap0);
 852         ((xge_bus_resource_t *)(pci_info->bar0resource))->bus_handle =
 853             rman_get_bushandle(pci_info->regmap0);
 854         ((xge_bus_resource_t *)(pci_info->bar0resource))->bar_start_addr =
 855             pci_info->regmap0;
 856
 857         /* Get virtual address for BAR1 */
 858         rid1 = PCIR_BAR(2);
 859         pci_info->regmap1 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid1,
 860             RF_ACTIVE);
 861         if(pci_info->regmap1 == NULL) {
 862             xge_resources_free(dev, xge_free_bar0_resource);
 863             XGE_EXIT_ON_ERR("Bus resource allocation for BAR1 failed",
 864                 attach_out, ENOMEM);
 865         }
 866         attr.bar1 = (char *)pci_info->regmap1;
 867
 868         pci_info->bar1resource = (xge_bus_resource_t*)
 869             xge_os_malloc(NULL, sizeof(xge_bus_resource_t));
 870         if(pci_info->bar1resource == NULL) {
 871             xge_resources_free(dev, xge_free_bar1);
 872             XGE_EXIT_ON_ERR("Memory allocation for BAR1 Resources failed",
 873                 attach_out, ENOMEM);
 874         }
 875         ((xge_bus_resource_t *)(pci_info->bar1resource))->bus_tag =
 876             rman_get_bustag(pci_info->regmap1);
 877         ((xge_bus_resource_t *)(pci_info->bar1resource))->bus_handle =
 878             rman_get_bushandle(pci_info->regmap1);
 879         ((xge_bus_resource_t *)(pci_info->bar1resource))->bar_start_addr =
 880             pci_info->regmap1;
 881
 882         /* Save PCI config space */
 883         xge_pci_space_save(dev);
 884
 885         attr.regh0 = (xge_bus_resource_t *) pci_info->bar0resource;
 886         attr.regh1 = (xge_bus_resource_t *) pci_info->bar1resource;
 887         attr.irqh  = lldev->irqhandle;
 888         attr.cfgh  = pci_info;
 889         attr.pdev  = pci_info;
 890
 891         /* Initialize device configuration parameters */
 892         xge_init_params(device_config, dev);
 893
 894         rid = 0;
 895         if(lldev->enabled_msi) {
 896             /* Number of MSI messages supported by device */
 897             msi_count = pci_msi_count(dev);
 898             if(msi_count > 1) {
 899                 /* Device supports MSI */
 900                 if(bootverbose) {
 901                     xge_trace(XGE_ERR, "MSI count: %d", msi_count);
 902                     xge_trace(XGE_ERR, "Now, driver supporting 1 message");
 903                 }
 904                 msi_count = 1;
 905                 error = pci_alloc_msi(dev, &msi_count);
 906                 if(error == 0) {
 907                     if(bootverbose)
 908                         xge_trace(XGE_ERR, "Allocated messages: %d", msi_count);
 909                     enable_msi = XGE_HAL_INTR_MODE_MSI;
 910                     rid = 1;
 911                 }
 912                 else {
 913                     if(bootverbose)
 914                         xge_trace(XGE_ERR, "pci_alloc_msi failed, %d", error);
 915                 }
 916             }
 917         }
 918         lldev->enabled_msi = enable_msi;
 919
 920         /* Allocate resource for irq */
 921         lldev->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 922             (RF_SHAREABLE | RF_ACTIVE));
 923         if(lldev->irq == NULL) {
 924             xge_trace(XGE_ERR, "Allocating irq resource for %s failed",
 925                 ((rid == 0) ? "line interrupt" : "MSI"));
 926             if(rid == 1) {
 927                 error = pci_release_msi(dev);
 928                 if(error != 0) {
 929                     xge_trace(XGE_ERR, "Releasing MSI resources failed %d",
 930                         error);
 931                     xge_trace(XGE_ERR, "Requires reboot to use MSI again");
 932                 }
 933                 xge_trace(XGE_ERR, "Trying line interrupts");
 934                 rid = 0;
 935                 lldev->enabled_msi = XGE_HAL_INTR_MODE_IRQLINE;
 936                 lldev->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 937                     (RF_SHAREABLE | RF_ACTIVE));
 938             }
 939             if(lldev->irq == NULL) {
 940                 xge_trace(XGE_ERR, "Allocating irq resource failed");
 941                 xge_resources_free(dev, xge_free_bar1_resource);
 942                 status = ENOMEM;
 943                 goto attach_out;
 944             }
 945         }
 946
 947         device_config->intr_mode = lldev->enabled_msi;
 948         if(bootverbose) {
 949             xge_trace(XGE_TRACE, "rid: %d, Mode: %d, MSI count: %d", rid,
 950                 lldev->enabled_msi, msi_count);
 951         }
 952
 953         /* Initialize HAL device */
 954         error = xge_hal_device_initialize(hldev, &attr, device_config);
 955         if(error != XGE_HAL_OK) {
 956             xge_resources_free(dev, xge_free_irq_resource);
 957             XGE_EXIT_ON_ERR("Initializing HAL device failed", attach_out,
 958                 ENXIO);
 959         }
 960
 961         xge_hal_device_private_set(hldev, lldev);
 962
 963         error = xge_interface_setup(dev);
 964         if(error != 0) {
 965             status = error;
 966             goto attach_out;
 967         }
 968
 969         ifnetp         = lldev->ifnetp;
 970         ifnetp->if_mtu = device_config->mtu;
 971
 972         xge_media_init(dev);
 973
 974         /* Associate interrupt handler with the device */
 975         if(lldev->enabled_msi == XGE_HAL_INTR_MODE_MSI) {
 976             error = bus_setup_intr(dev, lldev->irq,
 977                 (INTR_TYPE_NET | INTR_MPSAFE),
 978 #if __FreeBSD_version > 700030
 979                 NULL,
 980 #endif
 981                 xge_isr_msi, lldev, &lldev->irqhandle);
 982             xge_msi_info_save(lldev);
 983         }
 984         else {
 985             error = bus_setup_intr(dev, lldev->irq,
 986                 (INTR_TYPE_NET | INTR_MPSAFE),
 987 #if __FreeBSD_version > 700030
 988                 xge_isr_filter,
 989 #endif
 990                 xge_isr_line, lldev, &lldev->irqhandle);
 991         }
 992         if(error != 0) {
 993             xge_resources_free(dev, xge_free_media_interface);
 994             XGE_EXIT_ON_ERR("Associating interrupt handler with device failed",
 995                 attach_out, ENXIO);
 996         }
 997
 998         xge_print_info(lldev);
 999
1000         xge_add_sysctl_handlers(lldev);
1001
1002         xge_buffer_mode_init(lldev, device_config->mtu);
1003
1004 attach_out:
1005         xge_os_free(NULL, device_config, sizeof(xge_hal_device_config_t));
1006 attach_out_config:
1007         return status;
1008 }
1009
1010 /**
1011  * xge_resources_free
1012  * Undo what-all we did during load/attach
1013  *
1014  * @dev Device Handle
1015  * @error Identifies what-all to undo
1016  */
1017 void
1018 xge_resources_free(device_t dev, xge_lables_e error)
1019 {
1020         xge_lldev_t *lldev;
1021         xge_pci_info_t *pci_info;
1022         xge_hal_device_t *hldev;
1023         int rid, status;
1024
1025         /* LL Device */
1026         lldev = (xge_lldev_t *) device_get_softc(dev);
1027         pci_info = lldev->pdev;
1028
1029         /* HAL Device */
1030         hldev = lldev->devh;
1031
1032         switch(error) {
1033             case xge_free_all:
1034                 /* Teardown interrupt handler - device association */
1035                 bus_teardown_intr(dev, lldev->irq, lldev->irqhandle);
1036
1037             case xge_free_media_interface:
1038                 /* Media */
1039                 ifmedia_removeall(&lldev->media);
1040
1041                 /* Detach Ether */
1042                 ether_ifdetach(lldev->ifnetp);
1043                 if_free(lldev->ifnetp);
1044
1045                 xge_hal_device_private_set(hldev, NULL);
1046                 xge_hal_device_disable(hldev);
1047
1048             case xge_free_terminate_hal_device:
1049                 /* HAL Device */
1050                 xge_hal_device_terminate(hldev);
1051
1052             case xge_free_irq_resource:
1053                 /* Release IRQ resource */
1054                 bus_release_resource(dev, SYS_RES_IRQ,
1055                     ((lldev->enabled_msi == XGE_HAL_INTR_MODE_IRQLINE) ? 0:1),
1056                     lldev->irq);
1057
1058                 if(lldev->enabled_msi == XGE_HAL_INTR_MODE_MSI) {
1059                     status = pci_release_msi(dev);
1060                     if(status != 0) {
1061                         if(bootverbose) {
1062                             xge_trace(XGE_ERR,
1063                                 "pci_release_msi returned %d", status);
1064                         }
1065                     }
1066                 }
1067
1068             case xge_free_bar1_resource:
1069                 /* Restore PCI configuration space */
1070                 xge_pci_space_restore(dev);
1071
1072                 /* Free bar1resource */
1073                 xge_os_free(NULL, pci_info->bar1resource,
1074                     sizeof(xge_bus_resource_t));
1075
1076             case xge_free_bar1:
1077                 /* Release BAR1 */
1078                 rid = PCIR_BAR(2);
1079                 bus_release_resource(dev, SYS_RES_MEMORY, rid,
1080                     pci_info->regmap1);
1081
1082             case xge_free_bar0_resource:
1083                 /* Free bar0resource */
1084                 xge_os_free(NULL, pci_info->bar0resource,
1085                     sizeof(xge_bus_resource_t));
1086
1087             case xge_free_bar0:
1088                 /* Release BAR0 */
1089                 rid = PCIR_BAR(0);
1090                 bus_release_resource(dev, SYS_RES_MEMORY, rid,
1091                     pci_info->regmap0);
1092
1093             case xge_free_pci_info:
1094                 /* Disable Bus Master */
1095                 pci_disable_busmaster(dev);
1096
1097                 /* Free pci_info_t */
1098                 lldev->pdev = NULL;
1099                 xge_os_free(NULL, pci_info, sizeof(xge_pci_info_t));
1100
1101             case xge_free_hal_device:
1102                 /* Free device configuration struct and HAL device */
1103                 xge_os_free(NULL, hldev, sizeof(xge_hal_device_t));
1104
1105             case xge_free_terminate_hal_driver:
1106                 /* Terminate HAL driver */
1107                 hal_driver_init_count = hal_driver_init_count - 1;
1108                 if(!hal_driver_init_count) {
1109                     xge_hal_driver_terminate();
1110                 }
1111
1112             case xge_free_mutex:
1113                 xge_mutex_destroy(lldev);
1114         }
1115 }
1116
1117 /**
1118  * xge_detach
1119  * Detaches driver from the Kernel subsystem
1120  *
1121  * @dev Device Handle
1122  */
1123 int
1124 xge_detach(device_t dev)
1125 {
1126         xge_lldev_t *lldev = (xge_lldev_t *)device_get_softc(dev);
1127
1128         if(lldev->in_detach == 0) {
1129             lldev->in_detach = 1;
1130             xge_stop(lldev);
1131             xge_resources_free(dev, xge_free_all);
1132         }
1133
1134         return 0;
1135 }
1136
1137 /**
1138  * xge_shutdown
1139  * To shutdown device before system shutdown
1140  *
1141  * @dev Device Handle
1142  */
1143 int
1144 xge_shutdown(device_t dev)
1145 {
1146         xge_lldev_t *lldev = (xge_lldev_t *) device_get_softc(dev);
1147         xge_stop(lldev);
1148
1149         return 0;
1150 }
1151
1152 /**
1153  * xge_interface_setup
1154  * Setup interface
1155  *
1156  * @dev Device Handle
1157  *
1158  * Returns 0 on success, ENXIO/ENOMEM on failure
1159  */
1160 int
1161 xge_interface_setup(device_t dev)
1162 {
1163         u8 mcaddr[ETHER_ADDR_LEN];
1164         xge_hal_status_e status;
1165         xge_lldev_t *lldev = (xge_lldev_t *)device_get_softc(dev);
1166         struct ifnet *ifnetp;
1167         xge_hal_device_t *hldev = lldev->devh;
1168
1169         /* Get the MAC address of the device */
1170         status = xge_hal_device_macaddr_get(hldev, 0, &mcaddr);
1171         if(status != XGE_HAL_OK) {
1172             xge_resources_free(dev, xge_free_terminate_hal_device);
1173             XGE_EXIT_ON_ERR("Getting MAC address failed", ifsetup_out, ENXIO);
1174         }
1175
1176         /* Get interface ifnet structure for this Ether device */
1177         ifnetp = lldev->ifnetp = if_alloc(IFT_ETHER);
1178         if(ifnetp == NULL) {
1179             xge_resources_free(dev, xge_free_terminate_hal_device);
1180             XGE_EXIT_ON_ERR("Allocation ifnet failed", ifsetup_out, ENOMEM);
1181         }
1182
1183         /* Initialize interface ifnet structure */
1184         if_initname(ifnetp, device_get_name(dev), device_get_unit(dev));
1185         ifnetp->if_mtu      = XGE_HAL_DEFAULT_MTU;
1186         ifnetp->if_baudrate = XGE_BAUDRATE;
1187         ifnetp->if_init     = xge_init;
1188         ifnetp->if_softc    = lldev;
1189         ifnetp->if_flags    = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1190         ifnetp->if_ioctl    = xge_ioctl;
1191         ifnetp->if_start    = xge_send;
1192
1193         /* TODO: Check and assign optimal value */
1194         ifnetp->if_snd.ifq_maxlen = ifqmaxlen;
1195
1196         ifnetp->if_capabilities = IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU |
1197             IFCAP_HWCSUM;
1198         if(lldev->enabled_tso)
1199             ifnetp->if_capabilities |= IFCAP_TSO4;
1200         if(lldev->enabled_lro)
1201             ifnetp->if_capabilities |= IFCAP_LRO;
1202
1203         ifnetp->if_capenable = ifnetp->if_capabilities;
1204
1205         /* Attach the interface */
1206         ether_ifattach(ifnetp, mcaddr);
1207
1208 ifsetup_out:
1209         return status;
1210 }
1211
1212 /**
1213  * xge_callback_link_up
1214  * Callback for Link-up indication from HAL
1215  *
1216  * @userdata Per-adapter data
1217  */
1218 void
1219 xge_callback_link_up(void *userdata)
1220 {
1221         xge_lldev_t  *lldev  = (xge_lldev_t *)userdata;
1222         struct ifnet *ifnetp = lldev->ifnetp;
1223
1224         ifnetp->if_flags  &= ~IFF_DRV_OACTIVE;
1225         if_link_state_change(ifnetp, LINK_STATE_UP);
1226 }
1227
1228 /**
1229  * xge_callback_link_down
1230  * Callback for Link-down indication from HAL
1231  *
1232  * @userdata Per-adapter data
1233  */
1234 void
1235 xge_callback_link_down(void *userdata)
1236 {
1237         xge_lldev_t  *lldev  = (xge_lldev_t *)userdata;
1238         struct ifnet *ifnetp = lldev->ifnetp;
1239
1240         ifnetp->if_flags  |= IFF_DRV_OACTIVE;
1241         if_link_state_change(ifnetp, LINK_STATE_DOWN);
1242 }
1243
1244 /**
1245  * xge_callback_crit_err
1246  * Callback for Critical error indication from HAL
1247  *
1248  * @userdata Per-adapter data
1249  * @type Event type (Enumerated hardware error)
1250  * @serr_data Hardware status
1251  */
1252 void
1253 xge_callback_crit_err(void *userdata, xge_hal_event_e type, u64 serr_data)
1254 {
1255         xge_trace(XGE_ERR, "Critical Error");
1256         xge_reset(userdata);
1257 }
1258
1259 /**
1260  * xge_callback_event
1261  * Callback from HAL indicating that some event has been queued
1262  *
1263  * @item Queued event item
1264  */
1265 void
1266 xge_callback_event(xge_queue_item_t *item)
1267 {
1268         xge_lldev_t      *lldev  = NULL;
1269         xge_hal_device_t *hldev  = NULL;
1270         struct ifnet     *ifnetp = NULL;
1271
1272         hldev  = item->context;
1273         lldev  = xge_hal_device_private(hldev);
1274         ifnetp = lldev->ifnetp;
1275
1276         switch((int)item->event_type) {
1277             case XGE_LL_EVENT_TRY_XMIT_AGAIN:
1278                 if(lldev->initialized) {
1279                     if(xge_hal_channel_dtr_count(lldev->fifo_channel[0]) > 0) {
1280                         ifnetp->if_flags  &= ~IFF_DRV_OACTIVE;
1281                     }
1282                     else {
1283                         xge_queue_produce_context(
1284                             xge_hal_device_queue(lldev->devh),
1285                             XGE_LL_EVENT_TRY_XMIT_AGAIN, lldev->devh);
1286                     }
1287                 }
1288                 break;
1289
1290             case XGE_LL_EVENT_DEVICE_RESETTING:
1291                 xge_reset(item->context);
1292                 break;
1293
1294             default:
1295                 break;
1296         }
1297 }
1298
1299 /**
1300  * xge_ifmedia_change
1301  * Media change driver callback
1302  *
1303  * @ifnetp Interface Handle
1304  *
1305  * Returns 0 if media is Ether else EINVAL
1306  */
1307 int
1308 xge_ifmedia_change(struct ifnet *ifnetp)
1309 {
1310         xge_lldev_t    *lldev    = ifnetp->if_softc;
1311         struct ifmedia *ifmediap = &lldev->media;
1312
1313         return (IFM_TYPE(ifmediap->ifm_media) != IFM_ETHER) ?  EINVAL:0;
1314 }
1315
1316 /**
1317  * xge_ifmedia_status
1318  * Media status driver callback
1319  *
1320  * @ifnetp Interface Handle
1321  * @ifmr Interface Media Settings
1322  */
1323 void
1324 xge_ifmedia_status(struct ifnet *ifnetp, struct ifmediareq *ifmr)
1325 {
1326         xge_hal_status_e status;
1327         u64              regvalue;
1328         xge_lldev_t      *lldev = ifnetp->if_softc;
1329         xge_hal_device_t *hldev = lldev->devh;
1330
1331         ifmr->ifm_status = IFM_AVALID;
1332         ifmr->ifm_active = IFM_ETHER;
1333
1334         status = xge_hal_mgmt_reg_read(hldev, 0,
1335             xge_offsetof(xge_hal_pci_bar0_t, adapter_status), &regvalue);
1336         if(status != XGE_HAL_OK) {
1337             xge_trace(XGE_TRACE, "Getting adapter status failed");
1338             goto _exit;
1339         }
1340
1341         if((regvalue & (XGE_HAL_ADAPTER_STATUS_RMAC_REMOTE_FAULT |
1342             XGE_HAL_ADAPTER_STATUS_RMAC_LOCAL_FAULT)) == 0) {
1343             ifmr->ifm_status |= IFM_ACTIVE;
1344             ifmr->ifm_active |= IFM_10G_SR | IFM_FDX;
1345             if_link_state_change(ifnetp, LINK_STATE_UP);
1346         }
1347         else {
1348             if_link_state_change(ifnetp, LINK_STATE_DOWN);
1349         }
1350 _exit:
1351         return;
1352 }
1353
1354 /**
1355  * xge_ioctl_stats
1356  * IOCTL to get statistics
1357  *
1358  * @lldev Per-adapter data
1359  * @ifreqp Interface request
1360  */
1361 int
1362 xge_ioctl_stats(xge_lldev_t *lldev, struct ifreq *ifreqp)
1363 {
1364         xge_hal_status_e status = XGE_HAL_OK;
1365         char *data = (char *)ifreqp->ifr_data;
1366         void *info = NULL;
1367         int retValue = EINVAL;
1368
1369         switch(*data) {
1370             case XGE_QUERY_STATS:
1371                 mtx_lock(&lldev->mtx_drv);
1372                 status = xge_hal_stats_hw(lldev->devh,
1373                     (xge_hal_stats_hw_info_t **)&info);
1374                 mtx_unlock(&lldev->mtx_drv);
1375                 if(status == XGE_HAL_OK) {
1376                     if(copyout(info, ifreqp->ifr_data,
1377                         sizeof(xge_hal_stats_hw_info_t)) == 0)
1378                         retValue = 0;
1379                 }
1380                 else {
1381                     xge_trace(XGE_ERR, "Getting statistics failed (Status: %d)",
1382                         status);
1383                 }
1384                 break;
1385
1386             case XGE_QUERY_PCICONF:
1387                 info = xge_os_malloc(NULL, sizeof(xge_hal_pci_config_t));
1388                 if(info != NULL) {
1389                     mtx_lock(&lldev->mtx_drv);
1390                     status = xge_hal_mgmt_pci_config(lldev->devh, info,
1391                         sizeof(xge_hal_pci_config_t));
1392                     mtx_unlock(&lldev->mtx_drv);
1393                     if(status == XGE_HAL_OK) {
1394                         if(copyout(info, ifreqp->ifr_data,
1395                             sizeof(xge_hal_pci_config_t)) == 0)
1396                             retValue = 0;
1397                     }
1398                     else {
1399                         xge_trace(XGE_ERR,
1400                             "Getting PCI configuration failed (%d)", status);
1401                     }
1402                     xge_os_free(NULL, info, sizeof(xge_hal_pci_config_t));
1403                 }
1404                 break;
1405
1406             case XGE_QUERY_DEVSTATS:
1407                 info = xge_os_malloc(NULL, sizeof(xge_hal_stats_device_info_t));
1408                 if(info != NULL) {
1409                     mtx_lock(&lldev->mtx_drv);
1410                     status =xge_hal_mgmt_device_stats(lldev->devh, info,
1411                         sizeof(xge_hal_stats_device_info_t));
1412                     mtx_unlock(&lldev->mtx_drv);
1413                     if(status == XGE_HAL_OK) {
1414                         if(copyout(info, ifreqp->ifr_data,
1415                             sizeof(xge_hal_stats_device_info_t)) == 0)
1416                             retValue = 0;
1417                     }
1418                     else {
1419                         xge_trace(XGE_ERR, "Getting device info failed (%d)",
1420                             status);
1421                     }
1422                     xge_os_free(NULL, info,
1423                         sizeof(xge_hal_stats_device_info_t));
1424                 }
1425                 break;
1426
1427             case XGE_QUERY_SWSTATS:
1428                 info = xge_os_malloc(NULL, sizeof(xge_hal_stats_sw_err_t));
1429                 if(info != NULL) {
1430                     mtx_lock(&lldev->mtx_drv);
1431                     status =xge_hal_mgmt_sw_stats(lldev->devh, info,
1432                         sizeof(xge_hal_stats_sw_err_t));
1433                     mtx_unlock(&lldev->mtx_drv);
1434                     if(status == XGE_HAL_OK) {
1435                         if(copyout(info, ifreqp->ifr_data,
1436                             sizeof(xge_hal_stats_sw_err_t)) == 0)
1437                             retValue = 0;
1438                     }
1439                     else {
1440                         xge_trace(XGE_ERR,
1441                             "Getting tcode statistics failed (%d)", status);
1442                     }
1443                     xge_os_free(NULL, info, sizeof(xge_hal_stats_sw_err_t));
1444                 }
1445                 break;
1446
1447             case XGE_QUERY_DRIVERSTATS:
1448                 if(copyout(&lldev->driver_stats, ifreqp->ifr_data,
1449                     sizeof(xge_driver_stats_t)) == 0) {
1450                     retValue = 0;
1451                 }
1452                 else {
1453                     xge_trace(XGE_ERR,
1454                         "Copyout of driver statistics failed (%d)", status);
1455                 }
1456                 break;
1457
1458             case XGE_READ_VERSION:
1459                 info = xge_os_malloc(NULL, XGE_BUFFER_SIZE);
1460                 if(info != NULL) {
1461                     strcpy(info, XGE_DRIVER_VERSION);
1462                     if(copyout(info, ifreqp->ifr_data, XGE_BUFFER_SIZE) == 0)
1463                         retValue = 0;
1464                     xge_os_free(NULL, info, XGE_BUFFER_SIZE);
1465                 }
1466                 break;
1467
1468             case XGE_QUERY_DEVCONF:
1469                 info = xge_os_malloc(NULL, sizeof(xge_hal_device_config_t));
1470                 if(info != NULL) {
1471                     mtx_lock(&lldev->mtx_drv);
1472                     status = xge_hal_mgmt_device_config(lldev->devh, info,
1473                         sizeof(xge_hal_device_config_t));
1474                     mtx_unlock(&lldev->mtx_drv);
1475                     if(status == XGE_HAL_OK) {
1476                         if(copyout(info, ifreqp->ifr_data,
1477                             sizeof(xge_hal_device_config_t)) == 0)
1478                             retValue = 0;
1479                     }
1480                     else {
1481                         xge_trace(XGE_ERR, "Getting devconfig failed (%d)",
1482                             status);
1483                     }
1484                     xge_os_free(NULL, info, sizeof(xge_hal_device_config_t));
1485                 }
1486                 break;
1487
1488             case XGE_QUERY_BUFFER_MODE:
1489                 if(copyout(&lldev->buffer_mode, ifreqp->ifr_data,
1490                     sizeof(int)) == 0)
1491                     retValue = 0;
1492                 break;
1493
1494             case XGE_SET_BUFFER_MODE_1:
1495             case XGE_SET_BUFFER_MODE_2:
1496             case XGE_SET_BUFFER_MODE_5:
1497                 *data = (*data == XGE_SET_BUFFER_MODE_1) ? 'Y':'N';
1498                 if(copyout(data, ifreqp->ifr_data, sizeof(data)) == 0)
1499                     retValue = 0;
1500                 break;
1501             default:
1502                 xge_trace(XGE_TRACE, "Nothing is matching");
1503                 retValue = ENOTTY;
1504                 break;
1505         }
1506         return retValue;
1507 }
1508
1509 /**
1510  * xge_ioctl_registers
1511  * IOCTL to get registers
1512  *
1513  * @lldev Per-adapter data
1514  * @ifreqp Interface request
1515  */
1516 int
1517 xge_ioctl_registers(xge_lldev_t *lldev, struct ifreq *ifreqp)
1518 {
1519         xge_register_t *data = (xge_register_t *)ifreqp->ifr_data;
1520         xge_hal_status_e status = XGE_HAL_OK;
1521         int retValue = EINVAL, offset = 0, index = 0;
1522         u64 val64 = 0;
1523
1524         /* Reading a register */
1525         if(strcmp(data->option, "-r") == 0) {
1526             data->value = 0x0000;
1527             mtx_lock(&lldev->mtx_drv);
1528             status = xge_hal_mgmt_reg_read(lldev->devh, 0, data->offset,
1529                 &data->value);
1530             mtx_unlock(&lldev->mtx_drv);
1531             if(status == XGE_HAL_OK) {
1532                 if(copyout(data, ifreqp->ifr_data, sizeof(xge_register_t)) == 0)
1533                     retValue = 0;
1534             }
1535         }
1536         /* Writing to a register */
1537         else if(strcmp(data->option, "-w") == 0) {
1538             mtx_lock(&lldev->mtx_drv);
1539             status = xge_hal_mgmt_reg_write(lldev->devh, 0, data->offset,
1540                 data->value);
1541             if(status == XGE_HAL_OK) {
1542                 val64 = 0x0000;
1543                 status = xge_hal_mgmt_reg_read(lldev->devh, 0, data->offset,
1544                     &val64);
1545                 if(status != XGE_HAL_OK) {
1546                     xge_trace(XGE_ERR, "Reading back updated register failed");
1547                 }
1548                 else {
1549                     if(val64 != data->value) {
1550                         xge_trace(XGE_ERR,
1551                             "Read and written register values mismatched");
1552                     }
1553                     else retValue = 0;
1554                 }
1555             }
1556             else {
1557                 xge_trace(XGE_ERR, "Getting register value failed");
1558             }
1559             mtx_unlock(&lldev->mtx_drv);
1560         }
1561         else {
1562             mtx_lock(&lldev->mtx_drv);
1563             for(index = 0, offset = 0; offset <= XGE_OFFSET_OF_LAST_REG;
1564                 index++, offset += 0x0008) {
1565                 val64 = 0;
1566                 status = xge_hal_mgmt_reg_read(lldev->devh, 0, offset, &val64);
1567                 if(status != XGE_HAL_OK) {
1568                     xge_trace(XGE_ERR, "Getting register value failed");
1569                     break;
1570                 }
1571                 *((u64 *)((u64 *)data + index)) = val64;
1572                 retValue = 0;
1573             }
1574             mtx_unlock(&lldev->mtx_drv);
1575
1576             if(retValue == 0) {
1577                 if(copyout(data, ifreqp->ifr_data,
1578                     sizeof(xge_hal_pci_bar0_t)) != 0) {
1579                     xge_trace(XGE_ERR, "Copyout of register values failed");
1580                     retValue = EINVAL;
1581                 }
1582             }
1583             else {
1584                 xge_trace(XGE_ERR, "Getting register values failed");
1585             }
1586         }
1587         return retValue;
1588 }
1589
1590 /**
1591  * xge_ioctl
1592  * Callback to control the device - Interface configuration
1593  *
1594  * @ifnetp Interface Handle
1595  * @command Device control command
1596  * @data Parameters associated with command (if any)
1597  */
1598 int
1599 xge_ioctl(struct ifnet *ifnetp, unsigned long command, caddr_t data)
1600 {
1601         struct ifreq   *ifreqp   = (struct ifreq *)data;
1602         xge_lldev_t    *lldev    = ifnetp->if_softc;
1603         struct ifmedia *ifmediap = &lldev->media;
1604         int             retValue = 0, mask = 0;
1605
1606         if(lldev->in_detach) {
1607             return retValue;
1608         }
1609
1610         switch(command) {
1611             /* Set/Get ifnet address */
1612             case SIOCSIFADDR:
1613             case SIOCGIFADDR:
1614                 ether_ioctl(ifnetp, command, data);
1615                 break;
1616
1617             /* Set ifnet MTU */
1618             case SIOCSIFMTU:
1619                 retValue = xge_change_mtu(lldev, ifreqp->ifr_mtu);
1620                 break;
1621
1622             /* Set ifnet flags */
1623             case SIOCSIFFLAGS:
1624                 if(ifnetp->if_flags & IFF_UP) {
1625                     /* Link status is UP */
1626                     if(!(ifnetp->if_drv_flags & IFF_DRV_RUNNING)) {
1627                         xge_init(lldev);
1628                     }
1629                     xge_disable_promisc(lldev);
1630                     xge_enable_promisc(lldev);
1631                 }
1632                 else {
1633                     /* Link status is DOWN */
1634                     /* If device is in running, make it down */
1635                     if(ifnetp->if_drv_flags & IFF_DRV_RUNNING) {
1636                         xge_stop(lldev);
1637                     }
1638                 }
1639                 break;
1640
1641             /* Add/delete multicast address */
1642             case SIOCADDMULTI:
1643             case SIOCDELMULTI:
1644                 if(ifnetp->if_drv_flags & IFF_DRV_RUNNING) {
1645                     xge_setmulti(lldev);
1646                 }
1647                 break;
1648
1649             /* Set/Get net media */
1650             case SIOCSIFMEDIA:
1651             case SIOCGIFMEDIA:
1652                 retValue = ifmedia_ioctl(ifnetp, ifreqp, ifmediap, command);
1653                 break;
1654
1655             /* Set capabilities */
1656             case SIOCSIFCAP:
1657                 mtx_lock(&lldev->mtx_drv);
1658                 mask = ifreqp->ifr_reqcap ^ ifnetp->if_capenable;
1659                 if(mask & IFCAP_TXCSUM) {
1660                     if(ifnetp->if_capenable & IFCAP_TXCSUM) {
1661                         ifnetp->if_capenable &= ~(IFCAP_TSO4 | IFCAP_TXCSUM);
1662                         ifnetp->if_hwassist &=
1663                             ~(CSUM_TCP | CSUM_UDP | CSUM_TSO);
1664                     }
1665                     else {
1666                         ifnetp->if_capenable |= IFCAP_TXCSUM;
1667                         ifnetp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1668                     }
1669                 }
1670                 if(mask & IFCAP_TSO4) {
1671                     if(ifnetp->if_capenable & IFCAP_TSO4) {
1672                         ifnetp->if_capenable &= ~IFCAP_TSO4;
1673                         ifnetp->if_hwassist  &= ~CSUM_TSO;
1674
1675                         xge_os_printf("%s: TSO Disabled",
1676                             device_get_nameunit(lldev->device));
1677                     }
1678                     else if(ifnetp->if_capenable & IFCAP_TXCSUM) {
1679                         ifnetp->if_capenable |= IFCAP_TSO4;
1680                         ifnetp->if_hwassist  |= CSUM_TSO;
1681
1682                         xge_os_printf("%s: TSO Enabled",
1683                             device_get_nameunit(lldev->device));
1684                     }
1685                 }
1686
1687                 mtx_unlock(&lldev->mtx_drv);
1688                 break;
1689
1690             /* Custom IOCTL 0 */
1691             case SIOCGPRIVATE_0:
1692                 retValue = xge_ioctl_stats(lldev, ifreqp);
1693                 break;
1694
1695             /* Custom IOCTL 1 */
1696             case SIOCGPRIVATE_1:
1697                 retValue = xge_ioctl_registers(lldev, ifreqp);
1698                 break;
1699
1700             default:
1701                 retValue = EINVAL;
1702                 break;
1703         }
1704         return retValue;
1705 }
1706
1707 /**
1708  * xge_init
1709  * Initialize the interface
1710  *
1711  * @plldev Per-adapter Data
1712  */
1713 void
1714 xge_init(void *plldev)
1715 {
1716         xge_lldev_t *lldev = (xge_lldev_t *)plldev;
1717
1718         mtx_lock(&lldev->mtx_drv);
1719         xge_os_memzero(&lldev->driver_stats, sizeof(xge_driver_stats_t));
1720         xge_device_init(lldev, XGE_HAL_CHANNEL_OC_NORMAL);
1721         mtx_unlock(&lldev->mtx_drv);
1722 }
1723
1724 /**
1725  * xge_device_init
1726  * Initialize the interface (called by holding lock)
1727  *
1728  * @pdevin Per-adapter Data
1729  */
1730 void
1731 xge_device_init(xge_lldev_t *lldev, xge_hal_channel_reopen_e option)
1732 {
1733         struct ifnet     *ifnetp = lldev->ifnetp;
1734         xge_hal_device_t *hldev  = lldev->devh;
1735         struct ifaddr      *ifaddrp;
1736         unsigned char      *macaddr;
1737         struct sockaddr_dl *sockaddrp;
1738         int                 status   = XGE_HAL_OK;
1739
1740         mtx_assert((&lldev->mtx_drv), MA_OWNED);
1741
1742         /* If device is in running state, initializing is not required */
1743         if(ifnetp->if_drv_flags & IFF_DRV_RUNNING)
1744             return;
1745
1746         /* Initializing timer */
1747         callout_init(&lldev->timer, 1);
1748
1749         xge_trace(XGE_TRACE, "Set MTU size");
1750         status = xge_hal_device_mtu_set(hldev, ifnetp->if_mtu);
1751         if(status != XGE_HAL_OK) {
1752             xge_trace(XGE_ERR, "Setting MTU in HAL device failed");
1753             goto _exit;
1754         }
1755
1756         /* Enable HAL device */
1757         xge_hal_device_enable(hldev);
1758
1759         /* Get MAC address and update in HAL */
1760         ifaddrp             = ifnetp->if_addr;
1761         sockaddrp           = (struct sockaddr_dl *)ifaddrp->ifa_addr;
1762         sockaddrp->sdl_type = IFT_ETHER;
1763         sockaddrp->sdl_alen = ifnetp->if_addrlen;
1764         macaddr             = LLADDR(sockaddrp);
1765         xge_trace(XGE_TRACE,
1766             "Setting MAC address: %02x:%02x:%02x:%02x:%02x:%02x\n",
1767             *macaddr, *(macaddr + 1), *(macaddr + 2), *(macaddr + 3),
1768             *(macaddr + 4), *(macaddr + 5));
1769         status = xge_hal_device_macaddr_set(hldev, 0, macaddr);
1770         if(status != XGE_HAL_OK)
1771             xge_trace(XGE_ERR, "Setting MAC address failed (%d)", status);
1772
1773         /* Opening channels */
1774         mtx_unlock(&lldev->mtx_drv);
1775         status = xge_channel_open(lldev, option);
1776         mtx_lock(&lldev->mtx_drv);
1777         if(status != XGE_HAL_OK)
1778             goto _exit;
1779
1780         /* Set appropriate flags */
1781         ifnetp->if_drv_flags  |=  IFF_DRV_RUNNING;
1782         ifnetp->if_flags &= ~IFF_DRV_OACTIVE;
1783
1784         /* Checksum capability */
1785         ifnetp->if_hwassist = (ifnetp->if_capenable & IFCAP_TXCSUM) ?
1786             (CSUM_TCP | CSUM_UDP) : 0;
1787
1788         if((lldev->enabled_tso) && (ifnetp->if_capenable & IFCAP_TSO4))
1789             ifnetp->if_hwassist |= CSUM_TSO;
1790
1791         /* Enable interrupts */
1792         xge_hal_device_intr_enable(hldev);
1793
1794         callout_reset(&lldev->timer, 10*hz, xge_timer, lldev);
1795
1796         /* Disable promiscuous mode */
1797         xge_trace(XGE_TRACE, "If opted, enable promiscuous mode");
1798         xge_enable_promisc(lldev);
1799
1800         /* Device is initialized */
1801         lldev->initialized = 1;
1802         xge_os_mdelay(1000);
1803
1804 _exit:
1805         return;
1806 }
1807
1808 /**
1809  * xge_timer
1810  * Timer timeout function to handle link status
1811  *
1812  * @devp Per-adapter Data
1813  */
1814 void
1815 xge_timer(void *devp)
1816 {
1817         xge_lldev_t      *lldev = (xge_lldev_t *)devp;
1818         xge_hal_device_t *hldev = lldev->devh;
1819
1820         /* Poll for changes */
1821         xge_hal_device_poll(hldev);
1822
1823         /* Reset timer */
1824         callout_reset(&lldev->timer, hz, xge_timer, lldev);
1825
1826         return;
1827 }
1828
1829 /**
1830  * xge_stop
1831  * De-activate the interface
1832  *
1833  * @lldev Per-adater Data
1834  */
1835 void
1836 xge_stop(xge_lldev_t *lldev)
1837 {
1838         mtx_lock(&lldev->mtx_drv);
1839         xge_device_stop(lldev, XGE_HAL_CHANNEL_OC_NORMAL);
1840         mtx_unlock(&lldev->mtx_drv);
1841 }
1842
1843 /**
1844  * xge_isr_filter
1845  * ISR filter function - to filter interrupts from other devices (shared)
1846  *
1847  * @handle Per-adapter Data
1848  *
1849  * Returns
1850  * FILTER_STRAY if interrupt is from other device
1851  * FILTER_SCHEDULE_THREAD if interrupt is from Xframe device
1852  */
1853 int
1854 xge_isr_filter(void *handle)
1855 {
1856         xge_lldev_t *lldev       = (xge_lldev_t *)handle;
1857         xge_hal_pci_bar0_t *bar0 = (xge_hal_pci_bar0_t *)((lldev->devh)->bar0);
1858         u16 retValue = FILTER_STRAY;
1859         u64 val64    = 0;
1860
1861         XGE_DRV_STATS(isr_filter);
1862
1863         val64 = xge_os_pio_mem_read64(lldev->pdev, (lldev->devh)->regh0,
1864             &bar0->general_int_status);
1865         retValue = (!val64) ? FILTER_STRAY : FILTER_SCHEDULE_THREAD;
1866
1867         return retValue;
1868 }
1869
1870 /**
1871  * xge_isr_line
1872  * Interrupt service routine for Line interrupts
1873  *
1874  * @plldev Per-adapter Data
1875  */
1876 void
1877 xge_isr_line(void *plldev)
1878 {
1879         xge_hal_status_e status;
1880         xge_lldev_t      *lldev   = (xge_lldev_t *)plldev;
1881         xge_hal_device_t *hldev   = (xge_hal_device_t *)lldev->devh;
1882         struct ifnet     *ifnetp  = lldev->ifnetp;
1883
1884         XGE_DRV_STATS(isr_line);
1885
1886         if(ifnetp->if_drv_flags & IFF_DRV_RUNNING) {
1887             status = xge_hal_device_handle_irq(hldev);
1888             if(!(IFQ_DRV_IS_EMPTY(&ifnetp->if_snd)))
1889                 xge_send(ifnetp);
1890         }
1891 }
1892
1893 /*
1894  * xge_isr_msi
1895  * ISR for Message signaled interrupts
1896  */
1897 void
1898 xge_isr_msi(void *plldev)
1899 {
1900         xge_lldev_t *lldev = (xge_lldev_t *)plldev;
1901         XGE_DRV_STATS(isr_msi);
1902         xge_hal_device_continue_irq(lldev->devh);
1903 }
1904
1905 /**
1906  * xge_rx_open
1907  * Initiate and open all Rx channels
1908  *
1909  * @qid Ring Index
1910  * @lldev Per-adapter Data
1911  * @rflag Channel open/close/reopen flag
1912  *
1913  * Returns 0 or Error Number
1914  */
1915 int
1916 xge_rx_open(int qid, xge_lldev_t *lldev, xge_hal_channel_reopen_e rflag)
1917 {
1918         u64 adapter_status = 0x0;
1919         xge_hal_status_e status = XGE_HAL_FAIL;
1920
1921         xge_hal_channel_attr_t attr = {
1922             .post_qid      = qid,
1923             .compl_qid     = 0,
1924             .callback      = xge_rx_compl,
1925             .per_dtr_space = sizeof(xge_rx_priv_t),
1926             .flags         = 0,
1927             .type          = XGE_HAL_CHANNEL_TYPE_RING,
1928             .userdata      = lldev,
1929             .dtr_init      = xge_rx_initial_replenish,
1930             .dtr_term      = xge_rx_term
1931         };
1932
1933         /* If device is not ready, return */
1934         status = xge_hal_device_status(lldev->devh, &adapter_status);
1935         if(status != XGE_HAL_OK) {
1936             xge_os_printf("Adapter Status: 0x%llx", (long long) adapter_status);
1937             XGE_EXIT_ON_ERR("Device is not ready", _exit, XGE_HAL_FAIL);
1938         }
1939         else {
1940             status = xge_hal_channel_open(lldev->devh, &attr,
1941                 &lldev->ring_channel[qid], rflag);
1942         }
1943
1944 _exit:
1945         return status;
1946 }
1947
1948 /**
1949  * xge_tx_open
1950  * Initialize and open all Tx channels
1951  *
1952  * @lldev Per-adapter Data
1953  * @tflag Channel open/close/reopen flag
1954  *
1955  * Returns 0 or Error Number
1956  */
1957 int
1958 xge_tx_open(xge_lldev_t *lldev, xge_hal_channel_reopen_e tflag)
1959 {
1960         xge_hal_status_e status = XGE_HAL_FAIL;
1961         u64 adapter_status = 0x0;
1962         int qindex, index;
1963
1964         xge_hal_channel_attr_t attr = {
1965             .compl_qid     = 0,
1966             .callback      = xge_tx_compl,
1967             .per_dtr_space = sizeof(xge_tx_priv_t),
1968             .flags         = 0,
1969             .type          = XGE_HAL_CHANNEL_TYPE_FIFO,
1970             .userdata      = lldev,
1971             .dtr_init      = xge_tx_initial_replenish,
1972             .dtr_term      = xge_tx_term
1973         };
1974
1975         /* If device is not ready, return */
1976         status = xge_hal_device_status(lldev->devh, &adapter_status);
1977         if(status != XGE_HAL_OK) {
1978             xge_os_printf("Adapter Status: 0x%llx", (long long) adapter_status);
1979             XGE_EXIT_ON_ERR("Device is not ready", _exit, XGE_HAL_FAIL);
1980         }
1981
1982         for(qindex = 0; qindex < XGE_FIFO_COUNT; qindex++) {
1983             attr.post_qid = qindex,
1984             status = xge_hal_channel_open(lldev->devh, &attr,
1985                 &lldev->fifo_channel[qindex], tflag);
1986             if(status != XGE_HAL_OK) {
1987                 for(index = 0; index < qindex; index++)
1988                     xge_hal_channel_close(lldev->fifo_channel[index], tflag);
1989             }
1990         }
1991
1992 _exit:
1993         return status;
1994 }
1995
1996 /**
1997  * xge_enable_msi
1998  * Enables MSI
1999  *
2000  * @lldev Per-adapter Data
2001  */
2002 void
2003 xge_enable_msi(xge_lldev_t *lldev)
2004 {
2005         xge_list_t        *item    = NULL;
2006         xge_hal_device_t  *hldev   = lldev->devh;
2007         xge_hal_channel_t *channel = NULL;
2008         u16 offset = 0, val16 = 0;
2009
2010         xge_os_pci_read16(lldev->pdev, NULL,
2011             xge_offsetof(xge_hal_pci_config_le_t, msi_control), &val16);
2012
2013         /* Update msi_data */
2014         offset = (val16 & 0x80) ? 0x4c : 0x48;
2015         xge_os_pci_read16(lldev->pdev, NULL, offset, &val16);
2016         if(val16 & 0x1)
2017             val16 &= 0xfffe;
2018         else
2019             val16 |= 0x1;
2020         xge_os_pci_write16(lldev->pdev, NULL, offset, val16);
2021
2022         /* Update msi_control */
2023         xge_os_pci_read16(lldev->pdev, NULL,
2024             xge_offsetof(xge_hal_pci_config_le_t, msi_control), &val16);
2025         val16 |= 0x10;
2026         xge_os_pci_write16(lldev->pdev, NULL,
2027             xge_offsetof(xge_hal_pci_config_le_t, msi_control), val16);
2028
2029         /* Set TxMAT and RxMAT registers with MSI */
2030         xge_list_for_each(item, &hldev->free_channels) {
2031             channel = xge_container_of(item, xge_hal_channel_t, item);
2032             xge_hal_channel_msi_set(channel, 1, (u32)val16);
2033         }
2034 }
2035
2036 /**
2037  * xge_channel_open
2038  * Open both Tx and Rx channels
2039  *
2040  * @lldev Per-adapter Data
2041  * @option Channel reopen option
2042  */
2043 int
2044 xge_channel_open(xge_lldev_t *lldev, xge_hal_channel_reopen_e option)
2045 {
2046         xge_lro_entry_t *lro_session = NULL;
2047         xge_hal_status_e status   = XGE_HAL_OK;
2048         int index = 0, index2 = 0;
2049
2050         if(lldev->enabled_msi == XGE_HAL_INTR_MODE_MSI) {
2051             xge_msi_info_restore(lldev);
2052             xge_enable_msi(lldev);
2053         }
2054
2055 _exit2:
2056         status = xge_create_dma_tags(lldev->device);
2057         if(status != XGE_HAL_OK)
2058             XGE_EXIT_ON_ERR("DMA tag creation failed", _exit, status);
2059
2060         /* Open ring (Rx) channel */
2061         for(index = 0; index < XGE_RING_COUNT; index++) {
2062             status = xge_rx_open(index, lldev, option);
2063             if(status != XGE_HAL_OK) {
2064                 /*
2065                  * DMA mapping fails in the unpatched Kernel which can't
2066                  * allocate contiguous memory for Jumbo frames.
2067                  * Try using 5 buffer mode.
2068                  */
2069                 if((lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_1) &&
2070                     (((lldev->ifnetp)->if_mtu + XGE_HAL_MAC_HEADER_MAX_SIZE) >
2071                     MJUMPAGESIZE)) {
2072                     /* Close so far opened channels */
2073                     for(index2 = 0; index2 < index; index2++) {
2074                         xge_hal_channel_close(lldev->ring_channel[index2],
2075                             option);
2076                     }
2077
2078                     /* Destroy DMA tags intended to use for 1 buffer mode */
2079                     if(bus_dmamap_destroy(lldev->dma_tag_rx,
2080                         lldev->extra_dma_map)) {
2081                         xge_trace(XGE_ERR, "Rx extra DMA map destroy failed");
2082                     }
2083                     if(bus_dma_tag_destroy(lldev->dma_tag_rx))
2084                         xge_trace(XGE_ERR, "Rx DMA tag destroy failed");
2085                     if(bus_dma_tag_destroy(lldev->dma_tag_tx))
2086                         xge_trace(XGE_ERR, "Tx DMA tag destroy failed");
2087
2088                     /* Switch to 5 buffer mode */
2089                     lldev->buffer_mode = XGE_HAL_RING_QUEUE_BUFFER_MODE_5;
2090                     xge_buffer_mode_init(lldev, (lldev->ifnetp)->if_mtu);
2091
2092                     /* Restart init */
2093                     goto _exit2;
2094                 }
2095                 else {
2096                     XGE_EXIT_ON_ERR("Opening Rx channel failed", _exit1,
2097                         status);
2098                 }
2099             }
2100         }
2101
2102         if(lldev->enabled_lro) {
2103             SLIST_INIT(&lldev->lro_free);
2104             SLIST_INIT(&lldev->lro_active);
2105             lldev->lro_num = XGE_LRO_DEFAULT_ENTRIES;
2106
2107             for(index = 0; index < lldev->lro_num; index++) {
2108                 lro_session = (xge_lro_entry_t *)
2109                     xge_os_malloc(NULL, sizeof(xge_lro_entry_t));
2110                 if(lro_session == NULL) {
2111                     lldev->lro_num = index;
2112                     break;
2113                 }
2114                 SLIST_INSERT_HEAD(&lldev->lro_free, lro_session, next);
2115             }
2116         }
2117
2118         /* Open FIFO (Tx) channel */
2119         status = xge_tx_open(lldev, option);
2120         if(status != XGE_HAL_OK)
2121             XGE_EXIT_ON_ERR("Opening Tx channel failed", _exit1, status);
2122
2123         goto _exit;
2124
2125 _exit1:
2126         /*
2127          * Opening Rx channel(s) failed (index is <last ring index - 1>) or
2128          * Initialization of LRO failed (index is XGE_RING_COUNT)
2129          * Opening Tx channel failed    (index is XGE_RING_COUNT)
2130          */
2131         for(index2 = 0; index2 < index; index2++)
2132             xge_hal_channel_close(lldev->ring_channel[index2], option);
2133
2134 _exit:
2135         return status;
2136 }
2137
2138 /**
2139  * xge_channel_close
2140  * Close both Tx and Rx channels
2141  *
2142  * @lldev Per-adapter Data
2143  * @option Channel reopen option
2144  *
2145  */
2146 void
2147 xge_channel_close(xge_lldev_t *lldev, xge_hal_channel_reopen_e option)
2148 {
2149         int qindex = 0;
2150
2151         DELAY(1000 * 1000);
2152
2153         /* Close FIFO (Tx) channel */
2154         for(qindex = 0; qindex < XGE_FIFO_COUNT; qindex++)
2155             xge_hal_channel_close(lldev->fifo_channel[qindex], option);
2156
2157         /* Close Ring (Rx) channels */
2158         for(qindex = 0; qindex < XGE_RING_COUNT; qindex++)
2159             xge_hal_channel_close(lldev->ring_channel[qindex], option);
2160
2161         if(bus_dmamap_destroy(lldev->dma_tag_rx, lldev->extra_dma_map))
2162             xge_trace(XGE_ERR, "Rx extra map destroy failed");
2163         if(bus_dma_tag_destroy(lldev->dma_tag_rx))
2164             xge_trace(XGE_ERR, "Rx DMA tag destroy failed");
2165         if(bus_dma_tag_destroy(lldev->dma_tag_tx))
2166             xge_trace(XGE_ERR, "Tx DMA tag destroy failed");
2167 }
2168
2169 /**
2170  * dmamap_cb
2171  * DMA map callback
2172  *
2173  * @arg Parameter passed from dmamap
2174  * @segs Segments
2175  * @nseg Number of segments
2176  * @error Error
2177  */
2178 void
2179 dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
2180 {
2181         if(!error) {
2182             *(bus_addr_t *) arg = segs->ds_addr;
2183         }
2184 }
2185
2186 /**
2187  * xge_reset
2188  * Device Reset
2189  *
2190  * @lldev Per-adapter Data
2191  */
2192 void
2193 xge_reset(xge_lldev_t *lldev)
2194 {
2195         xge_trace(XGE_TRACE, "Reseting the chip");
2196
2197         /* If the device is not initialized, return */
2198         if(lldev->initialized) {
2199             mtx_lock(&lldev->mtx_drv);
2200             xge_device_stop(lldev, XGE_HAL_CHANNEL_OC_NORMAL);
2201             xge_device_init(lldev, XGE_HAL_CHANNEL_OC_NORMAL);
2202             mtx_unlock(&lldev->mtx_drv);
2203         }
2204
2205         return;
2206 }
2207
2208 /**
2209  * xge_setmulti
2210  * Set an address as a multicast address
2211  *
2212  * @lldev Per-adapter Data
2213  */
2214 void
2215 xge_setmulti(xge_lldev_t *lldev)
2216 {
2217         struct ifmultiaddr *ifma;
2218         u8                 *lladdr;
2219         xge_hal_device_t   *hldev        = (xge_hal_device_t *)lldev->devh;
2220         struct ifnet       *ifnetp       = lldev->ifnetp;
2221         int                index         = 0;
2222         int                offset        = 1;
2223         int                table_size    = 47;
2224         xge_hal_status_e   status        = XGE_HAL_OK;
2225         u8                 initial_addr[]= {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
2226
2227         if((ifnetp->if_flags & IFF_MULTICAST) && (!lldev->all_multicast)) {
2228             status = xge_hal_device_mcast_enable(hldev);
2229             lldev->all_multicast = 1;
2230         }
2231         else if((ifnetp->if_flags & IFF_MULTICAST) && (lldev->all_multicast)) {
2232             status = xge_hal_device_mcast_disable(hldev);
2233             lldev->all_multicast = 0;
2234         }
2235
2236         if(status != XGE_HAL_OK) {
2237             xge_trace(XGE_ERR, "Enabling/disabling multicast failed");
2238             goto _exit;
2239         }
2240
2241         /* Updating address list */
2242         if_maddr_rlock(ifnetp);
2243         index = 0;
2244         TAILQ_FOREACH(ifma, &ifnetp->if_multiaddrs, ifma_link) {
2245             if(ifma->ifma_addr->sa_family != AF_LINK) {
2246                 continue;
2247             }
2248             lladdr = LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
2249             index += 1;
2250         }
2251         if_maddr_runlock(ifnetp);
2252
2253         if((!lldev->all_multicast) && (index)) {
2254             lldev->macaddr_count = (index + 1);
2255             if(lldev->macaddr_count > table_size) {
2256                 goto _exit;
2257             }
2258
2259             /* Clear old addresses */
2260             for(index = 0; index < 48; index++) {
2261                 xge_hal_device_macaddr_set(hldev, (offset + index),
2262                     initial_addr);
2263             }
2264         }
2265
2266         /* Add new addresses */
2267         if_maddr_rlock(ifnetp);
2268         index = 0;
2269         TAILQ_FOREACH(ifma, &ifnetp->if_multiaddrs, ifma_link) {
2270             if(ifma->ifma_addr->sa_family != AF_LINK) {
2271                 continue;
2272             }
2273             lladdr = LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
2274             xge_hal_device_macaddr_set(hldev, (offset + index), lladdr);
2275             index += 1;
2276         }
2277         if_maddr_runlock(ifnetp);
2278
2279 _exit:
2280         return;
2281 }
2282
2283 /**
2284  * xge_enable_promisc
2285  * Enable Promiscuous Mode
2286  *
2287  * @lldev Per-adapter Data
2288  */
2289 void
2290 xge_enable_promisc(xge_lldev_t *lldev)
2291 {
2292         struct ifnet *ifnetp = lldev->ifnetp;
2293         xge_hal_device_t *hldev = lldev->devh;
2294         xge_hal_pci_bar0_t *bar0 = NULL;
2295         u64 val64 = 0;
2296
2297         bar0 = (xge_hal_pci_bar0_t *) hldev->bar0;
2298
2299         if(ifnetp->if_flags & IFF_PROMISC) {
2300             xge_hal_device_promisc_enable(lldev->devh);
2301
2302             /*
2303              * When operating in promiscuous mode, don't strip the VLAN tag
2304              */
2305             val64 = xge_os_pio_mem_read64(lldev->pdev, hldev->regh0,
2306                 &bar0->rx_pa_cfg);
2307             val64 &= ~XGE_HAL_RX_PA_CFG_STRIP_VLAN_TAG_MODE(1);
2308             val64 |= XGE_HAL_RX_PA_CFG_STRIP_VLAN_TAG_MODE(0);
2309             xge_os_pio_mem_write64(lldev->pdev, hldev->regh0, val64,
2310                 &bar0->rx_pa_cfg);
2311
2312             xge_trace(XGE_TRACE, "Promiscuous mode ON");
2313         }
2314 }
2315
2316 /**
2317  * xge_disable_promisc
2318  * Disable Promiscuous Mode
2319  *
2320  * @lldev Per-adapter Data
2321  */
2322 void
2323 xge_disable_promisc(xge_lldev_t *lldev)
2324 {
2325         xge_hal_device_t *hldev = lldev->devh;
2326         xge_hal_pci_bar0_t *bar0 = NULL;
2327         u64 val64 = 0;
2328
2329         bar0 = (xge_hal_pci_bar0_t *) hldev->bar0;
2330
2331         xge_hal_device_promisc_disable(lldev->devh);
2332
2333         /*
2334          * Strip VLAN tag when operating in non-promiscuous mode
2335          */
2336         val64 = xge_os_pio_mem_read64(lldev->pdev, hldev->regh0,
2337             &bar0->rx_pa_cfg);
2338         val64 &= ~XGE_HAL_RX_PA_CFG_STRIP_VLAN_TAG_MODE(1);
2339         val64 |= XGE_HAL_RX_PA_CFG_STRIP_VLAN_TAG_MODE(1);
2340         xge_os_pio_mem_write64(lldev->pdev, hldev->regh0, val64,
2341             &bar0->rx_pa_cfg);
2342
2343         xge_trace(XGE_TRACE, "Promiscuous mode OFF");
2344 }
2345
2346 /**
2347  * xge_change_mtu
2348  * Change interface MTU to a requested valid size
2349  *
2350  * @lldev Per-adapter Data
2351  * @NewMtu Requested MTU
2352  *
2353  * Returns 0 or Error Number
2354  */
2355 int
2356 xge_change_mtu(xge_lldev_t *lldev, int new_mtu)
2357 {
2358         int status = XGE_HAL_OK;
2359
2360         /* Check requested MTU size for boundary */
2361         if(xge_hal_device_mtu_check(lldev->devh, new_mtu) != XGE_HAL_OK) {
2362             XGE_EXIT_ON_ERR("Invalid MTU", _exit, EINVAL);
2363         }
2364
2365         lldev->mtu = new_mtu;
2366         xge_confirm_changes(lldev, XGE_SET_MTU);
2367
2368 _exit:
2369         return status;
2370 }
2371
2372 /**
2373  * xge_device_stop
2374  *
2375  * Common code for both stop and part of reset. Disables device, interrupts and
2376  * closes channels
2377  *
2378  * @dev Device Handle
2379  * @option Channel normal/reset option
2380  */
2381 void
2382 xge_device_stop(xge_lldev_t *lldev, xge_hal_channel_reopen_e option)
2383 {
2384         xge_hal_device_t *hldev  = lldev->devh;
2385         struct ifnet     *ifnetp = lldev->ifnetp;
2386         u64               val64  = 0;
2387
2388         mtx_assert((&lldev->mtx_drv), MA_OWNED);
2389
2390         /* If device is not in "Running" state, return */
2391         if (!(ifnetp->if_drv_flags & IFF_DRV_RUNNING))
2392             goto _exit;
2393
2394         /* Set appropriate flags */
2395         ifnetp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
2396
2397         /* Stop timer */
2398         callout_stop(&lldev->timer);
2399
2400         /* Disable interrupts */
2401         xge_hal_device_intr_disable(hldev);
2402
2403         mtx_unlock(&lldev->mtx_drv);
2404         xge_queue_flush(xge_hal_device_queue(lldev->devh));
2405         mtx_lock(&lldev->mtx_drv);
2406
2407         /* Disable HAL device */
2408         if(xge_hal_device_disable(hldev) != XGE_HAL_OK) {
2409             xge_trace(XGE_ERR, "Disabling HAL device failed");
2410             xge_hal_device_status(hldev, &val64);
2411             xge_trace(XGE_ERR, "Adapter Status: 0x%llx", (long long)val64);
2412         }
2413
2414         /* Close Tx and Rx channels */
2415         xge_channel_close(lldev, option);
2416
2417         /* Reset HAL device */
2418         xge_hal_device_reset(hldev);
2419
2420         xge_os_mdelay(1000);
2421         lldev->initialized = 0;
2422
2423         if_link_state_change(ifnetp, LINK_STATE_DOWN);
2424
2425 _exit:
2426         return;
2427 }
2428
2429 /**
2430  * xge_set_mbuf_cflags
2431  * set checksum flag for the mbuf
2432  *
2433  * @pkt Packet
2434  */
2435 void
2436 xge_set_mbuf_cflags(mbuf_t pkt)
2437 {
2438         pkt->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
2439         pkt->m_pkthdr.csum_flags |= CSUM_IP_VALID;
2440         pkt->m_pkthdr.csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
2441         pkt->m_pkthdr.csum_data = htons(0xffff);
2442 }
2443
2444 /**
2445  * xge_lro_flush_sessions
2446  * Flush LRO session and send accumulated LRO packet to upper layer
2447  *
2448  * @lldev Per-adapter Data
2449  */
2450 void
2451 xge_lro_flush_sessions(xge_lldev_t *lldev)
2452 {
2453         xge_lro_entry_t *lro_session = NULL;
2454
2455         while(!SLIST_EMPTY(&lldev->lro_active)) {
2456             lro_session = SLIST_FIRST(&lldev->lro_active);
2457             SLIST_REMOVE_HEAD(&lldev->lro_active, next);
2458             xge_lro_flush(lldev, lro_session);
2459         }
2460 }
2461
2462 /**
2463  * xge_lro_flush
2464  * Flush LRO session. Send accumulated LRO packet to upper layer
2465  *
2466  * @lldev Per-adapter Data
2467  * @lro LRO session to be flushed
2468  */
2469 static void
2470 xge_lro_flush(xge_lldev_t *lldev, xge_lro_entry_t *lro_session)
2471 {
2472         struct ip *header_ip;
2473         struct tcphdr *header_tcp;
2474         u32 *ptr;
2475
2476         if(lro_session->append_cnt) {
2477             header_ip = lro_session->lro_header_ip;
2478             header_ip->ip_len = htons(lro_session->len - ETHER_HDR_LEN);
2479             lro_session->m_head->m_pkthdr.len = lro_session->len;
2480             header_tcp = (struct tcphdr *)(header_ip + 1);
2481             header_tcp->th_ack = lro_session->ack_seq;
2482             header_tcp->th_win = lro_session->window;
2483             if(lro_session->timestamp) {
2484                 ptr = (u32 *)(header_tcp + 1);
2485                 ptr[1] = htonl(lro_session->tsval);
2486                 ptr[2] = lro_session->tsecr;
2487             }
2488         }
2489
2490         (*lldev->ifnetp->if_input)(lldev->ifnetp, lro_session->m_head);
2491         lro_session->m_head = NULL;
2492         lro_session->timestamp = 0;
2493         lro_session->append_cnt = 0;
2494         SLIST_INSERT_HEAD(&lldev->lro_free, lro_session, next);
2495 }
2496
2497 /**
2498  * xge_lro_accumulate
2499  * Accumulate packets to form a large LRO packet based on various conditions
2500  *
2501  * @lldev Per-adapter Data
2502  * @m_head Current Packet
2503  *
2504  * Returns XGE_HAL_OK or XGE_HAL_FAIL (failure)
2505  */
2506 static int
2507 xge_lro_accumulate(xge_lldev_t *lldev, struct mbuf *m_head)
2508 {
2509         struct ether_header *header_ethernet;
2510         struct ip *header_ip;
2511         struct tcphdr *header_tcp;
2512         u32 seq, *ptr;
2513         struct mbuf *buffer_next, *buffer_tail;
2514         xge_lro_entry_t *lro_session;
2515         xge_hal_status_e status = XGE_HAL_FAIL;
2516         int hlen, ip_len, tcp_hdr_len, tcp_data_len, tot_len, tcp_options;
2517         int trim;
2518
2519         /* Get Ethernet header */
2520         header_ethernet = mtod(m_head, struct ether_header *);
2521
2522         /* Return if it is not IP packet */
2523         if(header_ethernet->ether_type != htons(ETHERTYPE_IP))
2524             goto _exit;
2525
2526         /* Get IP header */
2527         header_ip = lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_1 ?
2528             (struct ip *)(header_ethernet + 1) :
2529             mtod(m_head->m_next, struct ip *);
2530
2531         /* Return if it is not TCP packet */
2532         if(header_ip->ip_p != IPPROTO_TCP)
2533             goto _exit;
2534
2535         /* Return if packet has options */
2536         if((header_ip->ip_hl << 2) != sizeof(*header_ip))
2537             goto _exit;
2538
2539         /* Return if packet is fragmented */
2540         if(header_ip->ip_off & htons(IP_MF | IP_OFFMASK))
2541             goto _exit;
2542
2543         /* Get TCP header */
2544         header_tcp = (struct tcphdr *)(header_ip + 1);
2545
2546         /* Return if not ACK or PUSH */
2547         if((header_tcp->th_flags & ~(TH_ACK | TH_PUSH)) != 0)
2548             goto _exit;
2549
2550         /* Only timestamp option is handled */
2551         tcp_options = (header_tcp->th_off << 2) - sizeof(*header_tcp);
2552         tcp_hdr_len = sizeof(*header_tcp) + tcp_options;
2553         ptr = (u32 *)(header_tcp + 1);
2554         if(tcp_options != 0) {
2555             if(__predict_false(tcp_options != TCPOLEN_TSTAMP_APPA) ||
2556                 (*ptr != ntohl(TCPOPT_NOP << 24 | TCPOPT_NOP << 16 |
2557                 TCPOPT_TIMESTAMP << 8 | TCPOLEN_TIMESTAMP))) {
2558                 goto _exit;
2559             }
2560         }
2561
2562         /* Total length of packet (IP) */
2563         ip_len = ntohs(header_ip->ip_len);
2564
2565         /* TCP data size */
2566         tcp_data_len = ip_len - (header_tcp->th_off << 2) - sizeof(*header_ip);
2567
2568         /* If the frame is padded, trim it */
2569         tot_len = m_head->m_pkthdr.len;
2570         trim = tot_len - (ip_len + ETHER_HDR_LEN);
2571         if(trim != 0) {
2572             if(trim < 0)
2573                 goto _exit;
2574             m_adj(m_head, -trim);
2575             tot_len = m_head->m_pkthdr.len;
2576         }
2577
2578         buffer_next = m_head;
2579         buffer_tail = NULL;
2580         while(buffer_next != NULL) {
2581             buffer_tail = buffer_next;
2582             buffer_next = buffer_tail->m_next;
2583         }
2584
2585         /* Total size of only headers */
2586         hlen = ip_len + ETHER_HDR_LEN - tcp_data_len;
2587
2588         /* Get sequence number */
2589         seq = ntohl(header_tcp->th_seq);
2590
2591         SLIST_FOREACH(lro_session, &lldev->lro_active, next) {
2592             if(lro_session->source_port == header_tcp->th_sport &&
2593                 lro_session->dest_port == header_tcp->th_dport &&
2594                 lro_session->source_ip == header_ip->ip_src.s_addr &&
2595                 lro_session->dest_ip == header_ip->ip_dst.s_addr) {
2596
2597                 /* Unmatched sequence number, flush LRO session */
2598                 if(__predict_false(seq != lro_session->next_seq)) {
2599                     SLIST_REMOVE(&lldev->lro_active, lro_session,
2600                         xge_lro_entry_t, next);
2601                     xge_lro_flush(lldev, lro_session);
2602                     goto _exit;
2603                 }
2604
2605                 /* Handle timestamp option */
2606                 if(tcp_options) {
2607                     u32 tsval = ntohl(*(ptr + 1));
2608                     if(__predict_false(lro_session->tsval > tsval ||
2609                         *(ptr + 2) == 0)) {
2610                         goto _exit;
2611                     }
2612                     lro_session->tsval = tsval;
2613                     lro_session->tsecr = *(ptr + 2);
2614                 }
2615
2616                 lro_session->next_seq += tcp_data_len;
2617                 lro_session->ack_seq = header_tcp->th_ack;
2618                 lro_session->window = header_tcp->th_win;
2619
2620                 /* If TCP data/payload is of 0 size, free mbuf */
2621                 if(tcp_data_len == 0) {
2622                     m_freem(m_head);
2623                     status = XGE_HAL_OK;
2624                     goto _exit;
2625                 }
2626
2627                 lro_session->append_cnt++;
2628                 lro_session->len += tcp_data_len;
2629
2630                 /* Adjust mbuf so that m_data points to payload than headers */
2631                 m_adj(m_head, hlen);
2632
2633                 /* Append this packet to LRO accumulated packet */
2634                 lro_session->m_tail->m_next = m_head;
2635                 lro_session->m_tail = buffer_tail;
2636
2637                 /* Flush if LRO packet is exceeding maximum size */
2638                 if(lro_session->len >
2639                     (XGE_HAL_LRO_DEFAULT_FRM_LEN - lldev->ifnetp->if_mtu)) {
2640                     SLIST_REMOVE(&lldev->lro_active, lro_session,
2641                         xge_lro_entry_t, next);
2642                     xge_lro_flush(lldev, lro_session);
2643                 }
2644                 status = XGE_HAL_OK;
2645                 goto _exit;
2646             }
2647         }
2648
2649         if(SLIST_EMPTY(&lldev->lro_free))
2650             goto _exit;
2651
2652         /* Start a new LRO session */
2653         lro_session = SLIST_FIRST(&lldev->lro_free);
2654         SLIST_REMOVE_HEAD(&lldev->lro_free, next);
2655         SLIST_INSERT_HEAD(&lldev->lro_active, lro_session, next);
2656         lro_session->source_port = header_tcp->th_sport;
2657         lro_session->dest_port = header_tcp->th_dport;
2658         lro_session->source_ip = header_ip->ip_src.s_addr;
2659         lro_session->dest_ip = header_ip->ip_dst.s_addr;
2660         lro_session->next_seq = seq + tcp_data_len;
2661         lro_session->mss = tcp_data_len;
2662         lro_session->ack_seq = header_tcp->th_ack;
2663         lro_session->window = header_tcp->th_win;
2664
2665         lro_session->lro_header_ip = header_ip;
2666
2667         /* Handle timestamp option */
2668         if(tcp_options) {
2669             lro_session->timestamp = 1;
2670             lro_session->tsval = ntohl(*(ptr + 1));
2671             lro_session->tsecr = *(ptr + 2);
2672         }
2673
2674         lro_session->len = tot_len;
2675         lro_session->m_head = m_head;
2676         lro_session->m_tail = buffer_tail;
2677         status = XGE_HAL_OK;
2678
2679 _exit:
2680         return status;
2681 }
2682
2683 /**
2684  * xge_accumulate_large_rx
2685  * Accumulate packets to form a large LRO packet based on various conditions
2686  *
2687  * @lldev Per-adapter Data
2688  * @pkt Current packet
2689  * @pkt_length Packet Length
2690  * @rxd_priv Rx Descriptor Private Data
2691  */
2692 void
2693 xge_accumulate_large_rx(xge_lldev_t *lldev, struct mbuf *pkt, int pkt_length,
2694         xge_rx_priv_t *rxd_priv)
2695 {
2696         if(xge_lro_accumulate(lldev, pkt) != XGE_HAL_OK) {
2697             bus_dmamap_sync(lldev->dma_tag_rx, rxd_priv->dmainfo[0].dma_map,
2698                 BUS_DMASYNC_POSTREAD);
2699             (*lldev->ifnetp->if_input)(lldev->ifnetp, pkt);
2700         }
2701 }
2702
2703 /**
2704  * xge_rx_compl
2705  * If the interrupt is due to received frame (Rx completion), send it up
2706  *
2707  * @channelh Ring Channel Handle
2708  * @dtr Current Descriptor
2709  * @t_code Transfer Code indicating success or error
2710  * @userdata Per-adapter Data
2711  *
2712  * Returns XGE_HAL_OK or HAL error enums
2713  */
2714 xge_hal_status_e
2715 xge_rx_compl(xge_hal_channel_h channelh, xge_hal_dtr_h dtr, u8 t_code,
2716         void *userdata)
2717 {
2718         struct ifnet       *ifnetp;
2719         xge_rx_priv_t      *rxd_priv = NULL;
2720         mbuf_t              mbuf_up  = NULL;
2721         xge_hal_status_e    status   = XGE_HAL_OK;
2722         xge_hal_dtr_info_t  ext_info;
2723         int                 index;
2724         u16                 vlan_tag;
2725
2726         /*get the user data portion*/
2727         xge_lldev_t *lldev = xge_hal_channel_userdata(channelh);
2728         if(!lldev) {
2729             XGE_EXIT_ON_ERR("Failed to get user data", _exit, XGE_HAL_FAIL);
2730         }
2731
2732         XGE_DRV_STATS(rx_completions);
2733
2734         /* get the interface pointer */
2735         ifnetp = lldev->ifnetp;
2736
2737         do {
2738             XGE_DRV_STATS(rx_desc_compl);
2739
2740             if(!(ifnetp->if_drv_flags & IFF_DRV_RUNNING)) {
2741                 status = XGE_HAL_FAIL;
2742                 goto _exit;
2743             }
2744
2745             if(t_code) {
2746                 xge_trace(XGE_TRACE, "Packet dropped because of %d", t_code);
2747                 XGE_DRV_STATS(rx_tcode);
2748                 xge_hal_device_handle_tcode(channelh, dtr, t_code);
2749                 xge_hal_ring_dtr_post(channelh,dtr);
2750                 continue;
2751             }
2752
2753             /* Get the private data for this descriptor*/
2754             rxd_priv = (xge_rx_priv_t *) xge_hal_ring_dtr_private(channelh,
2755                 dtr);
2756             if(!rxd_priv) {
2757                 XGE_EXIT_ON_ERR("Failed to get descriptor private data", _exit,
2758                     XGE_HAL_FAIL);
2759             }
2760
2761             /*
2762              * Prepare one buffer to send it to upper layer -- since the upper
2763              * layer frees the buffer do not use rxd_priv->buffer. Meanwhile
2764              * prepare a new buffer, do mapping, use it in the current
2765              * descriptor and post descriptor back to ring channel
2766              */
2767             mbuf_up = rxd_priv->bufferArray[0];
2768
2769             /* Gets details of mbuf i.e., packet length */
2770             xge_ring_dtr_get(mbuf_up, channelh, dtr, lldev, rxd_priv);
2771
2772             status =
2773                 (lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_1) ?
2774                 xge_get_buf(dtr, rxd_priv, lldev, 0) :
2775                 xge_get_buf_3b_5b(dtr, rxd_priv, lldev);
2776
2777             if(status != XGE_HAL_OK) {
2778                 xge_trace(XGE_ERR, "No memory");
2779                 XGE_DRV_STATS(rx_no_buf);
2780
2781                 /*
2782                  * Unable to allocate buffer. Instead of discarding, post
2783                  * descriptor back to channel for future processing of same
2784                  * packet.
2785                  */
2786                 xge_hal_ring_dtr_post(channelh, dtr);
2787                 continue;
2788             }
2789
2790             /* Get the extended information */
2791             xge_hal_ring_dtr_info_get(channelh, dtr, &ext_info);
2792
2793             /*
2794              * As we have allocated a new mbuf for this descriptor, post this
2795              * descriptor with new mbuf back to ring channel
2796              */
2797             vlan_tag = ext_info.vlan;
2798             xge_hal_ring_dtr_post(channelh, dtr);
2799             if ((!(ext_info.proto & XGE_HAL_FRAME_PROTO_IP_FRAGMENTED) &&
2800                 (ext_info.proto & XGE_HAL_FRAME_PROTO_TCP_OR_UDP) &&
2801                 (ext_info.l3_cksum == XGE_HAL_L3_CKSUM_OK) &&
2802                 (ext_info.l4_cksum == XGE_HAL_L4_CKSUM_OK))) {
2803
2804                 /* set Checksum Flag */
2805                 xge_set_mbuf_cflags(mbuf_up);
2806
2807                 if(lldev->enabled_lro) {
2808                     xge_accumulate_large_rx(lldev, mbuf_up, mbuf_up->m_len,
2809                         rxd_priv);
2810                 }
2811                 else {
2812                     /* Post-Read sync for buffers*/
2813                     for(index = 0; index < lldev->rxd_mbuf_cnt; index++) {
2814                         bus_dmamap_sync(lldev->dma_tag_rx,
2815                             rxd_priv->dmainfo[0].dma_map, BUS_DMASYNC_POSTREAD);
2816                     }
2817                     (*ifnetp->if_input)(ifnetp, mbuf_up);
2818                 }
2819             }
2820             else {
2821                 /*
2822                  * Packet with erroneous checksum , let the upper layer deal
2823                  * with it
2824                  */
2825
2826                 /* Post-Read sync for buffers*/
2827                 for(index = 0; index < lldev->rxd_mbuf_cnt; index++) {
2828                     bus_dmamap_sync(lldev->dma_tag_rx,
2829                          rxd_priv->dmainfo[0].dma_map, BUS_DMASYNC_POSTREAD);
2830                 }
2831
2832                 if(vlan_tag) {
2833                     mbuf_up->m_pkthdr.ether_vtag = vlan_tag;
2834                     mbuf_up->m_flags |= M_VLANTAG;
2835                 }
2836
2837                 if(lldev->enabled_lro)
2838                     xge_lro_flush_sessions(lldev);
2839
2840                 (*ifnetp->if_input)(ifnetp, mbuf_up);
2841             }
2842         } while(xge_hal_ring_dtr_next_completed(channelh, &dtr, &t_code)
2843             == XGE_HAL_OK);
2844
2845         if(lldev->enabled_lro)
2846             xge_lro_flush_sessions(lldev);
2847
2848 _exit:
2849         return status;
2850 }
2851
2852 /**
2853  * xge_ring_dtr_get
2854  * Get descriptors
2855  *
2856  * @mbuf_up Packet to send up
2857  * @channelh Ring Channel Handle
2858  * @dtr Descriptor
2859  * @lldev Per-adapter Data
2860  * @rxd_priv Rx Descriptor Private Data
2861  *
2862  * Returns XGE_HAL_OK or HAL error enums
2863  */
2864 int
2865 xge_ring_dtr_get(mbuf_t mbuf_up, xge_hal_channel_h channelh, xge_hal_dtr_h dtr,
2866         xge_lldev_t *lldev, xge_rx_priv_t *rxd_priv)
2867 {
2868         mbuf_t           m;
2869         int              pkt_length[5]={0,0}, pkt_len=0;
2870         dma_addr_t       dma_data[5];
2871         int              index;
2872
2873         m = mbuf_up;
2874         pkt_len = 0;
2875
2876         if(lldev->buffer_mode != XGE_HAL_RING_QUEUE_BUFFER_MODE_1) {
2877             xge_os_memzero(pkt_length, sizeof(pkt_length));
2878
2879             /*
2880              * Retrieve data of interest from the completed descriptor -- This
2881              * returns the packet length
2882              */
2883             if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5) {
2884                 xge_hal_ring_dtr_5b_get(channelh, dtr, dma_data, pkt_length);
2885             }
2886             else {
2887                 xge_hal_ring_dtr_3b_get(channelh, dtr, dma_data, pkt_length);
2888             }
2889
2890             for(index = 0; index < lldev->rxd_mbuf_cnt; index++) {
2891                 m->m_len  = pkt_length[index];
2892
2893                 if(index < (lldev->rxd_mbuf_cnt-1)) {
2894                     m->m_next = rxd_priv->bufferArray[index + 1];
2895                     m = m->m_next;
2896                 }
2897                 else {
2898                     m->m_next = NULL;
2899                 }
2900                 pkt_len+=pkt_length[index];
2901             }
2902
2903             /*
2904              * Since 2 buffer mode is an exceptional case where data is in 3rd
2905              * buffer but not in 2nd buffer
2906              */
2907             if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_2) {
2908                 m->m_len = pkt_length[2];
2909                 pkt_len+=pkt_length[2];
2910             }
2911
2912             /*
2913              * Update length of newly created buffer to be sent up with packet
2914              * length
2915              */
2916             mbuf_up->m_pkthdr.len = pkt_len;
2917         }
2918         else {
2919             /*
2920              * Retrieve data of interest from the completed descriptor -- This
2921              * returns the packet length
2922              */
2923             xge_hal_ring_dtr_1b_get(channelh, dtr,&dma_data[0], &pkt_length[0]);
2924
2925             /*
2926              * Update length of newly created buffer to be sent up with packet
2927              * length
2928              */
2929             mbuf_up->m_len =  mbuf_up->m_pkthdr.len = pkt_length[0];
2930         }
2931
2932         return XGE_HAL_OK;
2933 }
2934
2935 /**
2936  * xge_flush_txds
2937  * Flush Tx descriptors
2938  *
2939  * @channelh Channel handle
2940  */
2941 static void inline
2942 xge_flush_txds(xge_hal_channel_h channelh)
2943 {
2944         xge_lldev_t *lldev = xge_hal_channel_userdata(channelh);
2945         xge_hal_dtr_h tx_dtr;
2946         xge_tx_priv_t *tx_priv;
2947         u8 t_code;
2948
2949         while(xge_hal_fifo_dtr_next_completed(channelh, &tx_dtr, &t_code)
2950             == XGE_HAL_OK) {
2951             XGE_DRV_STATS(tx_desc_compl);
2952             if(t_code) {
2953                 xge_trace(XGE_TRACE, "Tx descriptor with t_code %d", t_code);
2954                 XGE_DRV_STATS(tx_tcode);
2955                 xge_hal_device_handle_tcode(channelh, tx_dtr, t_code);
2956             }
2957
2958             tx_priv = xge_hal_fifo_dtr_private(tx_dtr);
2959             bus_dmamap_unload(lldev->dma_tag_tx, tx_priv->dma_map);
2960             m_freem(tx_priv->buffer);
2961             tx_priv->buffer = NULL;
2962             xge_hal_fifo_dtr_free(channelh, tx_dtr);
2963         }
2964 }
2965
2966 /**
2967  * xge_send
2968  * Transmit function
2969  *
2970  * @ifnetp Interface Handle
2971  */
2972 void
2973 xge_send(struct ifnet *ifnetp)
2974 {
2975         int qindex = 0;
2976         xge_lldev_t *lldev = ifnetp->if_softc;
2977
2978         for(qindex = 0; qindex < XGE_FIFO_COUNT; qindex++) {
2979             if(mtx_trylock(&lldev->mtx_tx[qindex]) == 0) {
2980                 XGE_DRV_STATS(tx_lock_fail);
2981                 break;
2982             }
2983             xge_send_locked(ifnetp, qindex);
2984             mtx_unlock(&lldev->mtx_tx[qindex]);
2985         }
2986 }
2987
2988 static void inline
2989 xge_send_locked(struct ifnet *ifnetp, int qindex)
2990 {
2991         xge_hal_dtr_h            dtr;
2992         static bus_dma_segment_t segs[XGE_MAX_SEGS];
2993         xge_hal_status_e         status;
2994         unsigned int             max_fragments;
2995         xge_lldev_t              *lldev          = ifnetp->if_softc;
2996         xge_hal_channel_h        channelh        = lldev->fifo_channel[qindex];
2997         mbuf_t                   m_head          = NULL;
2998         mbuf_t                   m_buf           = NULL;
2999         xge_tx_priv_t            *ll_tx_priv     = NULL;
3000         register unsigned int    count           = 0;
3001         unsigned int             nsegs           = 0;
3002         u16                      vlan_tag;
3003
3004         max_fragments = ((xge_hal_fifo_t *)channelh)->config->max_frags;
3005
3006         /* If device is not initialized, return */
3007         if((!lldev->initialized) || (!(ifnetp->if_drv_flags & IFF_DRV_RUNNING)))
3008             return;
3009
3010         XGE_DRV_STATS(tx_calls);
3011
3012         /*
3013          * This loop will be executed for each packet in the kernel maintained
3014          * queue -- each packet can be with fragments as an mbuf chain
3015          */
3016         for(;;) {
3017             IF_DEQUEUE(&ifnetp->if_snd, m_head);
3018             if (m_head == NULL) {
3019                 ifnetp->if_drv_flags &= ~(IFF_DRV_OACTIVE);
3020                 return;
3021             }
3022
3023             for(m_buf = m_head; m_buf != NULL; m_buf = m_buf->m_next) {
3024                 if(m_buf->m_len) count += 1;
3025             }
3026
3027             if(count >= max_fragments) {
3028                 m_buf = m_defrag(m_head, M_NOWAIT);
3029                 if(m_buf != NULL) m_head = m_buf;
3030                 XGE_DRV_STATS(tx_defrag);
3031             }
3032
3033             /* Reserve descriptors */
3034             status = xge_hal_fifo_dtr_reserve(channelh, &dtr);
3035             if(status != XGE_HAL_OK) {
3036                 XGE_DRV_STATS(tx_no_txd);
3037                 xge_flush_txds(channelh);
3038                 break;
3039             }
3040
3041             vlan_tag =
3042                 (m_head->m_flags & M_VLANTAG) ? m_head->m_pkthdr.ether_vtag : 0;
3043             xge_hal_fifo_dtr_vlan_set(dtr, vlan_tag);
3044
3045             /* Update Tx private structure for this descriptor */
3046             ll_tx_priv         = xge_hal_fifo_dtr_private(dtr);
3047             ll_tx_priv->buffer = m_head;
3048
3049             /*
3050              * Do mapping -- Required DMA tag has been created in xge_init
3051              * function and DMA maps have already been created in the
3052              * xgell_tx_replenish function.
3053              * Returns number of segments through nsegs
3054              */
3055             if(bus_dmamap_load_mbuf_sg(lldev->dma_tag_tx,
3056                 ll_tx_priv->dma_map, m_head, segs, &nsegs, BUS_DMA_NOWAIT)) {
3057                 xge_trace(XGE_TRACE, "DMA map load failed");
3058                 XGE_DRV_STATS(tx_map_fail);
3059                 break;
3060             }
3061
3062             if(lldev->driver_stats.tx_max_frags < nsegs)
3063                 lldev->driver_stats.tx_max_frags = nsegs;
3064
3065             /* Set descriptor buffer for header and each fragment/segment */
3066             count = 0;
3067             do {
3068                 xge_hal_fifo_dtr_buffer_set(channelh, dtr, count,
3069                     (dma_addr_t)htole64(segs[count].ds_addr),
3070                     segs[count].ds_len);
3071                 count++;
3072             } while(count < nsegs);
3073
3074             /* Pre-write Sync of mapping */
3075             bus_dmamap_sync(lldev->dma_tag_tx, ll_tx_priv->dma_map,
3076                 BUS_DMASYNC_PREWRITE);
3077
3078             if((lldev->enabled_tso) &&
3079                 (m_head->m_pkthdr.csum_flags & CSUM_TSO)) {
3080                 XGE_DRV_STATS(tx_tso);
3081                 xge_hal_fifo_dtr_mss_set(dtr, m_head->m_pkthdr.tso_segsz);
3082             }
3083
3084             /* Checksum */
3085             if(ifnetp->if_hwassist > 0) {
3086                 xge_hal_fifo_dtr_cksum_set_bits(dtr, XGE_HAL_TXD_TX_CKO_IPV4_EN
3087                     | XGE_HAL_TXD_TX_CKO_TCP_EN | XGE_HAL_TXD_TX_CKO_UDP_EN);
3088             }
3089
3090             /* Post descriptor to FIFO channel */
3091             xge_hal_fifo_dtr_post(channelh, dtr);
3092             XGE_DRV_STATS(tx_posted);
3093
3094             /* Send the same copy of mbuf packet to BPF (Berkely Packet Filter)
3095              * listener so that we can use tools like tcpdump */
3096             ETHER_BPF_MTAP(ifnetp, m_head);
3097         }
3098
3099         /* Prepend the packet back to queue */
3100         IF_PREPEND(&ifnetp->if_snd, m_head);
3101         ifnetp->if_drv_flags |= IFF_DRV_OACTIVE;
3102
3103         xge_queue_produce_context(xge_hal_device_queue(lldev->devh),
3104             XGE_LL_EVENT_TRY_XMIT_AGAIN, lldev->devh);
3105         XGE_DRV_STATS(tx_again);
3106 }
3107
3108 /**
3109  * xge_get_buf
3110  * Allocates new mbufs to be placed into descriptors
3111  *
3112  * @dtrh Descriptor Handle
3113  * @rxd_priv Rx Descriptor Private Data
3114  * @lldev Per-adapter Data
3115  * @index Buffer Index (if multi-buffer mode)
3116  *
3117  * Returns XGE_HAL_OK or HAL error enums
3118  */
3119 int
3120 xge_get_buf(xge_hal_dtr_h dtrh, xge_rx_priv_t *rxd_priv,
3121         xge_lldev_t *lldev, int index)
3122 {
3123         register mbuf_t mp            = NULL;
3124         struct          ifnet *ifnetp = lldev->ifnetp;
3125         int             status        = XGE_HAL_OK;
3126         int             buffer_size = 0, cluster_size = 0, count;
3127         bus_dmamap_t    map = rxd_priv->dmainfo[index].dma_map;
3128         bus_dma_segment_t segs[3];
3129
3130         buffer_size = (lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_1) ?
3131             ifnetp->if_mtu + XGE_HAL_MAC_HEADER_MAX_SIZE :
3132             lldev->rxd_mbuf_len[index];
3133
3134         if(buffer_size <= MCLBYTES) {
3135             cluster_size = MCLBYTES;
3136             mp = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
3137         }
3138         else {
3139             cluster_size = MJUMPAGESIZE;
3140             if((lldev->buffer_mode != XGE_HAL_RING_QUEUE_BUFFER_MODE_5) &&
3141                 (buffer_size > MJUMPAGESIZE)) {
3142                 cluster_size = MJUM9BYTES;
3143             }
3144             mp = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, cluster_size);
3145         }
3146         if(!mp) {
3147             xge_trace(XGE_ERR, "Out of memory to allocate mbuf");
3148             status = XGE_HAL_FAIL;
3149             goto getbuf_out;
3150         }
3151
3152         /* Update mbuf's length, packet length and receive interface */
3153         mp->m_len = mp->m_pkthdr.len = buffer_size;
3154         mp->m_pkthdr.rcvif = ifnetp;
3155
3156         /* Load DMA map */
3157         if(bus_dmamap_load_mbuf_sg(lldev->dma_tag_rx, lldev->extra_dma_map,
3158             mp, segs, &count, BUS_DMA_NOWAIT)) {
3159             XGE_DRV_STATS(rx_map_fail);
3160             m_freem(mp);
3161             XGE_EXIT_ON_ERR("DMA map load failed", getbuf_out, XGE_HAL_FAIL);
3162         }
3163
3164         /* Update descriptor private data */
3165         rxd_priv->bufferArray[index]         = mp;
3166         rxd_priv->dmainfo[index].dma_phyaddr = htole64(segs->ds_addr);
3167         rxd_priv->dmainfo[index].dma_map     = lldev->extra_dma_map;
3168         lldev->extra_dma_map = map;
3169
3170         /* Pre-Read/Write sync */
3171         bus_dmamap_sync(lldev->dma_tag_rx, map, BUS_DMASYNC_POSTREAD);
3172
3173         /* Unload DMA map of mbuf in current descriptor */
3174         bus_dmamap_unload(lldev->dma_tag_rx, map);
3175
3176         /* Set descriptor buffer */
3177         if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_1) {
3178             xge_hal_ring_dtr_1b_set(dtrh, rxd_priv->dmainfo[0].dma_phyaddr,
3179                 cluster_size);
3180         }
3181
3182 getbuf_out:
3183         return status;
3184 }
3185
3186 /**
3187  * xge_get_buf_3b_5b
3188  * Allocates new mbufs to be placed into descriptors (in multi-buffer modes)
3189  *
3190  * @dtrh Descriptor Handle
3191  * @rxd_priv Rx Descriptor Private Data
3192  * @lldev Per-adapter Data
3193  *
3194  * Returns XGE_HAL_OK or HAL error enums
3195  */
3196 int
3197 xge_get_buf_3b_5b(xge_hal_dtr_h dtrh, xge_rx_priv_t *rxd_priv,
3198         xge_lldev_t *lldev)
3199 {
3200         bus_addr_t  dma_pointers[5];
3201         int         dma_sizes[5];
3202         int         status = XGE_HAL_OK, index;
3203         int         newindex = 0;
3204
3205         for(index = 0; index < lldev->rxd_mbuf_cnt; index++) {
3206             status = xge_get_buf(dtrh, rxd_priv, lldev, index);
3207             if(status != XGE_HAL_OK) {
3208                 for(newindex = 0; newindex < index; newindex++) {
3209                     m_freem(rxd_priv->bufferArray[newindex]);
3210                 }
3211                 XGE_EXIT_ON_ERR("mbuf allocation failed", _exit, status);
3212             }
3213         }
3214
3215         for(index = 0; index < lldev->buffer_mode; index++) {
3216             if(lldev->rxd_mbuf_len[index] != 0) {
3217                 dma_pointers[index] = rxd_priv->dmainfo[index].dma_phyaddr;
3218                 dma_sizes[index]    = lldev->rxd_mbuf_len[index];
3219             }
3220             else {
3221                 dma_pointers[index] = rxd_priv->dmainfo[index-1].dma_phyaddr;
3222                 dma_sizes[index]    = 1;
3223             }
3224         }
3225
3226         /* Assigning second buffer to third pointer in 2 buffer mode */
3227         if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_2) {
3228             dma_pointers[2] = dma_pointers[1];
3229             dma_sizes[2]    = dma_sizes[1];
3230             dma_sizes[1]    = 1;
3231         }
3232
3233         if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5) {
3234             xge_hal_ring_dtr_5b_set(dtrh, dma_pointers, dma_sizes);
3235         }
3236         else {
3237             xge_hal_ring_dtr_3b_set(dtrh, dma_pointers, dma_sizes);
3238         }
3239
3240 _exit:
3241         return status;
3242 }
3243
3244 /**
3245  * xge_tx_compl
3246  * If the interrupt is due to Tx completion, free the sent buffer
3247  *
3248  * @channelh Channel Handle
3249  * @dtr Descriptor
3250  * @t_code Transfer Code indicating success or error
3251  * @userdata Per-adapter Data
3252  *
3253  * Returns XGE_HAL_OK or HAL error enum
3254  */
3255 xge_hal_status_e
3256 xge_tx_compl(xge_hal_channel_h channelh,
3257         xge_hal_dtr_h dtr, u8 t_code, void *userdata)
3258 {
3259         xge_tx_priv_t *ll_tx_priv = NULL;
3260         xge_lldev_t   *lldev  = (xge_lldev_t *)userdata;
3261         struct ifnet  *ifnetp = lldev->ifnetp;
3262         mbuf_t         m_buffer = NULL;
3263         int            qindex   = xge_hal_channel_id(channelh);
3264
3265         mtx_lock(&lldev->mtx_tx[qindex]);
3266
3267         XGE_DRV_STATS(tx_completions);
3268
3269         /*
3270          * For each completed descriptor: Get private structure, free buffer,
3271          * do unmapping, and free descriptor
3272          */
3273         do {
3274             XGE_DRV_STATS(tx_desc_compl);
3275
3276             if(t_code) {
3277                 XGE_DRV_STATS(tx_tcode);
3278                 xge_trace(XGE_TRACE, "t_code %d", t_code);
3279                 xge_hal_device_handle_tcode(channelh, dtr, t_code);
3280             }
3281
3282             ll_tx_priv = xge_hal_fifo_dtr_private(dtr);
3283             m_buffer   = ll_tx_priv->buffer;
3284             bus_dmamap_unload(lldev->dma_tag_tx, ll_tx_priv->dma_map);
3285             m_freem(m_buffer);
3286             ll_tx_priv->buffer = NULL;
3287             xge_hal_fifo_dtr_free(channelh, dtr);
3288         } while(xge_hal_fifo_dtr_next_completed(channelh, &dtr, &t_code)
3289             == XGE_HAL_OK);
3290         xge_send_locked(ifnetp, qindex);
3291         ifnetp->if_drv_flags &= ~IFF_DRV_OACTIVE;
3292
3293         mtx_unlock(&lldev->mtx_tx[qindex]);
3294
3295         return XGE_HAL_OK;
3296 }
3297
3298 /**
3299  * xge_tx_initial_replenish
3300  * Initially allocate buffers and set them into descriptors for later use
3301  *
3302  * @channelh Tx Channel Handle
3303  * @dtrh Descriptor Handle
3304  * @index
3305  * @userdata Per-adapter Data
3306  * @reopen Channel open/reopen option
3307  *
3308  * Returns XGE_HAL_OK or HAL error enums
3309  */
3310 xge_hal_status_e
3311 xge_tx_initial_replenish(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh,
3312         int index, void *userdata, xge_hal_channel_reopen_e reopen)
3313 {
3314         xge_tx_priv_t *txd_priv = NULL;
3315         int            status   = XGE_HAL_OK;
3316
3317         /* Get the user data portion from channel handle */
3318         xge_lldev_t *lldev = xge_hal_channel_userdata(channelh);
3319         if(lldev == NULL) {
3320             XGE_EXIT_ON_ERR("Failed to get user data from channel", txinit_out,
3321                 XGE_HAL_FAIL);
3322         }
3323
3324         /* Get the private data */
3325         txd_priv = (xge_tx_priv_t *) xge_hal_fifo_dtr_private(dtrh);
3326         if(txd_priv == NULL) {
3327             XGE_EXIT_ON_ERR("Failed to get descriptor private data", txinit_out,
3328                 XGE_HAL_FAIL);
3329         }
3330
3331         /* Create DMA map for this descriptor */
3332         if(bus_dmamap_create(lldev->dma_tag_tx, BUS_DMA_NOWAIT,
3333             &txd_priv->dma_map)) {
3334             XGE_EXIT_ON_ERR("DMA map creation for Tx descriptor failed",
3335                 txinit_out, XGE_HAL_FAIL);
3336         }
3337
3338 txinit_out:
3339         return status;
3340 }
3341
3342 /**
3343  * xge_rx_initial_replenish
3344  * Initially allocate buffers and set them into descriptors for later use
3345  *
3346  * @channelh Tx Channel Handle
3347  * @dtrh Descriptor Handle
3348  * @index Ring Index
3349  * @userdata Per-adapter Data
3350  * @reopen Channel open/reopen option
3351  *
3352  * Returns XGE_HAL_OK or HAL error enums
3353  */
3354 xge_hal_status_e
3355 xge_rx_initial_replenish(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh,
3356         int index, void *userdata, xge_hal_channel_reopen_e reopen)
3357 {
3358         xge_rx_priv_t  *rxd_priv = NULL;
3359         int             status   = XGE_HAL_OK;
3360         int             index1 = 0, index2 = 0;
3361
3362         /* Get the user data portion from channel handle */
3363         xge_lldev_t *lldev = xge_hal_channel_userdata(channelh);
3364         if(lldev == NULL) {
3365             XGE_EXIT_ON_ERR("Failed to get user data from channel", rxinit_out,
3366                 XGE_HAL_FAIL);
3367         }
3368
3369         /* Get the private data */
3370         rxd_priv = (xge_rx_priv_t *) xge_hal_ring_dtr_private(channelh, dtrh);
3371         if(rxd_priv == NULL) {
3372             XGE_EXIT_ON_ERR("Failed to get descriptor private data", rxinit_out,
3373                 XGE_HAL_FAIL);
3374         }
3375
3376         rxd_priv->bufferArray = xge_os_malloc(NULL,
3377                 (sizeof(rxd_priv->bufferArray) * lldev->rxd_mbuf_cnt));
3378
3379         if(rxd_priv->bufferArray == NULL) {
3380             XGE_EXIT_ON_ERR("Failed to allocate Rxd private", rxinit_out,
3381                 XGE_HAL_FAIL);
3382         }
3383
3384         if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_1) {
3385             /* Create DMA map for these descriptors*/
3386             if(bus_dmamap_create(lldev->dma_tag_rx , BUS_DMA_NOWAIT,
3387                 &rxd_priv->dmainfo[0].dma_map)) {
3388                 XGE_EXIT_ON_ERR("DMA map creation for Rx descriptor failed",
3389                     rxinit_err_out, XGE_HAL_FAIL);
3390             }
3391             /* Get a buffer, attach it to this descriptor */
3392             status = xge_get_buf(dtrh, rxd_priv, lldev, 0);
3393         }
3394         else {
3395             for(index1 = 0; index1 < lldev->rxd_mbuf_cnt; index1++) {
3396                 /* Create DMA map for this descriptor */
3397                 if(bus_dmamap_create(lldev->dma_tag_rx , BUS_DMA_NOWAIT ,
3398                     &rxd_priv->dmainfo[index1].dma_map)) {
3399                     for(index2 = index1 - 1; index2 >= 0; index2--) {
3400                         bus_dmamap_destroy(lldev->dma_tag_rx,
3401                             rxd_priv->dmainfo[index2].dma_map);
3402                     }
3403                     XGE_EXIT_ON_ERR(
3404                         "Jumbo DMA map creation for Rx descriptor failed",
3405                         rxinit_err_out, XGE_HAL_FAIL);
3406                 }
3407             }
3408             status = xge_get_buf_3b_5b(dtrh, rxd_priv, lldev);
3409         }
3410
3411         if(status != XGE_HAL_OK) {
3412             for(index1 = 0; index1 < lldev->rxd_mbuf_cnt; index1++) {
3413                 bus_dmamap_destroy(lldev->dma_tag_rx,
3414                     rxd_priv->dmainfo[index1].dma_map);
3415             }
3416             goto rxinit_err_out;
3417         }
3418         else {
3419             goto rxinit_out;
3420         }
3421
3422 rxinit_err_out:
3423         xge_os_free(NULL, rxd_priv->bufferArray,
3424             (sizeof(rxd_priv->bufferArray) * lldev->rxd_mbuf_cnt));
3425 rxinit_out:
3426         return status;
3427 }
3428
3429 /**
3430  * xge_rx_term
3431  * During unload terminate and free all descriptors
3432  *
3433  * @channelh Rx Channel Handle
3434  * @dtrh Rx Descriptor Handle
3435  * @state Descriptor State
3436  * @userdata Per-adapter Data
3437  * @reopen Channel open/reopen option
3438  */
3439 void
3440 xge_rx_term(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh,
3441         xge_hal_dtr_state_e state, void *userdata,
3442         xge_hal_channel_reopen_e reopen)
3443 {
3444         xge_rx_priv_t *rxd_priv = NULL;
3445         xge_lldev_t   *lldev    = NULL;
3446         int            index = 0;
3447
3448         /* Descriptor state is not "Posted" */
3449         if(state != XGE_HAL_DTR_STATE_POSTED) goto rxterm_out;
3450
3451         /* Get the user data portion */
3452         lldev = xge_hal_channel_userdata(channelh);
3453
3454         /* Get the private data */
3455         rxd_priv = (xge_rx_priv_t *) xge_hal_ring_dtr_private(channelh, dtrh);
3456
3457         for(index = 0; index < lldev->rxd_mbuf_cnt; index++) {
3458             if(rxd_priv->dmainfo[index].dma_map != NULL) {
3459                 bus_dmamap_sync(lldev->dma_tag_rx,
3460                     rxd_priv->dmainfo[index].dma_map, BUS_DMASYNC_POSTREAD);
3461                 bus_dmamap_unload(lldev->dma_tag_rx,
3462                     rxd_priv->dmainfo[index].dma_map);
3463                 if(rxd_priv->bufferArray[index] != NULL)
3464                     m_free(rxd_priv->bufferArray[index]);
3465                 bus_dmamap_destroy(lldev->dma_tag_rx,
3466                     rxd_priv->dmainfo[index].dma_map);
3467             }
3468         }
3469         xge_os_free(NULL, rxd_priv->bufferArray,
3470             (sizeof(rxd_priv->bufferArray) * lldev->rxd_mbuf_cnt));
3471
3472         /* Free the descriptor */
3473         xge_hal_ring_dtr_free(channelh, dtrh);
3474
3475 rxterm_out:
3476         return;
3477 }
3478
3479 /**
3480  * xge_tx_term
3481  * During unload terminate and free all descriptors
3482  *
3483  * @channelh Rx Channel Handle
3484  * @dtrh Rx Descriptor Handle
3485  * @state Descriptor State
3486  * @userdata Per-adapter Data
3487  * @reopen Channel open/reopen option
3488  */
3489 void
3490 xge_tx_term(xge_hal_channel_h channelh, xge_hal_dtr_h dtr,
3491         xge_hal_dtr_state_e state, void *userdata,
3492         xge_hal_channel_reopen_e reopen)
3493 {
3494         xge_tx_priv_t *ll_tx_priv = xge_hal_fifo_dtr_private(dtr);
3495         xge_lldev_t   *lldev      = (xge_lldev_t *)userdata;
3496
3497         /* Destroy DMA map */
3498         bus_dmamap_destroy(lldev->dma_tag_tx, ll_tx_priv->dma_map);
3499 }
3500
3501 /**
3502  * xge_methods
3503  *
3504  * FreeBSD device interface entry points
3505  */
3506 static device_method_t xge_methods[] = {
3507         DEVMETHOD(device_probe,     xge_probe),
3508         DEVMETHOD(device_attach,    xge_attach),
3509         DEVMETHOD(device_detach,    xge_detach),
3510         DEVMETHOD(device_shutdown,  xge_shutdown),
3511
3512         DEVMETHOD_END
3513 };
3514
3515 static driver_t xge_driver = {
3516         "nxge",
3517         xge_methods,
3518         sizeof(xge_lldev_t),
3519 };
3520 static devclass_t xge_devclass;
3521 DRIVER_MODULE(nxge, pci, xge_driver, xge_devclass, 0, 0);
3522