2 * Copyright (c) 2002-2007 Neterion, Inc.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 #include <dev/nxge/if_nxge.h>
30 #include <dev/nxge/xge-osdep.h>
31 #include <net/if_arp.h>
32 #include <sys/types.h>
34 #include <net/if_var.h>
35 #include <net/if_vlan_var.h>
37 int copyright_print = 0;
38 int hal_driver_init_count = 0;
39 size_t size = sizeof(int);
41 static void inline xge_flush_txds(xge_hal_channel_h);
45 * Probes for Xframe devices
50 * BUS_PROBE_DEFAULT if device is supported
51 * ENXIO if device is not supported
54 xge_probe(device_t dev)
56 int devid = pci_get_device(dev);
57 int vendorid = pci_get_vendor(dev);
60 if(vendorid == XGE_PCI_VENDOR_ID) {
61 if((devid == XGE_PCI_DEVICE_ID_XENA_2) ||
62 (devid == XGE_PCI_DEVICE_ID_HERC_2)) {
63 if(!copyright_print) {
64 xge_os_printf(XGE_COPYRIGHT);
67 device_set_desc_copy(dev,
68 "Neterion Xframe 10 Gigabit Ethernet Adapter");
69 retValue = BUS_PROBE_DEFAULT;
78 * Sets HAL parameter values (from kenv).
80 * @dconfig Device Configuration
84 xge_init_params(xge_hal_device_config_t *dconfig, device_t dev)
86 int qindex, tindex, revision;
88 xge_lldev_t *lldev = (xge_lldev_t *)device_get_softc(dev);
90 dconfig->mtu = XGE_DEFAULT_INITIAL_MTU;
91 dconfig->pci_freq_mherz = XGE_DEFAULT_USER_HARDCODED;
92 dconfig->device_poll_millis = XGE_HAL_DEFAULT_DEVICE_POLL_MILLIS;
93 dconfig->link_stability_period = XGE_HAL_DEFAULT_LINK_STABILITY_PERIOD;
94 dconfig->mac.rmac_bcast_en = XGE_DEFAULT_MAC_RMAC_BCAST_EN;
95 dconfig->fifo.alignment_size = XGE_DEFAULT_FIFO_ALIGNMENT_SIZE;
97 XGE_GET_PARAM("hw.xge.enable_tso", (*lldev), enabled_tso,
98 XGE_DEFAULT_ENABLED_TSO);
99 XGE_GET_PARAM("hw.xge.enable_lro", (*lldev), enabled_lro,
100 XGE_DEFAULT_ENABLED_LRO);
101 XGE_GET_PARAM("hw.xge.enable_msi", (*lldev), enabled_msi,
102 XGE_DEFAULT_ENABLED_MSI);
104 XGE_GET_PARAM("hw.xge.latency_timer", (*dconfig), latency_timer,
105 XGE_DEFAULT_LATENCY_TIMER);
106 XGE_GET_PARAM("hw.xge.max_splits_trans", (*dconfig), max_splits_trans,
107 XGE_DEFAULT_MAX_SPLITS_TRANS);
108 XGE_GET_PARAM("hw.xge.mmrb_count", (*dconfig), mmrb_count,
109 XGE_DEFAULT_MMRB_COUNT);
110 XGE_GET_PARAM("hw.xge.shared_splits", (*dconfig), shared_splits,
111 XGE_DEFAULT_SHARED_SPLITS);
112 XGE_GET_PARAM("hw.xge.isr_polling_cnt", (*dconfig), isr_polling_cnt,
113 XGE_DEFAULT_ISR_POLLING_CNT);
114 XGE_GET_PARAM("hw.xge.stats_refresh_time_sec", (*dconfig),
115 stats_refresh_time_sec, XGE_DEFAULT_STATS_REFRESH_TIME_SEC);
117 XGE_GET_PARAM_MAC("hw.xge.mac_tmac_util_period", tmac_util_period,
118 XGE_DEFAULT_MAC_TMAC_UTIL_PERIOD);
119 XGE_GET_PARAM_MAC("hw.xge.mac_rmac_util_period", rmac_util_period,
120 XGE_DEFAULT_MAC_RMAC_UTIL_PERIOD);
121 XGE_GET_PARAM_MAC("hw.xge.mac_rmac_pause_gen_en", rmac_pause_gen_en,
122 XGE_DEFAULT_MAC_RMAC_PAUSE_GEN_EN);
123 XGE_GET_PARAM_MAC("hw.xge.mac_rmac_pause_rcv_en", rmac_pause_rcv_en,
124 XGE_DEFAULT_MAC_RMAC_PAUSE_RCV_EN);
125 XGE_GET_PARAM_MAC("hw.xge.mac_rmac_pause_time", rmac_pause_time,
126 XGE_DEFAULT_MAC_RMAC_PAUSE_TIME);
127 XGE_GET_PARAM_MAC("hw.xge.mac_mc_pause_threshold_q0q3",
128 mc_pause_threshold_q0q3, XGE_DEFAULT_MAC_MC_PAUSE_THRESHOLD_Q0Q3);
129 XGE_GET_PARAM_MAC("hw.xge.mac_mc_pause_threshold_q4q7",
130 mc_pause_threshold_q4q7, XGE_DEFAULT_MAC_MC_PAUSE_THRESHOLD_Q4Q7);
132 XGE_GET_PARAM_FIFO("hw.xge.fifo_memblock_size", memblock_size,
133 XGE_DEFAULT_FIFO_MEMBLOCK_SIZE);
134 XGE_GET_PARAM_FIFO("hw.xge.fifo_reserve_threshold", reserve_threshold,
135 XGE_DEFAULT_FIFO_RESERVE_THRESHOLD);
136 XGE_GET_PARAM_FIFO("hw.xge.fifo_max_frags", max_frags,
137 XGE_DEFAULT_FIFO_MAX_FRAGS);
139 for(qindex = 0; qindex < XGE_FIFO_COUNT; qindex++) {
140 XGE_GET_PARAM_FIFO_QUEUE("hw.xge.fifo_queue_intr", intr, qindex,
141 XGE_DEFAULT_FIFO_QUEUE_INTR);
142 XGE_GET_PARAM_FIFO_QUEUE("hw.xge.fifo_queue_max", max, qindex,
143 XGE_DEFAULT_FIFO_QUEUE_MAX);
144 XGE_GET_PARAM_FIFO_QUEUE("hw.xge.fifo_queue_initial", initial,
145 qindex, XGE_DEFAULT_FIFO_QUEUE_INITIAL);
147 for (tindex = 0; tindex < XGE_HAL_MAX_FIFO_TTI_NUM; tindex++) {
148 dconfig->fifo.queue[qindex].tti[tindex].enabled = 1;
149 dconfig->fifo.queue[qindex].configured = 1;
151 XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_urange_a",
152 urange_a, qindex, tindex,
153 XGE_DEFAULT_FIFO_QUEUE_TTI_URANGE_A);
154 XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_urange_b",
155 urange_b, qindex, tindex,
156 XGE_DEFAULT_FIFO_QUEUE_TTI_URANGE_B);
157 XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_urange_c",
158 urange_c, qindex, tindex,
159 XGE_DEFAULT_FIFO_QUEUE_TTI_URANGE_C);
160 XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_ufc_a",
161 ufc_a, qindex, tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_UFC_A);
162 XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_ufc_b",
163 ufc_b, qindex, tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_UFC_B);
164 XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_ufc_c",
165 ufc_c, qindex, tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_UFC_C);
166 XGE_GET_PARAM_FIFO_QUEUE_TTI("hw.xge.fifo_queue_tti_ufc_d",
167 ufc_d, qindex, tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_UFC_D);
168 XGE_GET_PARAM_FIFO_QUEUE_TTI(
169 "hw.xge.fifo_queue_tti_timer_ci_en", timer_ci_en, qindex,
170 tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_TIMER_CI_EN);
171 XGE_GET_PARAM_FIFO_QUEUE_TTI(
172 "hw.xge.fifo_queue_tti_timer_ac_en", timer_ac_en, qindex,
173 tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_TIMER_AC_EN);
174 XGE_GET_PARAM_FIFO_QUEUE_TTI(
175 "hw.xge.fifo_queue_tti_timer_val_us", timer_val_us, qindex,
176 tindex, XGE_DEFAULT_FIFO_QUEUE_TTI_TIMER_VAL_US);
180 XGE_GET_PARAM_RING("hw.xge.ring_memblock_size", memblock_size,
181 XGE_DEFAULT_RING_MEMBLOCK_SIZE);
183 XGE_GET_PARAM_RING("hw.xge.ring_strip_vlan_tag", strip_vlan_tag,
184 XGE_DEFAULT_RING_STRIP_VLAN_TAG);
186 XGE_GET_PARAM("hw.xge.buffer_mode", (*lldev), buffer_mode,
187 XGE_DEFAULT_BUFFER_MODE);
188 if((lldev->buffer_mode < XGE_HAL_RING_QUEUE_BUFFER_MODE_1) ||
189 (lldev->buffer_mode > XGE_HAL_RING_QUEUE_BUFFER_MODE_2)) {
190 xge_trace(XGE_ERR, "Supported buffer modes are 1 and 2");
191 lldev->buffer_mode = XGE_HAL_RING_QUEUE_BUFFER_MODE_1;
194 for (qindex = 0; qindex < XGE_RING_COUNT; qindex++) {
195 dconfig->ring.queue[qindex].max_frm_len = XGE_HAL_RING_USE_MTU;
196 dconfig->ring.queue[qindex].priority = 0;
197 dconfig->ring.queue[qindex].configured = 1;
198 dconfig->ring.queue[qindex].buffer_mode =
199 (lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_2) ?
200 XGE_HAL_RING_QUEUE_BUFFER_MODE_3 : lldev->buffer_mode;
202 XGE_GET_PARAM_RING_QUEUE("hw.xge.ring_queue_max", max, qindex,
203 XGE_DEFAULT_RING_QUEUE_MAX);
204 XGE_GET_PARAM_RING_QUEUE("hw.xge.ring_queue_initial", initial,
205 qindex, XGE_DEFAULT_RING_QUEUE_INITIAL);
206 XGE_GET_PARAM_RING_QUEUE("hw.xge.ring_queue_dram_size_mb",
207 dram_size_mb, qindex, XGE_DEFAULT_RING_QUEUE_DRAM_SIZE_MB);
208 XGE_GET_PARAM_RING_QUEUE("hw.xge.ring_queue_indicate_max_pkts",
209 indicate_max_pkts, qindex,
210 XGE_DEFAULT_RING_QUEUE_INDICATE_MAX_PKTS);
211 XGE_GET_PARAM_RING_QUEUE("hw.xge.ring_queue_backoff_interval_us",
212 backoff_interval_us, qindex,
213 XGE_DEFAULT_RING_QUEUE_BACKOFF_INTERVAL_US);
215 XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_ufc_a", ufc_a,
216 qindex, XGE_DEFAULT_RING_QUEUE_RTI_UFC_A);
217 XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_ufc_b", ufc_b,
218 qindex, XGE_DEFAULT_RING_QUEUE_RTI_UFC_B);
219 XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_ufc_c", ufc_c,
220 qindex, XGE_DEFAULT_RING_QUEUE_RTI_UFC_C);
221 XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_ufc_d", ufc_d,
222 qindex, XGE_DEFAULT_RING_QUEUE_RTI_UFC_D);
223 XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_timer_ac_en",
224 timer_ac_en, qindex, XGE_DEFAULT_RING_QUEUE_RTI_TIMER_AC_EN);
225 XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_timer_val_us",
226 timer_val_us, qindex, XGE_DEFAULT_RING_QUEUE_RTI_TIMER_VAL_US);
227 XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_urange_a",
228 urange_a, qindex, XGE_DEFAULT_RING_QUEUE_RTI_URANGE_A);
229 XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_urange_b",
230 urange_b, qindex, XGE_DEFAULT_RING_QUEUE_RTI_URANGE_B);
231 XGE_GET_PARAM_RING_QUEUE_RTI("hw.xge.ring_queue_rti_urange_c",
232 urange_c, qindex, XGE_DEFAULT_RING_QUEUE_RTI_URANGE_C);
235 if(dconfig->fifo.max_frags > (PAGE_SIZE/32)) {
236 xge_os_printf("fifo_max_frags = %d", dconfig->fifo.max_frags)
237 xge_os_printf("fifo_max_frags should be <= (PAGE_SIZE / 32) = %d",
238 (int)(PAGE_SIZE / 32))
239 xge_os_printf("Using fifo_max_frags = %d", (int)(PAGE_SIZE / 32))
240 dconfig->fifo.max_frags = (PAGE_SIZE / 32);
243 checkdev = pci_find_device(VENDOR_ID_AMD, DEVICE_ID_8131_PCI_BRIDGE);
244 if(checkdev != NULL) {
245 /* Check Revision for 0x12 */
246 revision = pci_read_config(checkdev,
247 xge_offsetof(xge_hal_pci_config_t, revision), 1);
248 if(revision <= 0x12) {
249 /* Set mmrb_count to 1k and max splits = 2 */
250 dconfig->mmrb_count = 1;
251 dconfig->max_splits_trans = XGE_HAL_THREE_SPLIT_TRANSACTION;
257 * xge_buffer_sizes_set
258 * Set buffer sizes based on Rx buffer mode
260 * @lldev Per-adapter Data
261 * @buffer_mode Rx Buffer Mode
264 xge_rx_buffer_sizes_set(xge_lldev_t *lldev, int buffer_mode, int mtu)
267 int frame_header = XGE_HAL_MAC_HEADER_MAX_SIZE;
268 int buffer_size = mtu + frame_header;
270 xge_os_memzero(lldev->rxd_mbuf_len, sizeof(lldev->rxd_mbuf_len));
272 if(buffer_mode != XGE_HAL_RING_QUEUE_BUFFER_MODE_5)
273 lldev->rxd_mbuf_len[buffer_mode - 1] = mtu;
275 lldev->rxd_mbuf_len[0] = (buffer_mode == 1) ? buffer_size:frame_header;
277 if(buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5)
278 lldev->rxd_mbuf_len[1] = XGE_HAL_TCPIP_HEADER_MAX_SIZE;
280 if(buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5) {
282 buffer_size -= XGE_HAL_TCPIP_HEADER_MAX_SIZE;
283 while(buffer_size > MJUMPAGESIZE) {
284 lldev->rxd_mbuf_len[index++] = MJUMPAGESIZE;
285 buffer_size -= MJUMPAGESIZE;
287 XGE_ALIGN_TO(buffer_size, 128);
288 lldev->rxd_mbuf_len[index] = buffer_size;
289 lldev->rxd_mbuf_cnt = index + 1;
292 for(index = 0; index < buffer_mode; index++)
293 xge_trace(XGE_TRACE, "Buffer[%d] %d\n", index,
294 lldev->rxd_mbuf_len[index]);
298 * xge_buffer_mode_init
299 * Init Rx buffer mode
301 * @lldev Per-adapter Data
305 xge_buffer_mode_init(xge_lldev_t *lldev, int mtu)
307 int index = 0, buffer_size = 0;
308 xge_hal_ring_config_t *ring_config = &((lldev->devh)->config.ring);
310 buffer_size = mtu + XGE_HAL_MAC_HEADER_MAX_SIZE;
312 if(lldev->enabled_lro)
313 (lldev->ifnetp)->if_capenable |= IFCAP_LRO;
315 (lldev->ifnetp)->if_capenable &= ~IFCAP_LRO;
317 lldev->rxd_mbuf_cnt = lldev->buffer_mode;
318 if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_2) {
319 XGE_SET_BUFFER_MODE_IN_RINGS(XGE_HAL_RING_QUEUE_BUFFER_MODE_3);
320 ring_config->scatter_mode = XGE_HAL_RING_QUEUE_SCATTER_MODE_B;
323 XGE_SET_BUFFER_MODE_IN_RINGS(lldev->buffer_mode);
324 ring_config->scatter_mode = XGE_HAL_RING_QUEUE_SCATTER_MODE_A;
326 xge_rx_buffer_sizes_set(lldev, lldev->buffer_mode, mtu);
328 xge_os_printf("%s: TSO %s", device_get_nameunit(lldev->device),
329 ((lldev->enabled_tso) ? "Enabled":"Disabled"));
330 xge_os_printf("%s: LRO %s", device_get_nameunit(lldev->device),
331 ((lldev->ifnetp)->if_capenable & IFCAP_LRO) ? "Enabled":"Disabled");
332 xge_os_printf("%s: Rx %d Buffer Mode Enabled",
333 device_get_nameunit(lldev->device), lldev->buffer_mode);
337 * xge_driver_initialize
338 * Initializes HAL driver (common for all devices)
341 * XGE_HAL_OK if success
342 * XGE_HAL_ERR_BAD_DRIVER_CONFIG if driver configuration parameters are invalid
345 xge_driver_initialize(void)
347 xge_hal_uld_cbs_t uld_callbacks;
348 xge_hal_driver_config_t driver_config;
349 xge_hal_status_e status = XGE_HAL_OK;
351 /* Initialize HAL driver */
352 if(!hal_driver_init_count) {
353 xge_os_memzero(&uld_callbacks, sizeof(xge_hal_uld_cbs_t));
354 xge_os_memzero(&driver_config, sizeof(xge_hal_driver_config_t));
357 * Initial and maximum size of the queue used to store the events
358 * like Link up/down (xge_hal_event_e)
360 driver_config.queue_size_initial = XGE_HAL_MIN_QUEUE_SIZE_INITIAL;
361 driver_config.queue_size_max = XGE_HAL_MAX_QUEUE_SIZE_MAX;
363 uld_callbacks.link_up = xge_callback_link_up;
364 uld_callbacks.link_down = xge_callback_link_down;
365 uld_callbacks.crit_err = xge_callback_crit_err;
366 uld_callbacks.event = xge_callback_event;
368 status = xge_hal_driver_initialize(&driver_config, &uld_callbacks);
369 if(status != XGE_HAL_OK) {
370 XGE_EXIT_ON_ERR("xgeX: Initialization of HAL driver failed",
374 hal_driver_init_count = hal_driver_init_count + 1;
376 xge_hal_driver_debug_module_mask_set(0xffffffff);
377 xge_hal_driver_debug_level_set(XGE_TRACE);
385 * Initializes, adds and sets media
387 * @devc Device Handle
390 xge_media_init(device_t devc)
392 xge_lldev_t *lldev = (xge_lldev_t *)device_get_softc(devc);
394 /* Initialize Media */
395 ifmedia_init(&lldev->media, IFM_IMASK, xge_ifmedia_change,
398 /* Add supported media */
399 ifmedia_add(&lldev->media, IFM_ETHER | IFM_1000_SX | IFM_FDX, 0, NULL);
400 ifmedia_add(&lldev->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
401 ifmedia_add(&lldev->media, IFM_ETHER | IFM_AUTO, 0, NULL);
402 ifmedia_add(&lldev->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
403 ifmedia_add(&lldev->media, IFM_ETHER | IFM_10G_LR, 0, NULL);
406 ifmedia_set(&lldev->media, IFM_ETHER | IFM_AUTO);
411 * Save PCI configuration space
416 xge_pci_space_save(device_t dev)
418 struct pci_devinfo *dinfo = NULL;
420 dinfo = device_get_ivars(dev);
421 xge_trace(XGE_TRACE, "Saving PCI configuration space");
422 pci_cfg_save(dev, dinfo, 0);
426 * xge_pci_space_restore
427 * Restore saved PCI configuration space
432 xge_pci_space_restore(device_t dev)
434 struct pci_devinfo *dinfo = NULL;
436 dinfo = device_get_ivars(dev);
437 xge_trace(XGE_TRACE, "Restoring PCI configuration space");
438 pci_cfg_restore(dev, dinfo);
445 * @lldev Per-adapter Data
448 xge_msi_info_save(xge_lldev_t * lldev)
450 xge_os_pci_read16(lldev->pdev, NULL,
451 xge_offsetof(xge_hal_pci_config_le_t, msi_control),
452 &lldev->msi_info.msi_control);
453 xge_os_pci_read32(lldev->pdev, NULL,
454 xge_offsetof(xge_hal_pci_config_le_t, msi_lower_address),
455 &lldev->msi_info.msi_lower_address);
456 xge_os_pci_read32(lldev->pdev, NULL,
457 xge_offsetof(xge_hal_pci_config_le_t, msi_higher_address),
458 &lldev->msi_info.msi_higher_address);
459 xge_os_pci_read16(lldev->pdev, NULL,
460 xge_offsetof(xge_hal_pci_config_le_t, msi_data),
461 &lldev->msi_info.msi_data);
465 * xge_msi_info_restore
466 * Restore saved MSI info
471 xge_msi_info_restore(xge_lldev_t *lldev)
474 * If interface is made down and up, traffic fails. It was observed that
475 * MSI information were getting reset on down. Restoring them.
477 xge_os_pci_write16(lldev->pdev, NULL,
478 xge_offsetof(xge_hal_pci_config_le_t, msi_control),
479 lldev->msi_info.msi_control);
481 xge_os_pci_write32(lldev->pdev, NULL,
482 xge_offsetof(xge_hal_pci_config_le_t, msi_lower_address),
483 lldev->msi_info.msi_lower_address);
485 xge_os_pci_write32(lldev->pdev, NULL,
486 xge_offsetof(xge_hal_pci_config_le_t, msi_higher_address),
487 lldev->msi_info.msi_higher_address);
489 xge_os_pci_write16(lldev->pdev, NULL,
490 xge_offsetof(xge_hal_pci_config_le_t, msi_data),
491 lldev->msi_info.msi_data);
496 * Initializes mutexes used in driver
498 * @lldev Per-adapter Data
501 xge_mutex_init(xge_lldev_t *lldev)
505 sprintf(lldev->mtx_name_drv, "%s_drv",
506 device_get_nameunit(lldev->device));
507 mtx_init(&lldev->mtx_drv, lldev->mtx_name_drv, MTX_NETWORK_LOCK,
510 for(qindex = 0; qindex < XGE_FIFO_COUNT; qindex++) {
511 sprintf(lldev->mtx_name_tx[qindex], "%s_tx_%d",
512 device_get_nameunit(lldev->device), qindex);
513 mtx_init(&lldev->mtx_tx[qindex], lldev->mtx_name_tx[qindex], NULL,
520 * Destroys mutexes used in driver
522 * @lldev Per-adapter Data
525 xge_mutex_destroy(xge_lldev_t *lldev)
529 for(qindex = 0; qindex < XGE_FIFO_COUNT; qindex++)
530 mtx_destroy(&lldev->mtx_tx[qindex]);
531 mtx_destroy(&lldev->mtx_drv);
536 * Print device and driver information
538 * @lldev Per-adapter Data
541 xge_print_info(xge_lldev_t *lldev)
543 device_t dev = lldev->device;
544 xge_hal_device_t *hldev = lldev->devh;
545 xge_hal_status_e status = XGE_HAL_OK;
547 const char *xge_pci_bus_speeds[17] = {
550 "PCIX(M1) 66MHz Bus",
551 "PCIX(M1) 100MHz Bus",
552 "PCIX(M1) 133MHz Bus",
553 "PCIX(M2) 133MHz Bus",
554 "PCIX(M2) 200MHz Bus",
555 "PCIX(M2) 266MHz Bus",
557 "PCIX(M1) 66MHz Bus (Not Supported)",
558 "PCIX(M1) 100MHz Bus (Not Supported)",
559 "PCIX(M1) 133MHz Bus (Not Supported)",
567 xge_os_printf("%s: Xframe%s %s Revision %d Driver v%s",
568 device_get_nameunit(dev),
569 ((hldev->device_id == XGE_PCI_DEVICE_ID_XENA_2) ? "I" : "II"),
570 hldev->vpd_data.product_name, hldev->revision, XGE_DRIVER_VERSION);
571 xge_os_printf("%s: Serial Number %s",
572 device_get_nameunit(dev), hldev->vpd_data.serial_num);
574 if(pci_get_device(dev) == XGE_PCI_DEVICE_ID_HERC_2) {
575 status = xge_hal_mgmt_reg_read(hldev, 0,
576 xge_offsetof(xge_hal_pci_bar0_t, pci_info), &val64);
577 if(status != XGE_HAL_OK)
578 xge_trace(XGE_ERR, "Error for getting bus speed");
580 xge_os_printf("%s: Adapter is on %s bit %s",
581 device_get_nameunit(dev), ((val64 & BIT(8)) ? "32":"64"),
582 (xge_pci_bus_speeds[((val64 & XGE_HAL_PCI_INFO) >> 60)]));
585 xge_os_printf("%s: Using %s Interrupts",
586 device_get_nameunit(dev),
587 (lldev->enabled_msi == XGE_HAL_INTR_MODE_MSI) ? "MSI":"Line");
591 * xge_create_dma_tags
592 * Creates DMA tags for both Tx and Rx
596 * Returns XGE_HAL_OK or XGE_HAL_FAIL (if errors)
599 xge_create_dma_tags(device_t dev)
601 xge_lldev_t *lldev = (xge_lldev_t *)device_get_softc(dev);
602 xge_hal_status_e status = XGE_HAL_FAIL;
603 int mtu = (lldev->ifnetp)->if_mtu, maxsize;
606 status = bus_dma_tag_create(
607 bus_get_dma_tag(dev), /* Parent */
608 PAGE_SIZE, /* Alignment */
610 BUS_SPACE_MAXADDR, /* Low Address */
611 BUS_SPACE_MAXADDR, /* High Address */
612 NULL, /* Filter Function */
613 NULL, /* Filter Function Arguments */
614 MCLBYTES * XGE_MAX_SEGS, /* Maximum Size */
615 XGE_MAX_SEGS, /* Number of Segments */
616 MCLBYTES, /* Maximum Segment Size */
617 BUS_DMA_ALLOCNOW, /* Flags */
618 NULL, /* Lock Function */
619 NULL, /* Lock Function Arguments */
620 (&lldev->dma_tag_tx)); /* DMA Tag */
624 maxsize = mtu + XGE_HAL_MAC_HEADER_MAX_SIZE;
625 if(maxsize <= MCLBYTES) {
629 if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5)
630 maxsize = MJUMPAGESIZE;
632 maxsize = (maxsize <= MJUMPAGESIZE) ? MJUMPAGESIZE : MJUM9BYTES;
636 status = bus_dma_tag_create(
637 bus_get_dma_tag(dev), /* Parent */
638 PAGE_SIZE, /* Alignment */
640 BUS_SPACE_MAXADDR, /* Low Address */
641 BUS_SPACE_MAXADDR, /* High Address */
642 NULL, /* Filter Function */
643 NULL, /* Filter Function Arguments */
644 maxsize, /* Maximum Size */
645 1, /* Number of Segments */
646 maxsize, /* Maximum Segment Size */
647 BUS_DMA_ALLOCNOW, /* Flags */
648 NULL, /* Lock Function */
649 NULL, /* Lock Function Arguments */
650 (&lldev->dma_tag_rx)); /* DMA Tag */
654 status = bus_dmamap_create(lldev->dma_tag_rx, BUS_DMA_NOWAIT,
655 &lldev->extra_dma_map);
663 status = bus_dma_tag_destroy(lldev->dma_tag_rx);
665 xge_trace(XGE_ERR, "Rx DMA tag destroy failed");
667 status = bus_dma_tag_destroy(lldev->dma_tag_tx);
669 xge_trace(XGE_ERR, "Tx DMA tag destroy failed");
670 status = XGE_HAL_FAIL;
676 * xge_confirm_changes
677 * Disables and Enables interface to apply requested change
679 * @lldev Per-adapter Data
680 * @mtu_set Is it called for changing MTU? (Yes: 1, No: 0)
682 * Returns 0 or Error Number
685 xge_confirm_changes(xge_lldev_t *lldev, xge_option_e option)
687 if(lldev->initialized == 0) goto _exit1;
689 mtx_lock(&lldev->mtx_drv);
690 if_down(lldev->ifnetp);
691 xge_device_stop(lldev, XGE_HAL_CHANNEL_OC_NORMAL);
693 if(option == XGE_SET_MTU)
694 (lldev->ifnetp)->if_mtu = lldev->mtu;
696 xge_buffer_mode_init(lldev, lldev->mtu);
698 xge_device_init(lldev, XGE_HAL_CHANNEL_OC_NORMAL);
699 if_up(lldev->ifnetp);
700 mtx_unlock(&lldev->mtx_drv);
704 /* Request was to change MTU and device not initialized */
705 if(option == XGE_SET_MTU) {
706 (lldev->ifnetp)->if_mtu = lldev->mtu;
707 xge_buffer_mode_init(lldev, lldev->mtu);
714 * xge_change_lro_status
715 * Enable/Disable LRO feature
717 * @SYSCTL_HANDLER_ARGS sysctl_oid structure with arguments
719 * Returns 0 or error number.
722 xge_change_lro_status(SYSCTL_HANDLER_ARGS)
724 xge_lldev_t *lldev = (xge_lldev_t *)arg1;
725 int request = lldev->enabled_lro, status = XGE_HAL_OK;
727 status = sysctl_handle_int(oidp, &request, arg2, req);
728 if((status != XGE_HAL_OK) || (!req->newptr))
731 if((request < 0) || (request > 1)) {
736 /* Return if current and requested states are same */
737 if(request == lldev->enabled_lro){
738 xge_trace(XGE_ERR, "LRO is already %s",
739 ((request) ? "enabled" : "disabled"));
743 lldev->enabled_lro = request;
744 xge_confirm_changes(lldev, XGE_CHANGE_LRO);
745 arg2 = lldev->enabled_lro;
752 * xge_add_sysctl_handlers
753 * Registers sysctl parameter value update handlers
755 * @lldev Per-adapter data
758 xge_add_sysctl_handlers(xge_lldev_t *lldev)
760 struct sysctl_ctx_list *context_list =
761 device_get_sysctl_ctx(lldev->device);
762 struct sysctl_oid *oid = device_get_sysctl_tree(lldev->device);
764 SYSCTL_ADD_PROC(context_list, SYSCTL_CHILDREN(oid), OID_AUTO,
765 "enable_lro", CTLTYPE_INT | CTLFLAG_RW, lldev, 0,
766 xge_change_lro_status, "I", "Enable or disable LRO feature");
771 * Connects driver to the system if probe was success
776 xge_attach(device_t dev)
778 xge_hal_device_config_t *device_config;
779 xge_hal_device_attr_t attr;
781 xge_hal_device_t *hldev;
782 xge_pci_info_t *pci_info;
783 struct ifnet *ifnetp;
784 int rid, rid0, rid1, error;
785 int msi_count = 0, status = XGE_HAL_OK;
786 int enable_msi = XGE_HAL_INTR_MODE_IRQLINE;
788 device_config = xge_os_malloc(NULL, sizeof(xge_hal_device_config_t));
790 XGE_EXIT_ON_ERR("Memory allocation for device configuration failed",
791 attach_out_config, ENOMEM);
794 lldev = (xge_lldev_t *) device_get_softc(dev);
796 XGE_EXIT_ON_ERR("Adapter softc is NULL", attach_out, ENOMEM);
800 xge_mutex_init(lldev);
802 error = xge_driver_initialize();
803 if(error != XGE_HAL_OK) {
804 xge_resources_free(dev, xge_free_mutex);
805 XGE_EXIT_ON_ERR("Initializing driver failed", attach_out, ENXIO);
810 (xge_hal_device_t *)xge_os_malloc(NULL, sizeof(xge_hal_device_t));
812 xge_resources_free(dev, xge_free_terminate_hal_driver);
813 XGE_EXIT_ON_ERR("Memory allocation for HAL device failed",
818 /* Our private structure */
820 (xge_pci_info_t*) xge_os_malloc(NULL, sizeof(xge_pci_info_t));
822 xge_resources_free(dev, xge_free_hal_device);
823 XGE_EXIT_ON_ERR("Memory allocation for PCI info. failed",
826 lldev->pdev = pci_info;
827 pci_info->device = dev;
830 pci_enable_busmaster(dev);
832 /* Get virtual address for BAR0 */
834 pci_info->regmap0 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid0,
836 if(pci_info->regmap0 == NULL) {
837 xge_resources_free(dev, xge_free_pci_info);
838 XGE_EXIT_ON_ERR("Bus resource allocation for BAR0 failed",
841 attr.bar0 = (char *)pci_info->regmap0;
843 pci_info->bar0resource = (xge_bus_resource_t*)
844 xge_os_malloc(NULL, sizeof(xge_bus_resource_t));
845 if(pci_info->bar0resource == NULL) {
846 xge_resources_free(dev, xge_free_bar0);
847 XGE_EXIT_ON_ERR("Memory allocation for BAR0 Resources failed",
850 ((xge_bus_resource_t *)(pci_info->bar0resource))->bus_tag =
851 rman_get_bustag(pci_info->regmap0);
852 ((xge_bus_resource_t *)(pci_info->bar0resource))->bus_handle =
853 rman_get_bushandle(pci_info->regmap0);
854 ((xge_bus_resource_t *)(pci_info->bar0resource))->bar_start_addr =
857 /* Get virtual address for BAR1 */
859 pci_info->regmap1 = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid1,
861 if(pci_info->regmap1 == NULL) {
862 xge_resources_free(dev, xge_free_bar0_resource);
863 XGE_EXIT_ON_ERR("Bus resource allocation for BAR1 failed",
866 attr.bar1 = (char *)pci_info->regmap1;
868 pci_info->bar1resource = (xge_bus_resource_t*)
869 xge_os_malloc(NULL, sizeof(xge_bus_resource_t));
870 if(pci_info->bar1resource == NULL) {
871 xge_resources_free(dev, xge_free_bar1);
872 XGE_EXIT_ON_ERR("Memory allocation for BAR1 Resources failed",
875 ((xge_bus_resource_t *)(pci_info->bar1resource))->bus_tag =
876 rman_get_bustag(pci_info->regmap1);
877 ((xge_bus_resource_t *)(pci_info->bar1resource))->bus_handle =
878 rman_get_bushandle(pci_info->regmap1);
879 ((xge_bus_resource_t *)(pci_info->bar1resource))->bar_start_addr =
882 /* Save PCI config space */
883 xge_pci_space_save(dev);
885 attr.regh0 = (xge_bus_resource_t *) pci_info->bar0resource;
886 attr.regh1 = (xge_bus_resource_t *) pci_info->bar1resource;
887 attr.irqh = lldev->irqhandle;
888 attr.cfgh = pci_info;
889 attr.pdev = pci_info;
891 /* Initialize device configuration parameters */
892 xge_init_params(device_config, dev);
895 if(lldev->enabled_msi) {
896 /* Number of MSI messages supported by device */
897 msi_count = pci_msi_count(dev);
899 /* Device supports MSI */
901 xge_trace(XGE_ERR, "MSI count: %d", msi_count);
902 xge_trace(XGE_ERR, "Now, driver supporting 1 message");
905 error = pci_alloc_msi(dev, &msi_count);
908 xge_trace(XGE_ERR, "Allocated messages: %d", msi_count);
909 enable_msi = XGE_HAL_INTR_MODE_MSI;
914 xge_trace(XGE_ERR, "pci_alloc_msi failed, %d", error);
918 lldev->enabled_msi = enable_msi;
920 /* Allocate resource for irq */
921 lldev->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
922 (RF_SHAREABLE | RF_ACTIVE));
923 if(lldev->irq == NULL) {
924 xge_trace(XGE_ERR, "Allocating irq resource for %s failed",
925 ((rid == 0) ? "line interrupt" : "MSI"));
927 error = pci_release_msi(dev);
929 xge_trace(XGE_ERR, "Releasing MSI resources failed %d",
931 xge_trace(XGE_ERR, "Requires reboot to use MSI again");
933 xge_trace(XGE_ERR, "Trying line interrupts");
935 lldev->enabled_msi = XGE_HAL_INTR_MODE_IRQLINE;
936 lldev->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
937 (RF_SHAREABLE | RF_ACTIVE));
939 if(lldev->irq == NULL) {
940 xge_trace(XGE_ERR, "Allocating irq resource failed");
941 xge_resources_free(dev, xge_free_bar1_resource);
947 device_config->intr_mode = lldev->enabled_msi;
949 xge_trace(XGE_TRACE, "rid: %d, Mode: %d, MSI count: %d", rid,
950 lldev->enabled_msi, msi_count);
953 /* Initialize HAL device */
954 error = xge_hal_device_initialize(hldev, &attr, device_config);
955 if(error != XGE_HAL_OK) {
956 xge_resources_free(dev, xge_free_irq_resource);
957 XGE_EXIT_ON_ERR("Initializing HAL device failed", attach_out,
961 xge_hal_device_private_set(hldev, lldev);
963 error = xge_interface_setup(dev);
969 ifnetp = lldev->ifnetp;
970 ifnetp->if_mtu = device_config->mtu;
974 /* Associate interrupt handler with the device */
975 if(lldev->enabled_msi == XGE_HAL_INTR_MODE_MSI) {
976 error = bus_setup_intr(dev, lldev->irq,
977 (INTR_TYPE_NET | INTR_MPSAFE),
978 #if __FreeBSD_version > 700030
981 xge_isr_msi, lldev, &lldev->irqhandle);
982 xge_msi_info_save(lldev);
985 error = bus_setup_intr(dev, lldev->irq,
986 (INTR_TYPE_NET | INTR_MPSAFE),
987 #if __FreeBSD_version > 700030
990 xge_isr_line, lldev, &lldev->irqhandle);
993 xge_resources_free(dev, xge_free_media_interface);
994 XGE_EXIT_ON_ERR("Associating interrupt handler with device failed",
998 xge_print_info(lldev);
1000 xge_add_sysctl_handlers(lldev);
1002 xge_buffer_mode_init(lldev, device_config->mtu);
1005 xge_os_free(NULL, device_config, sizeof(xge_hal_device_config_t));
1011 * xge_resources_free
1012 * Undo what-all we did during load/attach
1014 * @dev Device Handle
1015 * @error Identifies what-all to undo
1018 xge_resources_free(device_t dev, xge_lables_e error)
1021 xge_pci_info_t *pci_info;
1022 xge_hal_device_t *hldev;
1026 lldev = (xge_lldev_t *) device_get_softc(dev);
1027 pci_info = lldev->pdev;
1030 hldev = lldev->devh;
1034 /* Teardown interrupt handler - device association */
1035 bus_teardown_intr(dev, lldev->irq, lldev->irqhandle);
1037 case xge_free_media_interface:
1039 ifmedia_removeall(&lldev->media);
1042 ether_ifdetach(lldev->ifnetp);
1043 if_free(lldev->ifnetp);
1045 xge_hal_device_private_set(hldev, NULL);
1046 xge_hal_device_disable(hldev);
1048 case xge_free_terminate_hal_device:
1050 xge_hal_device_terminate(hldev);
1052 case xge_free_irq_resource:
1053 /* Release IRQ resource */
1054 bus_release_resource(dev, SYS_RES_IRQ,
1055 ((lldev->enabled_msi == XGE_HAL_INTR_MODE_IRQLINE) ? 0:1),
1058 if(lldev->enabled_msi == XGE_HAL_INTR_MODE_MSI) {
1059 status = pci_release_msi(dev);
1063 "pci_release_msi returned %d", status);
1068 case xge_free_bar1_resource:
1069 /* Restore PCI configuration space */
1070 xge_pci_space_restore(dev);
1072 /* Free bar1resource */
1073 xge_os_free(NULL, pci_info->bar1resource,
1074 sizeof(xge_bus_resource_t));
1079 bus_release_resource(dev, SYS_RES_MEMORY, rid,
1082 case xge_free_bar0_resource:
1083 /* Free bar0resource */
1084 xge_os_free(NULL, pci_info->bar0resource,
1085 sizeof(xge_bus_resource_t));
1090 bus_release_resource(dev, SYS_RES_MEMORY, rid,
1093 case xge_free_pci_info:
1094 /* Disable Bus Master */
1095 pci_disable_busmaster(dev);
1097 /* Free pci_info_t */
1099 xge_os_free(NULL, pci_info, sizeof(xge_pci_info_t));
1101 case xge_free_hal_device:
1102 /* Free device configuration struct and HAL device */
1103 xge_os_free(NULL, hldev, sizeof(xge_hal_device_t));
1105 case xge_free_terminate_hal_driver:
1106 /* Terminate HAL driver */
1107 hal_driver_init_count = hal_driver_init_count - 1;
1108 if(!hal_driver_init_count) {
1109 xge_hal_driver_terminate();
1112 case xge_free_mutex:
1113 xge_mutex_destroy(lldev);
1119 * Detaches driver from the Kernel subsystem
1121 * @dev Device Handle
1124 xge_detach(device_t dev)
1126 xge_lldev_t *lldev = (xge_lldev_t *)device_get_softc(dev);
1128 if(lldev->in_detach == 0) {
1129 lldev->in_detach = 1;
1131 xge_resources_free(dev, xge_free_all);
1139 * To shutdown device before system shutdown
1141 * @dev Device Handle
1144 xge_shutdown(device_t dev)
1146 xge_lldev_t *lldev = (xge_lldev_t *) device_get_softc(dev);
1153 * xge_interface_setup
1156 * @dev Device Handle
1158 * Returns 0 on success, ENXIO/ENOMEM on failure
1161 xge_interface_setup(device_t dev)
1163 u8 mcaddr[ETHER_ADDR_LEN];
1164 xge_hal_status_e status;
1165 xge_lldev_t *lldev = (xge_lldev_t *)device_get_softc(dev);
1166 struct ifnet *ifnetp;
1167 xge_hal_device_t *hldev = lldev->devh;
1169 /* Get the MAC address of the device */
1170 status = xge_hal_device_macaddr_get(hldev, 0, &mcaddr);
1171 if(status != XGE_HAL_OK) {
1172 xge_resources_free(dev, xge_free_terminate_hal_device);
1173 XGE_EXIT_ON_ERR("Getting MAC address failed", ifsetup_out, ENXIO);
1176 /* Get interface ifnet structure for this Ether device */
1177 ifnetp = lldev->ifnetp = if_alloc(IFT_ETHER);
1178 if(ifnetp == NULL) {
1179 xge_resources_free(dev, xge_free_terminate_hal_device);
1180 XGE_EXIT_ON_ERR("Allocation ifnet failed", ifsetup_out, ENOMEM);
1183 /* Initialize interface ifnet structure */
1184 if_initname(ifnetp, device_get_name(dev), device_get_unit(dev));
1185 ifnetp->if_mtu = XGE_HAL_DEFAULT_MTU;
1186 ifnetp->if_baudrate = XGE_BAUDRATE;
1187 ifnetp->if_init = xge_init;
1188 ifnetp->if_softc = lldev;
1189 ifnetp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1190 ifnetp->if_ioctl = xge_ioctl;
1191 ifnetp->if_start = xge_send;
1193 /* TODO: Check and assign optimal value */
1194 ifnetp->if_snd.ifq_maxlen = ifqmaxlen;
1196 ifnetp->if_capabilities = IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU |
1198 if(lldev->enabled_tso)
1199 ifnetp->if_capabilities |= IFCAP_TSO4;
1200 if(lldev->enabled_lro)
1201 ifnetp->if_capabilities |= IFCAP_LRO;
1203 ifnetp->if_capenable = ifnetp->if_capabilities;
1205 /* Attach the interface */
1206 ether_ifattach(ifnetp, mcaddr);
1213 * xge_callback_link_up
1214 * Callback for Link-up indication from HAL
1216 * @userdata Per-adapter data
1219 xge_callback_link_up(void *userdata)
1221 xge_lldev_t *lldev = (xge_lldev_t *)userdata;
1222 struct ifnet *ifnetp = lldev->ifnetp;
1224 ifnetp->if_flags &= ~IFF_DRV_OACTIVE;
1225 if_link_state_change(ifnetp, LINK_STATE_UP);
1229 * xge_callback_link_down
1230 * Callback for Link-down indication from HAL
1232 * @userdata Per-adapter data
1235 xge_callback_link_down(void *userdata)
1237 xge_lldev_t *lldev = (xge_lldev_t *)userdata;
1238 struct ifnet *ifnetp = lldev->ifnetp;
1240 ifnetp->if_flags |= IFF_DRV_OACTIVE;
1241 if_link_state_change(ifnetp, LINK_STATE_DOWN);
1245 * xge_callback_crit_err
1246 * Callback for Critical error indication from HAL
1248 * @userdata Per-adapter data
1249 * @type Event type (Enumerated hardware error)
1250 * @serr_data Hardware status
1253 xge_callback_crit_err(void *userdata, xge_hal_event_e type, u64 serr_data)
1255 xge_trace(XGE_ERR, "Critical Error");
1256 xge_reset(userdata);
1260 * xge_callback_event
1261 * Callback from HAL indicating that some event has been queued
1263 * @item Queued event item
1266 xge_callback_event(xge_queue_item_t *item)
1268 xge_lldev_t *lldev = NULL;
1269 xge_hal_device_t *hldev = NULL;
1270 struct ifnet *ifnetp = NULL;
1272 hldev = item->context;
1273 lldev = xge_hal_device_private(hldev);
1274 ifnetp = lldev->ifnetp;
1276 switch((int)item->event_type) {
1277 case XGE_LL_EVENT_TRY_XMIT_AGAIN:
1278 if(lldev->initialized) {
1279 if(xge_hal_channel_dtr_count(lldev->fifo_channel[0]) > 0) {
1280 ifnetp->if_flags &= ~IFF_DRV_OACTIVE;
1283 xge_queue_produce_context(
1284 xge_hal_device_queue(lldev->devh),
1285 XGE_LL_EVENT_TRY_XMIT_AGAIN, lldev->devh);
1290 case XGE_LL_EVENT_DEVICE_RESETTING:
1291 xge_reset(item->context);
1300 * xge_ifmedia_change
1301 * Media change driver callback
1303 * @ifnetp Interface Handle
1305 * Returns 0 if media is Ether else EINVAL
1308 xge_ifmedia_change(struct ifnet *ifnetp)
1310 xge_lldev_t *lldev = ifnetp->if_softc;
1311 struct ifmedia *ifmediap = &lldev->media;
1313 return (IFM_TYPE(ifmediap->ifm_media) != IFM_ETHER) ? EINVAL:0;
1317 * xge_ifmedia_status
1318 * Media status driver callback
1320 * @ifnetp Interface Handle
1321 * @ifmr Interface Media Settings
1324 xge_ifmedia_status(struct ifnet *ifnetp, struct ifmediareq *ifmr)
1326 xge_hal_status_e status;
1328 xge_lldev_t *lldev = ifnetp->if_softc;
1329 xge_hal_device_t *hldev = lldev->devh;
1331 ifmr->ifm_status = IFM_AVALID;
1332 ifmr->ifm_active = IFM_ETHER;
1334 status = xge_hal_mgmt_reg_read(hldev, 0,
1335 xge_offsetof(xge_hal_pci_bar0_t, adapter_status), ®value);
1336 if(status != XGE_HAL_OK) {
1337 xge_trace(XGE_TRACE, "Getting adapter status failed");
1341 if((regvalue & (XGE_HAL_ADAPTER_STATUS_RMAC_REMOTE_FAULT |
1342 XGE_HAL_ADAPTER_STATUS_RMAC_LOCAL_FAULT)) == 0) {
1343 ifmr->ifm_status |= IFM_ACTIVE;
1344 ifmr->ifm_active |= IFM_10G_SR | IFM_FDX;
1345 if_link_state_change(ifnetp, LINK_STATE_UP);
1348 if_link_state_change(ifnetp, LINK_STATE_DOWN);
1356 * IOCTL to get statistics
1358 * @lldev Per-adapter data
1359 * @ifreqp Interface request
1362 xge_ioctl_stats(xge_lldev_t *lldev, struct ifreq *ifreqp)
1364 xge_hal_status_e status = XGE_HAL_OK;
1365 char *data = (char *)ifreqp->ifr_data;
1367 int retValue = EINVAL;
1370 case XGE_QUERY_STATS:
1371 mtx_lock(&lldev->mtx_drv);
1372 status = xge_hal_stats_hw(lldev->devh,
1373 (xge_hal_stats_hw_info_t **)&info);
1374 mtx_unlock(&lldev->mtx_drv);
1375 if(status == XGE_HAL_OK) {
1376 if(copyout(info, ifreqp->ifr_data,
1377 sizeof(xge_hal_stats_hw_info_t)) == 0)
1381 xge_trace(XGE_ERR, "Getting statistics failed (Status: %d)",
1386 case XGE_QUERY_PCICONF:
1387 info = xge_os_malloc(NULL, sizeof(xge_hal_pci_config_t));
1389 mtx_lock(&lldev->mtx_drv);
1390 status = xge_hal_mgmt_pci_config(lldev->devh, info,
1391 sizeof(xge_hal_pci_config_t));
1392 mtx_unlock(&lldev->mtx_drv);
1393 if(status == XGE_HAL_OK) {
1394 if(copyout(info, ifreqp->ifr_data,
1395 sizeof(xge_hal_pci_config_t)) == 0)
1400 "Getting PCI configuration failed (%d)", status);
1402 xge_os_free(NULL, info, sizeof(xge_hal_pci_config_t));
1406 case XGE_QUERY_DEVSTATS:
1407 info = xge_os_malloc(NULL, sizeof(xge_hal_stats_device_info_t));
1409 mtx_lock(&lldev->mtx_drv);
1410 status =xge_hal_mgmt_device_stats(lldev->devh, info,
1411 sizeof(xge_hal_stats_device_info_t));
1412 mtx_unlock(&lldev->mtx_drv);
1413 if(status == XGE_HAL_OK) {
1414 if(copyout(info, ifreqp->ifr_data,
1415 sizeof(xge_hal_stats_device_info_t)) == 0)
1419 xge_trace(XGE_ERR, "Getting device info failed (%d)",
1422 xge_os_free(NULL, info,
1423 sizeof(xge_hal_stats_device_info_t));
1427 case XGE_QUERY_SWSTATS:
1428 info = xge_os_malloc(NULL, sizeof(xge_hal_stats_sw_err_t));
1430 mtx_lock(&lldev->mtx_drv);
1431 status =xge_hal_mgmt_sw_stats(lldev->devh, info,
1432 sizeof(xge_hal_stats_sw_err_t));
1433 mtx_unlock(&lldev->mtx_drv);
1434 if(status == XGE_HAL_OK) {
1435 if(copyout(info, ifreqp->ifr_data,
1436 sizeof(xge_hal_stats_sw_err_t)) == 0)
1441 "Getting tcode statistics failed (%d)", status);
1443 xge_os_free(NULL, info, sizeof(xge_hal_stats_sw_err_t));
1447 case XGE_QUERY_DRIVERSTATS:
1448 if(copyout(&lldev->driver_stats, ifreqp->ifr_data,
1449 sizeof(xge_driver_stats_t)) == 0) {
1454 "Copyout of driver statistics failed (%d)", status);
1458 case XGE_READ_VERSION:
1459 info = xge_os_malloc(NULL, XGE_BUFFER_SIZE);
1461 strcpy(info, XGE_DRIVER_VERSION);
1462 if(copyout(info, ifreqp->ifr_data, XGE_BUFFER_SIZE) == 0)
1464 xge_os_free(NULL, info, XGE_BUFFER_SIZE);
1468 case XGE_QUERY_DEVCONF:
1469 info = xge_os_malloc(NULL, sizeof(xge_hal_device_config_t));
1471 mtx_lock(&lldev->mtx_drv);
1472 status = xge_hal_mgmt_device_config(lldev->devh, info,
1473 sizeof(xge_hal_device_config_t));
1474 mtx_unlock(&lldev->mtx_drv);
1475 if(status == XGE_HAL_OK) {
1476 if(copyout(info, ifreqp->ifr_data,
1477 sizeof(xge_hal_device_config_t)) == 0)
1481 xge_trace(XGE_ERR, "Getting devconfig failed (%d)",
1484 xge_os_free(NULL, info, sizeof(xge_hal_device_config_t));
1488 case XGE_QUERY_BUFFER_MODE:
1489 if(copyout(&lldev->buffer_mode, ifreqp->ifr_data,
1494 case XGE_SET_BUFFER_MODE_1:
1495 case XGE_SET_BUFFER_MODE_2:
1496 case XGE_SET_BUFFER_MODE_5:
1497 *data = (*data == XGE_SET_BUFFER_MODE_1) ? 'Y':'N';
1498 if(copyout(data, ifreqp->ifr_data, sizeof(data)) == 0)
1502 xge_trace(XGE_TRACE, "Nothing is matching");
1510 * xge_ioctl_registers
1511 * IOCTL to get registers
1513 * @lldev Per-adapter data
1514 * @ifreqp Interface request
1517 xge_ioctl_registers(xge_lldev_t *lldev, struct ifreq *ifreqp)
1519 xge_register_t *data = (xge_register_t *)ifreqp->ifr_data;
1520 xge_hal_status_e status = XGE_HAL_OK;
1521 int retValue = EINVAL, offset = 0, index = 0;
1524 /* Reading a register */
1525 if(strcmp(data->option, "-r") == 0) {
1526 data->value = 0x0000;
1527 mtx_lock(&lldev->mtx_drv);
1528 status = xge_hal_mgmt_reg_read(lldev->devh, 0, data->offset,
1530 mtx_unlock(&lldev->mtx_drv);
1531 if(status == XGE_HAL_OK) {
1532 if(copyout(data, ifreqp->ifr_data, sizeof(xge_register_t)) == 0)
1536 /* Writing to a register */
1537 else if(strcmp(data->option, "-w") == 0) {
1538 mtx_lock(&lldev->mtx_drv);
1539 status = xge_hal_mgmt_reg_write(lldev->devh, 0, data->offset,
1541 if(status == XGE_HAL_OK) {
1543 status = xge_hal_mgmt_reg_read(lldev->devh, 0, data->offset,
1545 if(status != XGE_HAL_OK) {
1546 xge_trace(XGE_ERR, "Reading back updated register failed");
1549 if(val64 != data->value) {
1551 "Read and written register values mismatched");
1557 xge_trace(XGE_ERR, "Getting register value failed");
1559 mtx_unlock(&lldev->mtx_drv);
1562 mtx_lock(&lldev->mtx_drv);
1563 for(index = 0, offset = 0; offset <= XGE_OFFSET_OF_LAST_REG;
1564 index++, offset += 0x0008) {
1566 status = xge_hal_mgmt_reg_read(lldev->devh, 0, offset, &val64);
1567 if(status != XGE_HAL_OK) {
1568 xge_trace(XGE_ERR, "Getting register value failed");
1571 *((u64 *)((u64 *)data + index)) = val64;
1574 mtx_unlock(&lldev->mtx_drv);
1577 if(copyout(data, ifreqp->ifr_data,
1578 sizeof(xge_hal_pci_bar0_t)) != 0) {
1579 xge_trace(XGE_ERR, "Copyout of register values failed");
1584 xge_trace(XGE_ERR, "Getting register values failed");
1592 * Callback to control the device - Interface configuration
1594 * @ifnetp Interface Handle
1595 * @command Device control command
1596 * @data Parameters associated with command (if any)
1599 xge_ioctl(struct ifnet *ifnetp, unsigned long command, caddr_t data)
1601 struct ifreq *ifreqp = (struct ifreq *)data;
1602 xge_lldev_t *lldev = ifnetp->if_softc;
1603 struct ifmedia *ifmediap = &lldev->media;
1604 int retValue = 0, mask = 0;
1606 if(lldev->in_detach) {
1611 /* Set/Get ifnet address */
1614 ether_ioctl(ifnetp, command, data);
1619 retValue = xge_change_mtu(lldev, ifreqp->ifr_mtu);
1622 /* Set ifnet flags */
1624 if(ifnetp->if_flags & IFF_UP) {
1625 /* Link status is UP */
1626 if(!(ifnetp->if_drv_flags & IFF_DRV_RUNNING)) {
1629 xge_disable_promisc(lldev);
1630 xge_enable_promisc(lldev);
1633 /* Link status is DOWN */
1634 /* If device is in running, make it down */
1635 if(ifnetp->if_drv_flags & IFF_DRV_RUNNING) {
1641 /* Add/delete multicast address */
1644 if(ifnetp->if_drv_flags & IFF_DRV_RUNNING) {
1645 xge_setmulti(lldev);
1649 /* Set/Get net media */
1652 retValue = ifmedia_ioctl(ifnetp, ifreqp, ifmediap, command);
1655 /* Set capabilities */
1657 mtx_lock(&lldev->mtx_drv);
1658 mask = ifreqp->ifr_reqcap ^ ifnetp->if_capenable;
1659 if(mask & IFCAP_TXCSUM) {
1660 if(ifnetp->if_capenable & IFCAP_TXCSUM) {
1661 ifnetp->if_capenable &= ~(IFCAP_TSO4 | IFCAP_TXCSUM);
1662 ifnetp->if_hwassist &=
1663 ~(CSUM_TCP | CSUM_UDP | CSUM_TSO);
1666 ifnetp->if_capenable |= IFCAP_TXCSUM;
1667 ifnetp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1670 if(mask & IFCAP_TSO4) {
1671 if(ifnetp->if_capenable & IFCAP_TSO4) {
1672 ifnetp->if_capenable &= ~IFCAP_TSO4;
1673 ifnetp->if_hwassist &= ~CSUM_TSO;
1675 xge_os_printf("%s: TSO Disabled",
1676 device_get_nameunit(lldev->device));
1678 else if(ifnetp->if_capenable & IFCAP_TXCSUM) {
1679 ifnetp->if_capenable |= IFCAP_TSO4;
1680 ifnetp->if_hwassist |= CSUM_TSO;
1682 xge_os_printf("%s: TSO Enabled",
1683 device_get_nameunit(lldev->device));
1687 mtx_unlock(&lldev->mtx_drv);
1690 /* Custom IOCTL 0 */
1691 case SIOCGPRIVATE_0:
1692 retValue = xge_ioctl_stats(lldev, ifreqp);
1695 /* Custom IOCTL 1 */
1696 case SIOCGPRIVATE_1:
1697 retValue = xge_ioctl_registers(lldev, ifreqp);
1709 * Initialize the interface
1711 * @plldev Per-adapter Data
1714 xge_init(void *plldev)
1716 xge_lldev_t *lldev = (xge_lldev_t *)plldev;
1718 mtx_lock(&lldev->mtx_drv);
1719 xge_os_memzero(&lldev->driver_stats, sizeof(xge_driver_stats_t));
1720 xge_device_init(lldev, XGE_HAL_CHANNEL_OC_NORMAL);
1721 mtx_unlock(&lldev->mtx_drv);
1726 * Initialize the interface (called by holding lock)
1728 * @pdevin Per-adapter Data
1731 xge_device_init(xge_lldev_t *lldev, xge_hal_channel_reopen_e option)
1733 struct ifnet *ifnetp = lldev->ifnetp;
1734 xge_hal_device_t *hldev = lldev->devh;
1735 struct ifaddr *ifaddrp;
1736 unsigned char *macaddr;
1737 struct sockaddr_dl *sockaddrp;
1738 int status = XGE_HAL_OK;
1740 mtx_assert((&lldev->mtx_drv), MA_OWNED);
1742 /* If device is in running state, initializing is not required */
1743 if(ifnetp->if_drv_flags & IFF_DRV_RUNNING)
1746 /* Initializing timer */
1747 callout_init(&lldev->timer, 1);
1749 xge_trace(XGE_TRACE, "Set MTU size");
1750 status = xge_hal_device_mtu_set(hldev, ifnetp->if_mtu);
1751 if(status != XGE_HAL_OK) {
1752 xge_trace(XGE_ERR, "Setting MTU in HAL device failed");
1756 /* Enable HAL device */
1757 xge_hal_device_enable(hldev);
1759 /* Get MAC address and update in HAL */
1760 ifaddrp = ifnetp->if_addr;
1761 sockaddrp = (struct sockaddr_dl *)ifaddrp->ifa_addr;
1762 sockaddrp->sdl_type = IFT_ETHER;
1763 sockaddrp->sdl_alen = ifnetp->if_addrlen;
1764 macaddr = LLADDR(sockaddrp);
1765 xge_trace(XGE_TRACE,
1766 "Setting MAC address: %02x:%02x:%02x:%02x:%02x:%02x\n",
1767 *macaddr, *(macaddr + 1), *(macaddr + 2), *(macaddr + 3),
1768 *(macaddr + 4), *(macaddr + 5));
1769 status = xge_hal_device_macaddr_set(hldev, 0, macaddr);
1770 if(status != XGE_HAL_OK)
1771 xge_trace(XGE_ERR, "Setting MAC address failed (%d)", status);
1773 /* Opening channels */
1774 mtx_unlock(&lldev->mtx_drv);
1775 status = xge_channel_open(lldev, option);
1776 mtx_lock(&lldev->mtx_drv);
1777 if(status != XGE_HAL_OK)
1780 /* Set appropriate flags */
1781 ifnetp->if_drv_flags |= IFF_DRV_RUNNING;
1782 ifnetp->if_flags &= ~IFF_DRV_OACTIVE;
1784 /* Checksum capability */
1785 ifnetp->if_hwassist = (ifnetp->if_capenable & IFCAP_TXCSUM) ?
1786 (CSUM_TCP | CSUM_UDP) : 0;
1788 if((lldev->enabled_tso) && (ifnetp->if_capenable & IFCAP_TSO4))
1789 ifnetp->if_hwassist |= CSUM_TSO;
1791 /* Enable interrupts */
1792 xge_hal_device_intr_enable(hldev);
1794 callout_reset(&lldev->timer, 10*hz, xge_timer, lldev);
1796 /* Disable promiscuous mode */
1797 xge_trace(XGE_TRACE, "If opted, enable promiscuous mode");
1798 xge_enable_promisc(lldev);
1800 /* Device is initialized */
1801 lldev->initialized = 1;
1802 xge_os_mdelay(1000);
1810 * Timer timeout function to handle link status
1812 * @devp Per-adapter Data
1815 xge_timer(void *devp)
1817 xge_lldev_t *lldev = (xge_lldev_t *)devp;
1818 xge_hal_device_t *hldev = lldev->devh;
1820 /* Poll for changes */
1821 xge_hal_device_poll(hldev);
1824 callout_reset(&lldev->timer, hz, xge_timer, lldev);
1831 * De-activate the interface
1833 * @lldev Per-adater Data
1836 xge_stop(xge_lldev_t *lldev)
1838 mtx_lock(&lldev->mtx_drv);
1839 xge_device_stop(lldev, XGE_HAL_CHANNEL_OC_NORMAL);
1840 mtx_unlock(&lldev->mtx_drv);
1845 * ISR filter function - to filter interrupts from other devices (shared)
1847 * @handle Per-adapter Data
1850 * FILTER_STRAY if interrupt is from other device
1851 * FILTER_SCHEDULE_THREAD if interrupt is from Xframe device
1854 xge_isr_filter(void *handle)
1856 xge_lldev_t *lldev = (xge_lldev_t *)handle;
1857 xge_hal_pci_bar0_t *bar0 = (xge_hal_pci_bar0_t *)((lldev->devh)->bar0);
1858 u16 retValue = FILTER_STRAY;
1861 XGE_DRV_STATS(isr_filter);
1863 val64 = xge_os_pio_mem_read64(lldev->pdev, (lldev->devh)->regh0,
1864 &bar0->general_int_status);
1865 retValue = (!val64) ? FILTER_STRAY : FILTER_SCHEDULE_THREAD;
1872 * Interrupt service routine for Line interrupts
1874 * @plldev Per-adapter Data
1877 xge_isr_line(void *plldev)
1879 xge_hal_status_e status;
1880 xge_lldev_t *lldev = (xge_lldev_t *)plldev;
1881 xge_hal_device_t *hldev = (xge_hal_device_t *)lldev->devh;
1882 struct ifnet *ifnetp = lldev->ifnetp;
1884 XGE_DRV_STATS(isr_line);
1886 if(ifnetp->if_drv_flags & IFF_DRV_RUNNING) {
1887 status = xge_hal_device_handle_irq(hldev);
1888 if(!(IFQ_DRV_IS_EMPTY(&ifnetp->if_snd)))
1895 * ISR for Message signaled interrupts
1898 xge_isr_msi(void *plldev)
1900 xge_lldev_t *lldev = (xge_lldev_t *)plldev;
1901 XGE_DRV_STATS(isr_msi);
1902 xge_hal_device_continue_irq(lldev->devh);
1907 * Initiate and open all Rx channels
1910 * @lldev Per-adapter Data
1911 * @rflag Channel open/close/reopen flag
1913 * Returns 0 or Error Number
1916 xge_rx_open(int qid, xge_lldev_t *lldev, xge_hal_channel_reopen_e rflag)
1918 u64 adapter_status = 0x0;
1919 xge_hal_status_e status = XGE_HAL_FAIL;
1921 xge_hal_channel_attr_t attr = {
1924 .callback = xge_rx_compl,
1925 .per_dtr_space = sizeof(xge_rx_priv_t),
1927 .type = XGE_HAL_CHANNEL_TYPE_RING,
1929 .dtr_init = xge_rx_initial_replenish,
1930 .dtr_term = xge_rx_term
1933 /* If device is not ready, return */
1934 status = xge_hal_device_status(lldev->devh, &adapter_status);
1935 if(status != XGE_HAL_OK) {
1936 xge_os_printf("Adapter Status: 0x%llx", (long long) adapter_status);
1937 XGE_EXIT_ON_ERR("Device is not ready", _exit, XGE_HAL_FAIL);
1940 status = xge_hal_channel_open(lldev->devh, &attr,
1941 &lldev->ring_channel[qid], rflag);
1950 * Initialize and open all Tx channels
1952 * @lldev Per-adapter Data
1953 * @tflag Channel open/close/reopen flag
1955 * Returns 0 or Error Number
1958 xge_tx_open(xge_lldev_t *lldev, xge_hal_channel_reopen_e tflag)
1960 xge_hal_status_e status = XGE_HAL_FAIL;
1961 u64 adapter_status = 0x0;
1964 xge_hal_channel_attr_t attr = {
1966 .callback = xge_tx_compl,
1967 .per_dtr_space = sizeof(xge_tx_priv_t),
1969 .type = XGE_HAL_CHANNEL_TYPE_FIFO,
1971 .dtr_init = xge_tx_initial_replenish,
1972 .dtr_term = xge_tx_term
1975 /* If device is not ready, return */
1976 status = xge_hal_device_status(lldev->devh, &adapter_status);
1977 if(status != XGE_HAL_OK) {
1978 xge_os_printf("Adapter Status: 0x%llx", (long long) adapter_status);
1979 XGE_EXIT_ON_ERR("Device is not ready", _exit, XGE_HAL_FAIL);
1982 for(qindex = 0; qindex < XGE_FIFO_COUNT; qindex++) {
1983 attr.post_qid = qindex,
1984 status = xge_hal_channel_open(lldev->devh, &attr,
1985 &lldev->fifo_channel[qindex], tflag);
1986 if(status != XGE_HAL_OK) {
1987 for(index = 0; index < qindex; index++)
1988 xge_hal_channel_close(lldev->fifo_channel[index], tflag);
2000 * @lldev Per-adapter Data
2003 xge_enable_msi(xge_lldev_t *lldev)
2005 xge_list_t *item = NULL;
2006 xge_hal_device_t *hldev = lldev->devh;
2007 xge_hal_channel_t *channel = NULL;
2008 u16 offset = 0, val16 = 0;
2010 xge_os_pci_read16(lldev->pdev, NULL,
2011 xge_offsetof(xge_hal_pci_config_le_t, msi_control), &val16);
2013 /* Update msi_data */
2014 offset = (val16 & 0x80) ? 0x4c : 0x48;
2015 xge_os_pci_read16(lldev->pdev, NULL, offset, &val16);
2020 xge_os_pci_write16(lldev->pdev, NULL, offset, val16);
2022 /* Update msi_control */
2023 xge_os_pci_read16(lldev->pdev, NULL,
2024 xge_offsetof(xge_hal_pci_config_le_t, msi_control), &val16);
2026 xge_os_pci_write16(lldev->pdev, NULL,
2027 xge_offsetof(xge_hal_pci_config_le_t, msi_control), val16);
2029 /* Set TxMAT and RxMAT registers with MSI */
2030 xge_list_for_each(item, &hldev->free_channels) {
2031 channel = xge_container_of(item, xge_hal_channel_t, item);
2032 xge_hal_channel_msi_set(channel, 1, (u32)val16);
2038 * Open both Tx and Rx channels
2040 * @lldev Per-adapter Data
2041 * @option Channel reopen option
2044 xge_channel_open(xge_lldev_t *lldev, xge_hal_channel_reopen_e option)
2046 xge_lro_entry_t *lro_session = NULL;
2047 xge_hal_status_e status = XGE_HAL_OK;
2048 int index = 0, index2 = 0;
2050 if(lldev->enabled_msi == XGE_HAL_INTR_MODE_MSI) {
2051 xge_msi_info_restore(lldev);
2052 xge_enable_msi(lldev);
2056 status = xge_create_dma_tags(lldev->device);
2057 if(status != XGE_HAL_OK)
2058 XGE_EXIT_ON_ERR("DMA tag creation failed", _exit, status);
2060 /* Open ring (Rx) channel */
2061 for(index = 0; index < XGE_RING_COUNT; index++) {
2062 status = xge_rx_open(index, lldev, option);
2063 if(status != XGE_HAL_OK) {
2065 * DMA mapping fails in the unpatched Kernel which can't
2066 * allocate contiguous memory for Jumbo frames.
2067 * Try using 5 buffer mode.
2069 if((lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_1) &&
2070 (((lldev->ifnetp)->if_mtu + XGE_HAL_MAC_HEADER_MAX_SIZE) >
2072 /* Close so far opened channels */
2073 for(index2 = 0; index2 < index; index2++) {
2074 xge_hal_channel_close(lldev->ring_channel[index2],
2078 /* Destroy DMA tags intended to use for 1 buffer mode */
2079 if(bus_dmamap_destroy(lldev->dma_tag_rx,
2080 lldev->extra_dma_map)) {
2081 xge_trace(XGE_ERR, "Rx extra DMA map destroy failed");
2083 if(bus_dma_tag_destroy(lldev->dma_tag_rx))
2084 xge_trace(XGE_ERR, "Rx DMA tag destroy failed");
2085 if(bus_dma_tag_destroy(lldev->dma_tag_tx))
2086 xge_trace(XGE_ERR, "Tx DMA tag destroy failed");
2088 /* Switch to 5 buffer mode */
2089 lldev->buffer_mode = XGE_HAL_RING_QUEUE_BUFFER_MODE_5;
2090 xge_buffer_mode_init(lldev, (lldev->ifnetp)->if_mtu);
2096 XGE_EXIT_ON_ERR("Opening Rx channel failed", _exit1,
2102 if(lldev->enabled_lro) {
2103 SLIST_INIT(&lldev->lro_free);
2104 SLIST_INIT(&lldev->lro_active);
2105 lldev->lro_num = XGE_LRO_DEFAULT_ENTRIES;
2107 for(index = 0; index < lldev->lro_num; index++) {
2108 lro_session = (xge_lro_entry_t *)
2109 xge_os_malloc(NULL, sizeof(xge_lro_entry_t));
2110 if(lro_session == NULL) {
2111 lldev->lro_num = index;
2114 SLIST_INSERT_HEAD(&lldev->lro_free, lro_session, next);
2118 /* Open FIFO (Tx) channel */
2119 status = xge_tx_open(lldev, option);
2120 if(status != XGE_HAL_OK)
2121 XGE_EXIT_ON_ERR("Opening Tx channel failed", _exit1, status);
2127 * Opening Rx channel(s) failed (index is <last ring index - 1>) or
2128 * Initialization of LRO failed (index is XGE_RING_COUNT)
2129 * Opening Tx channel failed (index is XGE_RING_COUNT)
2131 for(index2 = 0; index2 < index; index2++)
2132 xge_hal_channel_close(lldev->ring_channel[index2], option);
2140 * Close both Tx and Rx channels
2142 * @lldev Per-adapter Data
2143 * @option Channel reopen option
2147 xge_channel_close(xge_lldev_t *lldev, xge_hal_channel_reopen_e option)
2153 /* Close FIFO (Tx) channel */
2154 for(qindex = 0; qindex < XGE_FIFO_COUNT; qindex++)
2155 xge_hal_channel_close(lldev->fifo_channel[qindex], option);
2157 /* Close Ring (Rx) channels */
2158 for(qindex = 0; qindex < XGE_RING_COUNT; qindex++)
2159 xge_hal_channel_close(lldev->ring_channel[qindex], option);
2161 if(bus_dmamap_destroy(lldev->dma_tag_rx, lldev->extra_dma_map))
2162 xge_trace(XGE_ERR, "Rx extra map destroy failed");
2163 if(bus_dma_tag_destroy(lldev->dma_tag_rx))
2164 xge_trace(XGE_ERR, "Rx DMA tag destroy failed");
2165 if(bus_dma_tag_destroy(lldev->dma_tag_tx))
2166 xge_trace(XGE_ERR, "Tx DMA tag destroy failed");
2173 * @arg Parameter passed from dmamap
2175 * @nseg Number of segments
2179 dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
2182 *(bus_addr_t *) arg = segs->ds_addr;
2190 * @lldev Per-adapter Data
2193 xge_reset(xge_lldev_t *lldev)
2195 xge_trace(XGE_TRACE, "Reseting the chip");
2197 /* If the device is not initialized, return */
2198 if(lldev->initialized) {
2199 mtx_lock(&lldev->mtx_drv);
2200 xge_device_stop(lldev, XGE_HAL_CHANNEL_OC_NORMAL);
2201 xge_device_init(lldev, XGE_HAL_CHANNEL_OC_NORMAL);
2202 mtx_unlock(&lldev->mtx_drv);
2210 * Set an address as a multicast address
2212 * @lldev Per-adapter Data
2215 xge_setmulti(xge_lldev_t *lldev)
2217 struct ifmultiaddr *ifma;
2219 xge_hal_device_t *hldev = (xge_hal_device_t *)lldev->devh;
2220 struct ifnet *ifnetp = lldev->ifnetp;
2223 int table_size = 47;
2224 xge_hal_status_e status = XGE_HAL_OK;
2225 u8 initial_addr[]= {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
2227 if((ifnetp->if_flags & IFF_MULTICAST) && (!lldev->all_multicast)) {
2228 status = xge_hal_device_mcast_enable(hldev);
2229 lldev->all_multicast = 1;
2231 else if((ifnetp->if_flags & IFF_MULTICAST) && (lldev->all_multicast)) {
2232 status = xge_hal_device_mcast_disable(hldev);
2233 lldev->all_multicast = 0;
2236 if(status != XGE_HAL_OK) {
2237 xge_trace(XGE_ERR, "Enabling/disabling multicast failed");
2241 /* Updating address list */
2242 if_maddr_rlock(ifnetp);
2244 TAILQ_FOREACH(ifma, &ifnetp->if_multiaddrs, ifma_link) {
2245 if(ifma->ifma_addr->sa_family != AF_LINK) {
2248 lladdr = LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
2251 if_maddr_runlock(ifnetp);
2253 if((!lldev->all_multicast) && (index)) {
2254 lldev->macaddr_count = (index + 1);
2255 if(lldev->macaddr_count > table_size) {
2259 /* Clear old addresses */
2260 for(index = 0; index < 48; index++) {
2261 xge_hal_device_macaddr_set(hldev, (offset + index),
2266 /* Add new addresses */
2267 if_maddr_rlock(ifnetp);
2269 TAILQ_FOREACH(ifma, &ifnetp->if_multiaddrs, ifma_link) {
2270 if(ifma->ifma_addr->sa_family != AF_LINK) {
2273 lladdr = LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
2274 xge_hal_device_macaddr_set(hldev, (offset + index), lladdr);
2277 if_maddr_runlock(ifnetp);
2284 * xge_enable_promisc
2285 * Enable Promiscuous Mode
2287 * @lldev Per-adapter Data
2290 xge_enable_promisc(xge_lldev_t *lldev)
2292 struct ifnet *ifnetp = lldev->ifnetp;
2293 xge_hal_device_t *hldev = lldev->devh;
2294 xge_hal_pci_bar0_t *bar0 = NULL;
2297 bar0 = (xge_hal_pci_bar0_t *) hldev->bar0;
2299 if(ifnetp->if_flags & IFF_PROMISC) {
2300 xge_hal_device_promisc_enable(lldev->devh);
2303 * When operating in promiscuous mode, don't strip the VLAN tag
2305 val64 = xge_os_pio_mem_read64(lldev->pdev, hldev->regh0,
2307 val64 &= ~XGE_HAL_RX_PA_CFG_STRIP_VLAN_TAG_MODE(1);
2308 val64 |= XGE_HAL_RX_PA_CFG_STRIP_VLAN_TAG_MODE(0);
2309 xge_os_pio_mem_write64(lldev->pdev, hldev->regh0, val64,
2312 xge_trace(XGE_TRACE, "Promiscuous mode ON");
2317 * xge_disable_promisc
2318 * Disable Promiscuous Mode
2320 * @lldev Per-adapter Data
2323 xge_disable_promisc(xge_lldev_t *lldev)
2325 xge_hal_device_t *hldev = lldev->devh;
2326 xge_hal_pci_bar0_t *bar0 = NULL;
2329 bar0 = (xge_hal_pci_bar0_t *) hldev->bar0;
2331 xge_hal_device_promisc_disable(lldev->devh);
2334 * Strip VLAN tag when operating in non-promiscuous mode
2336 val64 = xge_os_pio_mem_read64(lldev->pdev, hldev->regh0,
2338 val64 &= ~XGE_HAL_RX_PA_CFG_STRIP_VLAN_TAG_MODE(1);
2339 val64 |= XGE_HAL_RX_PA_CFG_STRIP_VLAN_TAG_MODE(1);
2340 xge_os_pio_mem_write64(lldev->pdev, hldev->regh0, val64,
2343 xge_trace(XGE_TRACE, "Promiscuous mode OFF");
2348 * Change interface MTU to a requested valid size
2350 * @lldev Per-adapter Data
2351 * @NewMtu Requested MTU
2353 * Returns 0 or Error Number
2356 xge_change_mtu(xge_lldev_t *lldev, int new_mtu)
2358 int status = XGE_HAL_OK;
2360 /* Check requested MTU size for boundary */
2361 if(xge_hal_device_mtu_check(lldev->devh, new_mtu) != XGE_HAL_OK) {
2362 XGE_EXIT_ON_ERR("Invalid MTU", _exit, EINVAL);
2365 lldev->mtu = new_mtu;
2366 xge_confirm_changes(lldev, XGE_SET_MTU);
2375 * Common code for both stop and part of reset. Disables device, interrupts and
2378 * @dev Device Handle
2379 * @option Channel normal/reset option
2382 xge_device_stop(xge_lldev_t *lldev, xge_hal_channel_reopen_e option)
2384 xge_hal_device_t *hldev = lldev->devh;
2385 struct ifnet *ifnetp = lldev->ifnetp;
2388 mtx_assert((&lldev->mtx_drv), MA_OWNED);
2390 /* If device is not in "Running" state, return */
2391 if (!(ifnetp->if_drv_flags & IFF_DRV_RUNNING))
2394 /* Set appropriate flags */
2395 ifnetp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
2398 callout_stop(&lldev->timer);
2400 /* Disable interrupts */
2401 xge_hal_device_intr_disable(hldev);
2403 mtx_unlock(&lldev->mtx_drv);
2404 xge_queue_flush(xge_hal_device_queue(lldev->devh));
2405 mtx_lock(&lldev->mtx_drv);
2407 /* Disable HAL device */
2408 if(xge_hal_device_disable(hldev) != XGE_HAL_OK) {
2409 xge_trace(XGE_ERR, "Disabling HAL device failed");
2410 xge_hal_device_status(hldev, &val64);
2411 xge_trace(XGE_ERR, "Adapter Status: 0x%llx", (long long)val64);
2414 /* Close Tx and Rx channels */
2415 xge_channel_close(lldev, option);
2417 /* Reset HAL device */
2418 xge_hal_device_reset(hldev);
2420 xge_os_mdelay(1000);
2421 lldev->initialized = 0;
2423 if_link_state_change(ifnetp, LINK_STATE_DOWN);
2430 * xge_set_mbuf_cflags
2431 * set checksum flag for the mbuf
2436 xge_set_mbuf_cflags(mbuf_t pkt)
2438 pkt->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
2439 pkt->m_pkthdr.csum_flags |= CSUM_IP_VALID;
2440 pkt->m_pkthdr.csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
2441 pkt->m_pkthdr.csum_data = htons(0xffff);
2445 * xge_lro_flush_sessions
2446 * Flush LRO session and send accumulated LRO packet to upper layer
2448 * @lldev Per-adapter Data
2451 xge_lro_flush_sessions(xge_lldev_t *lldev)
2453 xge_lro_entry_t *lro_session = NULL;
2455 while(!SLIST_EMPTY(&lldev->lro_active)) {
2456 lro_session = SLIST_FIRST(&lldev->lro_active);
2457 SLIST_REMOVE_HEAD(&lldev->lro_active, next);
2458 xge_lro_flush(lldev, lro_session);
2464 * Flush LRO session. Send accumulated LRO packet to upper layer
2466 * @lldev Per-adapter Data
2467 * @lro LRO session to be flushed
2470 xge_lro_flush(xge_lldev_t *lldev, xge_lro_entry_t *lro_session)
2472 struct ip *header_ip;
2473 struct tcphdr *header_tcp;
2476 if(lro_session->append_cnt) {
2477 header_ip = lro_session->lro_header_ip;
2478 header_ip->ip_len = htons(lro_session->len - ETHER_HDR_LEN);
2479 lro_session->m_head->m_pkthdr.len = lro_session->len;
2480 header_tcp = (struct tcphdr *)(header_ip + 1);
2481 header_tcp->th_ack = lro_session->ack_seq;
2482 header_tcp->th_win = lro_session->window;
2483 if(lro_session->timestamp) {
2484 ptr = (u32 *)(header_tcp + 1);
2485 ptr[1] = htonl(lro_session->tsval);
2486 ptr[2] = lro_session->tsecr;
2490 (*lldev->ifnetp->if_input)(lldev->ifnetp, lro_session->m_head);
2491 lro_session->m_head = NULL;
2492 lro_session->timestamp = 0;
2493 lro_session->append_cnt = 0;
2494 SLIST_INSERT_HEAD(&lldev->lro_free, lro_session, next);
2498 * xge_lro_accumulate
2499 * Accumulate packets to form a large LRO packet based on various conditions
2501 * @lldev Per-adapter Data
2502 * @m_head Current Packet
2504 * Returns XGE_HAL_OK or XGE_HAL_FAIL (failure)
2507 xge_lro_accumulate(xge_lldev_t *lldev, struct mbuf *m_head)
2509 struct ether_header *header_ethernet;
2510 struct ip *header_ip;
2511 struct tcphdr *header_tcp;
2513 struct mbuf *buffer_next, *buffer_tail;
2514 xge_lro_entry_t *lro_session;
2515 xge_hal_status_e status = XGE_HAL_FAIL;
2516 int hlen, ip_len, tcp_hdr_len, tcp_data_len, tot_len, tcp_options;
2519 /* Get Ethernet header */
2520 header_ethernet = mtod(m_head, struct ether_header *);
2522 /* Return if it is not IP packet */
2523 if(header_ethernet->ether_type != htons(ETHERTYPE_IP))
2527 header_ip = lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_1 ?
2528 (struct ip *)(header_ethernet + 1) :
2529 mtod(m_head->m_next, struct ip *);
2531 /* Return if it is not TCP packet */
2532 if(header_ip->ip_p != IPPROTO_TCP)
2535 /* Return if packet has options */
2536 if((header_ip->ip_hl << 2) != sizeof(*header_ip))
2539 /* Return if packet is fragmented */
2540 if(header_ip->ip_off & htons(IP_MF | IP_OFFMASK))
2543 /* Get TCP header */
2544 header_tcp = (struct tcphdr *)(header_ip + 1);
2546 /* Return if not ACK or PUSH */
2547 if((header_tcp->th_flags & ~(TH_ACK | TH_PUSH)) != 0)
2550 /* Only timestamp option is handled */
2551 tcp_options = (header_tcp->th_off << 2) - sizeof(*header_tcp);
2552 tcp_hdr_len = sizeof(*header_tcp) + tcp_options;
2553 ptr = (u32 *)(header_tcp + 1);
2554 if(tcp_options != 0) {
2555 if(__predict_false(tcp_options != TCPOLEN_TSTAMP_APPA) ||
2556 (*ptr != ntohl(TCPOPT_NOP << 24 | TCPOPT_NOP << 16 |
2557 TCPOPT_TIMESTAMP << 8 | TCPOLEN_TIMESTAMP))) {
2562 /* Total length of packet (IP) */
2563 ip_len = ntohs(header_ip->ip_len);
2566 tcp_data_len = ip_len - (header_tcp->th_off << 2) - sizeof(*header_ip);
2568 /* If the frame is padded, trim it */
2569 tot_len = m_head->m_pkthdr.len;
2570 trim = tot_len - (ip_len + ETHER_HDR_LEN);
2574 m_adj(m_head, -trim);
2575 tot_len = m_head->m_pkthdr.len;
2578 buffer_next = m_head;
2580 while(buffer_next != NULL) {
2581 buffer_tail = buffer_next;
2582 buffer_next = buffer_tail->m_next;
2585 /* Total size of only headers */
2586 hlen = ip_len + ETHER_HDR_LEN - tcp_data_len;
2588 /* Get sequence number */
2589 seq = ntohl(header_tcp->th_seq);
2591 SLIST_FOREACH(lro_session, &lldev->lro_active, next) {
2592 if(lro_session->source_port == header_tcp->th_sport &&
2593 lro_session->dest_port == header_tcp->th_dport &&
2594 lro_session->source_ip == header_ip->ip_src.s_addr &&
2595 lro_session->dest_ip == header_ip->ip_dst.s_addr) {
2597 /* Unmatched sequence number, flush LRO session */
2598 if(__predict_false(seq != lro_session->next_seq)) {
2599 SLIST_REMOVE(&lldev->lro_active, lro_session,
2600 xge_lro_entry_t, next);
2601 xge_lro_flush(lldev, lro_session);
2605 /* Handle timestamp option */
2607 u32 tsval = ntohl(*(ptr + 1));
2608 if(__predict_false(lro_session->tsval > tsval ||
2612 lro_session->tsval = tsval;
2613 lro_session->tsecr = *(ptr + 2);
2616 lro_session->next_seq += tcp_data_len;
2617 lro_session->ack_seq = header_tcp->th_ack;
2618 lro_session->window = header_tcp->th_win;
2620 /* If TCP data/payload is of 0 size, free mbuf */
2621 if(tcp_data_len == 0) {
2623 status = XGE_HAL_OK;
2627 lro_session->append_cnt++;
2628 lro_session->len += tcp_data_len;
2630 /* Adjust mbuf so that m_data points to payload than headers */
2631 m_adj(m_head, hlen);
2633 /* Append this packet to LRO accumulated packet */
2634 lro_session->m_tail->m_next = m_head;
2635 lro_session->m_tail = buffer_tail;
2637 /* Flush if LRO packet is exceeding maximum size */
2638 if(lro_session->len >
2639 (XGE_HAL_LRO_DEFAULT_FRM_LEN - lldev->ifnetp->if_mtu)) {
2640 SLIST_REMOVE(&lldev->lro_active, lro_session,
2641 xge_lro_entry_t, next);
2642 xge_lro_flush(lldev, lro_session);
2644 status = XGE_HAL_OK;
2649 if(SLIST_EMPTY(&lldev->lro_free))
2652 /* Start a new LRO session */
2653 lro_session = SLIST_FIRST(&lldev->lro_free);
2654 SLIST_REMOVE_HEAD(&lldev->lro_free, next);
2655 SLIST_INSERT_HEAD(&lldev->lro_active, lro_session, next);
2656 lro_session->source_port = header_tcp->th_sport;
2657 lro_session->dest_port = header_tcp->th_dport;
2658 lro_session->source_ip = header_ip->ip_src.s_addr;
2659 lro_session->dest_ip = header_ip->ip_dst.s_addr;
2660 lro_session->next_seq = seq + tcp_data_len;
2661 lro_session->mss = tcp_data_len;
2662 lro_session->ack_seq = header_tcp->th_ack;
2663 lro_session->window = header_tcp->th_win;
2665 lro_session->lro_header_ip = header_ip;
2667 /* Handle timestamp option */
2669 lro_session->timestamp = 1;
2670 lro_session->tsval = ntohl(*(ptr + 1));
2671 lro_session->tsecr = *(ptr + 2);
2674 lro_session->len = tot_len;
2675 lro_session->m_head = m_head;
2676 lro_session->m_tail = buffer_tail;
2677 status = XGE_HAL_OK;
2684 * xge_accumulate_large_rx
2685 * Accumulate packets to form a large LRO packet based on various conditions
2687 * @lldev Per-adapter Data
2688 * @pkt Current packet
2689 * @pkt_length Packet Length
2690 * @rxd_priv Rx Descriptor Private Data
2693 xge_accumulate_large_rx(xge_lldev_t *lldev, struct mbuf *pkt, int pkt_length,
2694 xge_rx_priv_t *rxd_priv)
2696 if(xge_lro_accumulate(lldev, pkt) != XGE_HAL_OK) {
2697 bus_dmamap_sync(lldev->dma_tag_rx, rxd_priv->dmainfo[0].dma_map,
2698 BUS_DMASYNC_POSTREAD);
2699 (*lldev->ifnetp->if_input)(lldev->ifnetp, pkt);
2705 * If the interrupt is due to received frame (Rx completion), send it up
2707 * @channelh Ring Channel Handle
2708 * @dtr Current Descriptor
2709 * @t_code Transfer Code indicating success or error
2710 * @userdata Per-adapter Data
2712 * Returns XGE_HAL_OK or HAL error enums
2715 xge_rx_compl(xge_hal_channel_h channelh, xge_hal_dtr_h dtr, u8 t_code,
2718 struct ifnet *ifnetp;
2719 xge_rx_priv_t *rxd_priv = NULL;
2720 mbuf_t mbuf_up = NULL;
2721 xge_hal_status_e status = XGE_HAL_OK;
2722 xge_hal_dtr_info_t ext_info;
2726 /*get the user data portion*/
2727 xge_lldev_t *lldev = xge_hal_channel_userdata(channelh);
2729 XGE_EXIT_ON_ERR("Failed to get user data", _exit, XGE_HAL_FAIL);
2732 XGE_DRV_STATS(rx_completions);
2734 /* get the interface pointer */
2735 ifnetp = lldev->ifnetp;
2738 XGE_DRV_STATS(rx_desc_compl);
2740 if(!(ifnetp->if_drv_flags & IFF_DRV_RUNNING)) {
2741 status = XGE_HAL_FAIL;
2746 xge_trace(XGE_TRACE, "Packet dropped because of %d", t_code);
2747 XGE_DRV_STATS(rx_tcode);
2748 xge_hal_device_handle_tcode(channelh, dtr, t_code);
2749 xge_hal_ring_dtr_post(channelh,dtr);
2753 /* Get the private data for this descriptor*/
2754 rxd_priv = (xge_rx_priv_t *) xge_hal_ring_dtr_private(channelh,
2757 XGE_EXIT_ON_ERR("Failed to get descriptor private data", _exit,
2762 * Prepare one buffer to send it to upper layer -- since the upper
2763 * layer frees the buffer do not use rxd_priv->buffer. Meanwhile
2764 * prepare a new buffer, do mapping, use it in the current
2765 * descriptor and post descriptor back to ring channel
2767 mbuf_up = rxd_priv->bufferArray[0];
2769 /* Gets details of mbuf i.e., packet length */
2770 xge_ring_dtr_get(mbuf_up, channelh, dtr, lldev, rxd_priv);
2773 (lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_1) ?
2774 xge_get_buf(dtr, rxd_priv, lldev, 0) :
2775 xge_get_buf_3b_5b(dtr, rxd_priv, lldev);
2777 if(status != XGE_HAL_OK) {
2778 xge_trace(XGE_ERR, "No memory");
2779 XGE_DRV_STATS(rx_no_buf);
2782 * Unable to allocate buffer. Instead of discarding, post
2783 * descriptor back to channel for future processing of same
2786 xge_hal_ring_dtr_post(channelh, dtr);
2790 /* Get the extended information */
2791 xge_hal_ring_dtr_info_get(channelh, dtr, &ext_info);
2794 * As we have allocated a new mbuf for this descriptor, post this
2795 * descriptor with new mbuf back to ring channel
2797 vlan_tag = ext_info.vlan;
2798 xge_hal_ring_dtr_post(channelh, dtr);
2799 if ((!(ext_info.proto & XGE_HAL_FRAME_PROTO_IP_FRAGMENTED) &&
2800 (ext_info.proto & XGE_HAL_FRAME_PROTO_TCP_OR_UDP) &&
2801 (ext_info.l3_cksum == XGE_HAL_L3_CKSUM_OK) &&
2802 (ext_info.l4_cksum == XGE_HAL_L4_CKSUM_OK))) {
2804 /* set Checksum Flag */
2805 xge_set_mbuf_cflags(mbuf_up);
2807 if(lldev->enabled_lro) {
2808 xge_accumulate_large_rx(lldev, mbuf_up, mbuf_up->m_len,
2812 /* Post-Read sync for buffers*/
2813 for(index = 0; index < lldev->rxd_mbuf_cnt; index++) {
2814 bus_dmamap_sync(lldev->dma_tag_rx,
2815 rxd_priv->dmainfo[0].dma_map, BUS_DMASYNC_POSTREAD);
2817 (*ifnetp->if_input)(ifnetp, mbuf_up);
2822 * Packet with erroneous checksum , let the upper layer deal
2826 /* Post-Read sync for buffers*/
2827 for(index = 0; index < lldev->rxd_mbuf_cnt; index++) {
2828 bus_dmamap_sync(lldev->dma_tag_rx,
2829 rxd_priv->dmainfo[0].dma_map, BUS_DMASYNC_POSTREAD);
2833 mbuf_up->m_pkthdr.ether_vtag = vlan_tag;
2834 mbuf_up->m_flags |= M_VLANTAG;
2837 if(lldev->enabled_lro)
2838 xge_lro_flush_sessions(lldev);
2840 (*ifnetp->if_input)(ifnetp, mbuf_up);
2842 } while(xge_hal_ring_dtr_next_completed(channelh, &dtr, &t_code)
2845 if(lldev->enabled_lro)
2846 xge_lro_flush_sessions(lldev);
2856 * @mbuf_up Packet to send up
2857 * @channelh Ring Channel Handle
2859 * @lldev Per-adapter Data
2860 * @rxd_priv Rx Descriptor Private Data
2862 * Returns XGE_HAL_OK or HAL error enums
2865 xge_ring_dtr_get(mbuf_t mbuf_up, xge_hal_channel_h channelh, xge_hal_dtr_h dtr,
2866 xge_lldev_t *lldev, xge_rx_priv_t *rxd_priv)
2869 int pkt_length[5]={0,0}, pkt_len=0;
2870 dma_addr_t dma_data[5];
2876 if(lldev->buffer_mode != XGE_HAL_RING_QUEUE_BUFFER_MODE_1) {
2877 xge_os_memzero(pkt_length, sizeof(pkt_length));
2880 * Retrieve data of interest from the completed descriptor -- This
2881 * returns the packet length
2883 if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5) {
2884 xge_hal_ring_dtr_5b_get(channelh, dtr, dma_data, pkt_length);
2887 xge_hal_ring_dtr_3b_get(channelh, dtr, dma_data, pkt_length);
2890 for(index = 0; index < lldev->rxd_mbuf_cnt; index++) {
2891 m->m_len = pkt_length[index];
2893 if(index < (lldev->rxd_mbuf_cnt-1)) {
2894 m->m_next = rxd_priv->bufferArray[index + 1];
2900 pkt_len+=pkt_length[index];
2904 * Since 2 buffer mode is an exceptional case where data is in 3rd
2905 * buffer but not in 2nd buffer
2907 if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_2) {
2908 m->m_len = pkt_length[2];
2909 pkt_len+=pkt_length[2];
2913 * Update length of newly created buffer to be sent up with packet
2916 mbuf_up->m_pkthdr.len = pkt_len;
2920 * Retrieve data of interest from the completed descriptor -- This
2921 * returns the packet length
2923 xge_hal_ring_dtr_1b_get(channelh, dtr,&dma_data[0], &pkt_length[0]);
2926 * Update length of newly created buffer to be sent up with packet
2929 mbuf_up->m_len = mbuf_up->m_pkthdr.len = pkt_length[0];
2937 * Flush Tx descriptors
2939 * @channelh Channel handle
2942 xge_flush_txds(xge_hal_channel_h channelh)
2944 xge_lldev_t *lldev = xge_hal_channel_userdata(channelh);
2945 xge_hal_dtr_h tx_dtr;
2946 xge_tx_priv_t *tx_priv;
2949 while(xge_hal_fifo_dtr_next_completed(channelh, &tx_dtr, &t_code)
2951 XGE_DRV_STATS(tx_desc_compl);
2953 xge_trace(XGE_TRACE, "Tx descriptor with t_code %d", t_code);
2954 XGE_DRV_STATS(tx_tcode);
2955 xge_hal_device_handle_tcode(channelh, tx_dtr, t_code);
2958 tx_priv = xge_hal_fifo_dtr_private(tx_dtr);
2959 bus_dmamap_unload(lldev->dma_tag_tx, tx_priv->dma_map);
2960 m_freem(tx_priv->buffer);
2961 tx_priv->buffer = NULL;
2962 xge_hal_fifo_dtr_free(channelh, tx_dtr);
2970 * @ifnetp Interface Handle
2973 xge_send(struct ifnet *ifnetp)
2976 xge_lldev_t *lldev = ifnetp->if_softc;
2978 for(qindex = 0; qindex < XGE_FIFO_COUNT; qindex++) {
2979 if(mtx_trylock(&lldev->mtx_tx[qindex]) == 0) {
2980 XGE_DRV_STATS(tx_lock_fail);
2983 xge_send_locked(ifnetp, qindex);
2984 mtx_unlock(&lldev->mtx_tx[qindex]);
2989 xge_send_locked(struct ifnet *ifnetp, int qindex)
2992 static bus_dma_segment_t segs[XGE_MAX_SEGS];
2993 xge_hal_status_e status;
2994 unsigned int max_fragments;
2995 xge_lldev_t *lldev = ifnetp->if_softc;
2996 xge_hal_channel_h channelh = lldev->fifo_channel[qindex];
2997 mbuf_t m_head = NULL;
2998 mbuf_t m_buf = NULL;
2999 xge_tx_priv_t *ll_tx_priv = NULL;
3000 register unsigned int count = 0;
3001 unsigned int nsegs = 0;
3004 max_fragments = ((xge_hal_fifo_t *)channelh)->config->max_frags;
3006 /* If device is not initialized, return */
3007 if((!lldev->initialized) || (!(ifnetp->if_drv_flags & IFF_DRV_RUNNING)))
3010 XGE_DRV_STATS(tx_calls);
3013 * This loop will be executed for each packet in the kernel maintained
3014 * queue -- each packet can be with fragments as an mbuf chain
3017 IF_DEQUEUE(&ifnetp->if_snd, m_head);
3018 if (m_head == NULL) {
3019 ifnetp->if_drv_flags &= ~(IFF_DRV_OACTIVE);
3023 for(m_buf = m_head; m_buf != NULL; m_buf = m_buf->m_next) {
3024 if(m_buf->m_len) count += 1;
3027 if(count >= max_fragments) {
3028 m_buf = m_defrag(m_head, M_NOWAIT);
3029 if(m_buf != NULL) m_head = m_buf;
3030 XGE_DRV_STATS(tx_defrag);
3033 /* Reserve descriptors */
3034 status = xge_hal_fifo_dtr_reserve(channelh, &dtr);
3035 if(status != XGE_HAL_OK) {
3036 XGE_DRV_STATS(tx_no_txd);
3037 xge_flush_txds(channelh);
3042 (m_head->m_flags & M_VLANTAG) ? m_head->m_pkthdr.ether_vtag : 0;
3043 xge_hal_fifo_dtr_vlan_set(dtr, vlan_tag);
3045 /* Update Tx private structure for this descriptor */
3046 ll_tx_priv = xge_hal_fifo_dtr_private(dtr);
3047 ll_tx_priv->buffer = m_head;
3050 * Do mapping -- Required DMA tag has been created in xge_init
3051 * function and DMA maps have already been created in the
3052 * xgell_tx_replenish function.
3053 * Returns number of segments through nsegs
3055 if(bus_dmamap_load_mbuf_sg(lldev->dma_tag_tx,
3056 ll_tx_priv->dma_map, m_head, segs, &nsegs, BUS_DMA_NOWAIT)) {
3057 xge_trace(XGE_TRACE, "DMA map load failed");
3058 XGE_DRV_STATS(tx_map_fail);
3062 if(lldev->driver_stats.tx_max_frags < nsegs)
3063 lldev->driver_stats.tx_max_frags = nsegs;
3065 /* Set descriptor buffer for header and each fragment/segment */
3068 xge_hal_fifo_dtr_buffer_set(channelh, dtr, count,
3069 (dma_addr_t)htole64(segs[count].ds_addr),
3070 segs[count].ds_len);
3072 } while(count < nsegs);
3074 /* Pre-write Sync of mapping */
3075 bus_dmamap_sync(lldev->dma_tag_tx, ll_tx_priv->dma_map,
3076 BUS_DMASYNC_PREWRITE);
3078 if((lldev->enabled_tso) &&
3079 (m_head->m_pkthdr.csum_flags & CSUM_TSO)) {
3080 XGE_DRV_STATS(tx_tso);
3081 xge_hal_fifo_dtr_mss_set(dtr, m_head->m_pkthdr.tso_segsz);
3085 if(ifnetp->if_hwassist > 0) {
3086 xge_hal_fifo_dtr_cksum_set_bits(dtr, XGE_HAL_TXD_TX_CKO_IPV4_EN
3087 | XGE_HAL_TXD_TX_CKO_TCP_EN | XGE_HAL_TXD_TX_CKO_UDP_EN);
3090 /* Post descriptor to FIFO channel */
3091 xge_hal_fifo_dtr_post(channelh, dtr);
3092 XGE_DRV_STATS(tx_posted);
3094 /* Send the same copy of mbuf packet to BPF (Berkely Packet Filter)
3095 * listener so that we can use tools like tcpdump */
3096 ETHER_BPF_MTAP(ifnetp, m_head);
3099 /* Prepend the packet back to queue */
3100 IF_PREPEND(&ifnetp->if_snd, m_head);
3101 ifnetp->if_drv_flags |= IFF_DRV_OACTIVE;
3103 xge_queue_produce_context(xge_hal_device_queue(lldev->devh),
3104 XGE_LL_EVENT_TRY_XMIT_AGAIN, lldev->devh);
3105 XGE_DRV_STATS(tx_again);
3110 * Allocates new mbufs to be placed into descriptors
3112 * @dtrh Descriptor Handle
3113 * @rxd_priv Rx Descriptor Private Data
3114 * @lldev Per-adapter Data
3115 * @index Buffer Index (if multi-buffer mode)
3117 * Returns XGE_HAL_OK or HAL error enums
3120 xge_get_buf(xge_hal_dtr_h dtrh, xge_rx_priv_t *rxd_priv,
3121 xge_lldev_t *lldev, int index)
3123 register mbuf_t mp = NULL;
3124 struct ifnet *ifnetp = lldev->ifnetp;
3125 int status = XGE_HAL_OK;
3126 int buffer_size = 0, cluster_size = 0, count;
3127 bus_dmamap_t map = rxd_priv->dmainfo[index].dma_map;
3128 bus_dma_segment_t segs[3];
3130 buffer_size = (lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_1) ?
3131 ifnetp->if_mtu + XGE_HAL_MAC_HEADER_MAX_SIZE :
3132 lldev->rxd_mbuf_len[index];
3134 if(buffer_size <= MCLBYTES) {
3135 cluster_size = MCLBYTES;
3136 mp = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
3139 cluster_size = MJUMPAGESIZE;
3140 if((lldev->buffer_mode != XGE_HAL_RING_QUEUE_BUFFER_MODE_5) &&
3141 (buffer_size > MJUMPAGESIZE)) {
3142 cluster_size = MJUM9BYTES;
3144 mp = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, cluster_size);
3147 xge_trace(XGE_ERR, "Out of memory to allocate mbuf");
3148 status = XGE_HAL_FAIL;
3152 /* Update mbuf's length, packet length and receive interface */
3153 mp->m_len = mp->m_pkthdr.len = buffer_size;
3154 mp->m_pkthdr.rcvif = ifnetp;
3157 if(bus_dmamap_load_mbuf_sg(lldev->dma_tag_rx, lldev->extra_dma_map,
3158 mp, segs, &count, BUS_DMA_NOWAIT)) {
3159 XGE_DRV_STATS(rx_map_fail);
3161 XGE_EXIT_ON_ERR("DMA map load failed", getbuf_out, XGE_HAL_FAIL);
3164 /* Update descriptor private data */
3165 rxd_priv->bufferArray[index] = mp;
3166 rxd_priv->dmainfo[index].dma_phyaddr = htole64(segs->ds_addr);
3167 rxd_priv->dmainfo[index].dma_map = lldev->extra_dma_map;
3168 lldev->extra_dma_map = map;
3170 /* Pre-Read/Write sync */
3171 bus_dmamap_sync(lldev->dma_tag_rx, map, BUS_DMASYNC_POSTREAD);
3173 /* Unload DMA map of mbuf in current descriptor */
3174 bus_dmamap_unload(lldev->dma_tag_rx, map);
3176 /* Set descriptor buffer */
3177 if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_1) {
3178 xge_hal_ring_dtr_1b_set(dtrh, rxd_priv->dmainfo[0].dma_phyaddr,
3188 * Allocates new mbufs to be placed into descriptors (in multi-buffer modes)
3190 * @dtrh Descriptor Handle
3191 * @rxd_priv Rx Descriptor Private Data
3192 * @lldev Per-adapter Data
3194 * Returns XGE_HAL_OK or HAL error enums
3197 xge_get_buf_3b_5b(xge_hal_dtr_h dtrh, xge_rx_priv_t *rxd_priv,
3200 bus_addr_t dma_pointers[5];
3202 int status = XGE_HAL_OK, index;
3205 for(index = 0; index < lldev->rxd_mbuf_cnt; index++) {
3206 status = xge_get_buf(dtrh, rxd_priv, lldev, index);
3207 if(status != XGE_HAL_OK) {
3208 for(newindex = 0; newindex < index; newindex++) {
3209 m_freem(rxd_priv->bufferArray[newindex]);
3211 XGE_EXIT_ON_ERR("mbuf allocation failed", _exit, status);
3215 for(index = 0; index < lldev->buffer_mode; index++) {
3216 if(lldev->rxd_mbuf_len[index] != 0) {
3217 dma_pointers[index] = rxd_priv->dmainfo[index].dma_phyaddr;
3218 dma_sizes[index] = lldev->rxd_mbuf_len[index];
3221 dma_pointers[index] = rxd_priv->dmainfo[index-1].dma_phyaddr;
3222 dma_sizes[index] = 1;
3226 /* Assigning second buffer to third pointer in 2 buffer mode */
3227 if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_2) {
3228 dma_pointers[2] = dma_pointers[1];
3229 dma_sizes[2] = dma_sizes[1];
3233 if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5) {
3234 xge_hal_ring_dtr_5b_set(dtrh, dma_pointers, dma_sizes);
3237 xge_hal_ring_dtr_3b_set(dtrh, dma_pointers, dma_sizes);
3246 * If the interrupt is due to Tx completion, free the sent buffer
3248 * @channelh Channel Handle
3250 * @t_code Transfer Code indicating success or error
3251 * @userdata Per-adapter Data
3253 * Returns XGE_HAL_OK or HAL error enum
3256 xge_tx_compl(xge_hal_channel_h channelh,
3257 xge_hal_dtr_h dtr, u8 t_code, void *userdata)
3259 xge_tx_priv_t *ll_tx_priv = NULL;
3260 xge_lldev_t *lldev = (xge_lldev_t *)userdata;
3261 struct ifnet *ifnetp = lldev->ifnetp;
3262 mbuf_t m_buffer = NULL;
3263 int qindex = xge_hal_channel_id(channelh);
3265 mtx_lock(&lldev->mtx_tx[qindex]);
3267 XGE_DRV_STATS(tx_completions);
3270 * For each completed descriptor: Get private structure, free buffer,
3271 * do unmapping, and free descriptor
3274 XGE_DRV_STATS(tx_desc_compl);
3277 XGE_DRV_STATS(tx_tcode);
3278 xge_trace(XGE_TRACE, "t_code %d", t_code);
3279 xge_hal_device_handle_tcode(channelh, dtr, t_code);
3282 ll_tx_priv = xge_hal_fifo_dtr_private(dtr);
3283 m_buffer = ll_tx_priv->buffer;
3284 bus_dmamap_unload(lldev->dma_tag_tx, ll_tx_priv->dma_map);
3286 ll_tx_priv->buffer = NULL;
3287 xge_hal_fifo_dtr_free(channelh, dtr);
3288 } while(xge_hal_fifo_dtr_next_completed(channelh, &dtr, &t_code)
3290 xge_send_locked(ifnetp, qindex);
3291 ifnetp->if_drv_flags &= ~IFF_DRV_OACTIVE;
3293 mtx_unlock(&lldev->mtx_tx[qindex]);
3299 * xge_tx_initial_replenish
3300 * Initially allocate buffers and set them into descriptors for later use
3302 * @channelh Tx Channel Handle
3303 * @dtrh Descriptor Handle
3305 * @userdata Per-adapter Data
3306 * @reopen Channel open/reopen option
3308 * Returns XGE_HAL_OK or HAL error enums
3311 xge_tx_initial_replenish(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh,
3312 int index, void *userdata, xge_hal_channel_reopen_e reopen)
3314 xge_tx_priv_t *txd_priv = NULL;
3315 int status = XGE_HAL_OK;
3317 /* Get the user data portion from channel handle */
3318 xge_lldev_t *lldev = xge_hal_channel_userdata(channelh);
3320 XGE_EXIT_ON_ERR("Failed to get user data from channel", txinit_out,
3324 /* Get the private data */
3325 txd_priv = (xge_tx_priv_t *) xge_hal_fifo_dtr_private(dtrh);
3326 if(txd_priv == NULL) {
3327 XGE_EXIT_ON_ERR("Failed to get descriptor private data", txinit_out,
3331 /* Create DMA map for this descriptor */
3332 if(bus_dmamap_create(lldev->dma_tag_tx, BUS_DMA_NOWAIT,
3333 &txd_priv->dma_map)) {
3334 XGE_EXIT_ON_ERR("DMA map creation for Tx descriptor failed",
3335 txinit_out, XGE_HAL_FAIL);
3343 * xge_rx_initial_replenish
3344 * Initially allocate buffers and set them into descriptors for later use
3346 * @channelh Tx Channel Handle
3347 * @dtrh Descriptor Handle
3349 * @userdata Per-adapter Data
3350 * @reopen Channel open/reopen option
3352 * Returns XGE_HAL_OK or HAL error enums
3355 xge_rx_initial_replenish(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh,
3356 int index, void *userdata, xge_hal_channel_reopen_e reopen)
3358 xge_rx_priv_t *rxd_priv = NULL;
3359 int status = XGE_HAL_OK;
3360 int index1 = 0, index2 = 0;
3362 /* Get the user data portion from channel handle */
3363 xge_lldev_t *lldev = xge_hal_channel_userdata(channelh);
3365 XGE_EXIT_ON_ERR("Failed to get user data from channel", rxinit_out,
3369 /* Get the private data */
3370 rxd_priv = (xge_rx_priv_t *) xge_hal_ring_dtr_private(channelh, dtrh);
3371 if(rxd_priv == NULL) {
3372 XGE_EXIT_ON_ERR("Failed to get descriptor private data", rxinit_out,
3376 rxd_priv->bufferArray = xge_os_malloc(NULL,
3377 (sizeof(rxd_priv->bufferArray) * lldev->rxd_mbuf_cnt));
3379 if(rxd_priv->bufferArray == NULL) {
3380 XGE_EXIT_ON_ERR("Failed to allocate Rxd private", rxinit_out,
3384 if(lldev->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_1) {
3385 /* Create DMA map for these descriptors*/
3386 if(bus_dmamap_create(lldev->dma_tag_rx , BUS_DMA_NOWAIT,
3387 &rxd_priv->dmainfo[0].dma_map)) {
3388 XGE_EXIT_ON_ERR("DMA map creation for Rx descriptor failed",
3389 rxinit_err_out, XGE_HAL_FAIL);
3391 /* Get a buffer, attach it to this descriptor */
3392 status = xge_get_buf(dtrh, rxd_priv, lldev, 0);
3395 for(index1 = 0; index1 < lldev->rxd_mbuf_cnt; index1++) {
3396 /* Create DMA map for this descriptor */
3397 if(bus_dmamap_create(lldev->dma_tag_rx , BUS_DMA_NOWAIT ,
3398 &rxd_priv->dmainfo[index1].dma_map)) {
3399 for(index2 = index1 - 1; index2 >= 0; index2--) {
3400 bus_dmamap_destroy(lldev->dma_tag_rx,
3401 rxd_priv->dmainfo[index2].dma_map);
3404 "Jumbo DMA map creation for Rx descriptor failed",
3405 rxinit_err_out, XGE_HAL_FAIL);
3408 status = xge_get_buf_3b_5b(dtrh, rxd_priv, lldev);
3411 if(status != XGE_HAL_OK) {
3412 for(index1 = 0; index1 < lldev->rxd_mbuf_cnt; index1++) {
3413 bus_dmamap_destroy(lldev->dma_tag_rx,
3414 rxd_priv->dmainfo[index1].dma_map);
3416 goto rxinit_err_out;
3423 xge_os_free(NULL, rxd_priv->bufferArray,
3424 (sizeof(rxd_priv->bufferArray) * lldev->rxd_mbuf_cnt));
3431 * During unload terminate and free all descriptors
3433 * @channelh Rx Channel Handle
3434 * @dtrh Rx Descriptor Handle
3435 * @state Descriptor State
3436 * @userdata Per-adapter Data
3437 * @reopen Channel open/reopen option
3440 xge_rx_term(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh,
3441 xge_hal_dtr_state_e state, void *userdata,
3442 xge_hal_channel_reopen_e reopen)
3444 xge_rx_priv_t *rxd_priv = NULL;
3445 xge_lldev_t *lldev = NULL;
3448 /* Descriptor state is not "Posted" */
3449 if(state != XGE_HAL_DTR_STATE_POSTED) goto rxterm_out;
3451 /* Get the user data portion */
3452 lldev = xge_hal_channel_userdata(channelh);
3454 /* Get the private data */
3455 rxd_priv = (xge_rx_priv_t *) xge_hal_ring_dtr_private(channelh, dtrh);
3457 for(index = 0; index < lldev->rxd_mbuf_cnt; index++) {
3458 if(rxd_priv->dmainfo[index].dma_map != NULL) {
3459 bus_dmamap_sync(lldev->dma_tag_rx,
3460 rxd_priv->dmainfo[index].dma_map, BUS_DMASYNC_POSTREAD);
3461 bus_dmamap_unload(lldev->dma_tag_rx,
3462 rxd_priv->dmainfo[index].dma_map);
3463 if(rxd_priv->bufferArray[index] != NULL)
3464 m_free(rxd_priv->bufferArray[index]);
3465 bus_dmamap_destroy(lldev->dma_tag_rx,
3466 rxd_priv->dmainfo[index].dma_map);
3469 xge_os_free(NULL, rxd_priv->bufferArray,
3470 (sizeof(rxd_priv->bufferArray) * lldev->rxd_mbuf_cnt));
3472 /* Free the descriptor */
3473 xge_hal_ring_dtr_free(channelh, dtrh);
3481 * During unload terminate and free all descriptors
3483 * @channelh Rx Channel Handle
3484 * @dtrh Rx Descriptor Handle
3485 * @state Descriptor State
3486 * @userdata Per-adapter Data
3487 * @reopen Channel open/reopen option
3490 xge_tx_term(xge_hal_channel_h channelh, xge_hal_dtr_h dtr,
3491 xge_hal_dtr_state_e state, void *userdata,
3492 xge_hal_channel_reopen_e reopen)
3494 xge_tx_priv_t *ll_tx_priv = xge_hal_fifo_dtr_private(dtr);
3495 xge_lldev_t *lldev = (xge_lldev_t *)userdata;
3497 /* Destroy DMA map */
3498 bus_dmamap_destroy(lldev->dma_tag_tx, ll_tx_priv->dma_map);
3504 * FreeBSD device interface entry points
3506 static device_method_t xge_methods[] = {
3507 DEVMETHOD(device_probe, xge_probe),
3508 DEVMETHOD(device_attach, xge_attach),
3509 DEVMETHOD(device_detach, xge_detach),
3510 DEVMETHOD(device_shutdown, xge_shutdown),
3515 static driver_t xge_driver = {
3518 sizeof(xge_lldev_t),
3520 static devclass_t xge_devclass;
3521 DRIVER_MODULE(nxge, pci, xge_driver, xge_devclass, 0, 0);