2 * Copyright (c) 2002-2007 Neterion, Inc.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 #include <dev/nxge/include/xgehal-device.h>
33 #include <dev/nxge/include/xgehal-ring.h>
34 #include <dev/nxge/include/xgehal-fifo.h>
37 * xge_hal_device_bar0 - Get BAR0 mapped address.
38 * @hldev: HAL device handle.
40 * Returns: BAR0 address of the specified device.
42 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE char *
43 xge_hal_device_bar0(xge_hal_device_t *hldev)
49 * xge_hal_device_isrbar0 - Get BAR0 mapped address.
50 * @hldev: HAL device handle.
52 * Returns: BAR0 address of the specified device.
54 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE char *
55 xge_hal_device_isrbar0(xge_hal_device_t *hldev)
57 return hldev->isrbar0;
61 * xge_hal_device_bar1 - Get BAR1 mapped address.
62 * @hldev: HAL device handle.
64 * Returns: BAR1 address of the specified device.
66 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE char *
67 xge_hal_device_bar1(xge_hal_device_t *hldev)
73 * xge_hal_device_bar0_set - Set BAR0 mapped address.
74 * @hldev: HAL device handle.
75 * @bar0: BAR0 mapped address.
76 * * Set BAR0 address in the HAL device object.
78 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
79 xge_hal_device_bar0_set(xge_hal_device_t *hldev, char *bar0)
86 * xge_hal_device_isrbar0_set - Set BAR0 mapped address.
87 * @hldev: HAL device handle.
88 * @isrbar0: BAR0 mapped address.
89 * * Set BAR0 address in the HAL device object.
91 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
92 xge_hal_device_isrbar0_set(xge_hal_device_t *hldev, char *isrbar0)
95 hldev->isrbar0 = isrbar0;
99 * xge_hal_device_bar1_set - Set BAR1 mapped address.
100 * @hldev: HAL device handle.
101 * @channelh: Channel handle.
102 * @bar1: BAR1 mapped address.
104 * Set BAR1 address for the given channel.
106 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
107 xge_hal_device_bar1_set(xge_hal_device_t *hldev, xge_hal_channel_h channelh,
110 xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh;
115 /* Initializing the BAR1 address as the start of
116 * the FIFO queue pointer and as a location of FIFO control
119 (xge_hal_fifo_hw_pair_t *) (bar1 +
120 (fifo->channel.post_qid * XGE_HAL_FIFO_HW_PAIR_OFFSET));
126 * xge_hal_device_rev - Get Device revision number.
127 * @hldev: HAL device handle.
129 * Returns: Device revision number
131 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE int
132 xge_hal_device_rev(xge_hal_device_t *hldev)
134 return hldev->revision;
139 * xge_hal_device_begin_irq - Begin IRQ processing.
140 * @hldev: HAL device handle.
141 * @reason: "Reason" for the interrupt, the value of Xframe's
142 * general_int_status register.
144 * The function performs two actions, It first checks whether (shared IRQ) the
145 * interrupt was raised by the device. Next, it masks the device interrupts.
148 * xge_hal_device_begin_irq() does not flush MMIO writes through the
149 * bridge. Therefore, two back-to-back interrupts are potentially possible.
150 * It is the responsibility of the ULD to make sure that only one
151 * xge_hal_device_continue_irq() runs at a time.
153 * Returns: 0, if the interrupt is not "ours" (note that in this case the
154 * device remain enabled).
155 * Otherwise, xge_hal_device_begin_irq() returns 64bit general adapter
157 * See also: xge_hal_device_handle_irq()
159 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e
160 xge_hal_device_begin_irq(xge_hal_device_t *hldev, u64 *reason)
163 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
165 hldev->stats.sw_dev_info_stats.total_intr_cnt++;
167 val64 = xge_os_pio_mem_read64(hldev->pdev,
168 hldev->regh0, &isrbar0->general_int_status);
169 if (xge_os_unlikely(!val64)) {
170 /* not Xframe interrupt */
171 hldev->stats.sw_dev_info_stats.not_xge_intr_cnt++;
173 return XGE_HAL_ERR_WRONG_IRQ;
176 if (xge_os_unlikely(val64 == XGE_HAL_ALL_FOXES)) {
178 xge_os_pio_mem_read64(hldev->pdev, hldev->regh0,
179 &isrbar0->adapter_status);
180 if (adapter_status == XGE_HAL_ALL_FOXES) {
181 (void) xge_queue_produce(hldev->queueh,
182 XGE_HAL_EVENT_SLOT_FREEZE,
184 1, /* critical: slot freeze */
186 (void*)&adapter_status);
188 return XGE_HAL_ERR_CRITICAL;
194 /* separate fast path, i.e. no errors */
195 if (val64 & XGE_HAL_GEN_INTR_RXTRAFFIC) {
196 hldev->stats.sw_dev_info_stats.rx_traffic_intr_cnt++;
199 if (val64 & XGE_HAL_GEN_INTR_TXTRAFFIC) {
200 hldev->stats.sw_dev_info_stats.tx_traffic_intr_cnt++;
204 hldev->stats.sw_dev_info_stats.not_traffic_intr_cnt++;
205 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_TXPIC)) {
206 xge_hal_status_e status;
207 hldev->stats.sw_dev_info_stats.txpic_intr_cnt++;
208 status = __hal_device_handle_txpic(hldev, val64);
209 if (status != XGE_HAL_OK) {
214 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_TXDMA)) {
215 xge_hal_status_e status;
216 hldev->stats.sw_dev_info_stats.txdma_intr_cnt++;
217 status = __hal_device_handle_txdma(hldev, val64);
218 if (status != XGE_HAL_OK) {
223 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_TXMAC)) {
224 xge_hal_status_e status;
225 hldev->stats.sw_dev_info_stats.txmac_intr_cnt++;
226 status = __hal_device_handle_txmac(hldev, val64);
227 if (status != XGE_HAL_OK) {
232 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_TXXGXS)) {
233 xge_hal_status_e status;
234 hldev->stats.sw_dev_info_stats.txxgxs_intr_cnt++;
235 status = __hal_device_handle_txxgxs(hldev, val64);
236 if (status != XGE_HAL_OK) {
241 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_RXPIC)) {
242 xge_hal_status_e status;
243 hldev->stats.sw_dev_info_stats.rxpic_intr_cnt++;
244 status = __hal_device_handle_rxpic(hldev, val64);
245 if (status != XGE_HAL_OK) {
250 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_RXDMA)) {
251 xge_hal_status_e status;
252 hldev->stats.sw_dev_info_stats.rxdma_intr_cnt++;
253 status = __hal_device_handle_rxdma(hldev, val64);
254 if (status != XGE_HAL_OK) {
259 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_RXMAC)) {
260 xge_hal_status_e status;
261 hldev->stats.sw_dev_info_stats.rxmac_intr_cnt++;
262 status = __hal_device_handle_rxmac(hldev, val64);
263 if (status != XGE_HAL_OK) {
268 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_RXXGXS)) {
269 xge_hal_status_e status;
270 hldev->stats.sw_dev_info_stats.rxxgxs_intr_cnt++;
271 status = __hal_device_handle_rxxgxs(hldev, val64);
272 if (status != XGE_HAL_OK) {
277 if (xge_os_unlikely(val64 & XGE_HAL_GEN_INTR_MC)) {
278 xge_hal_status_e status;
279 hldev->stats.sw_dev_info_stats.mc_intr_cnt++;
280 status = __hal_device_handle_mc(hldev, val64);
281 if (status != XGE_HAL_OK) {
290 * xge_hal_device_clear_rx - Acknowledge (that is, clear) the
291 * condition that has caused the RX interrupt.
292 * @hldev: HAL device handle.
294 * Acknowledge (that is, clear) the condition that has caused
296 * See also: xge_hal_device_begin_irq(), xge_hal_device_continue_irq(),
297 * xge_hal_device_clear_tx(), xge_hal_device_mask_rx().
299 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
300 xge_hal_device_clear_rx(xge_hal_device_t *hldev)
302 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
304 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
305 0xFFFFFFFFFFFFFFFFULL,
306 &isrbar0->rx_traffic_int);
310 * xge_hal_device_clear_tx - Acknowledge (that is, clear) the
311 * condition that has caused the TX interrupt.
312 * @hldev: HAL device handle.
314 * Acknowledge (that is, clear) the condition that has caused
316 * See also: xge_hal_device_begin_irq(), xge_hal_device_continue_irq(),
317 * xge_hal_device_clear_rx(), xge_hal_device_mask_tx().
319 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
320 xge_hal_device_clear_tx(xge_hal_device_t *hldev)
322 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
324 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
325 0xFFFFFFFFFFFFFFFFULL,
326 &isrbar0->tx_traffic_int);
330 * xge_hal_device_poll_rx_channel - Poll Rx channel for completed
331 * descriptors and process the same.
332 * @channel: HAL channel.
333 * @got_rx: Buffer to return the flag set if receive interrupt is occured
335 * The function polls the Rx channel for the completed descriptors and calls
336 * the upper-layer driver (ULD) via supplied completion callback.
338 * Returns: XGE_HAL_OK, if the polling is completed successful.
339 * XGE_HAL_COMPLETIONS_REMAIN: There are still more completed
340 * descriptors available which are yet to be processed.
342 * See also: xge_hal_device_poll_tx_channel()
344 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e
345 xge_hal_device_poll_rx_channel(xge_hal_channel_t *channel, int *got_rx)
347 xge_hal_status_e ret = XGE_HAL_OK;
348 xge_hal_dtr_h first_dtrh;
349 xge_hal_device_t *hldev = (xge_hal_device_t *)channel->devh;
353 /* for each opened rx channel */
354 got_bytes = *got_rx = 0;
355 ((xge_hal_ring_t *)channel)->cmpl_cnt = 0;
356 channel->poll_bytes = 0;
357 if ((ret = xge_hal_ring_dtr_next_completed (channel, &first_dtrh,
358 &t_code)) == XGE_HAL_OK) {
359 if (channel->callback(channel, first_dtrh,
360 t_code, channel->userdata) != XGE_HAL_OK) {
361 (*got_rx) += ((xge_hal_ring_t *)channel)->cmpl_cnt + 1;
362 got_bytes += channel->poll_bytes + 1;
363 ret = XGE_HAL_COMPLETIONS_REMAIN;
365 (*got_rx) += ((xge_hal_ring_t *)channel)->cmpl_cnt + 1;
366 got_bytes += channel->poll_bytes + 1;
371 hldev->irq_workload_rxd[channel->post_qid] += *got_rx;
372 hldev->irq_workload_rxcnt[channel->post_qid] ++;
374 hldev->irq_workload_rxlen[channel->post_qid] += got_bytes;
380 * xge_hal_device_poll_tx_channel - Poll Tx channel for completed
381 * descriptors and process the same.
382 * @channel: HAL channel.
383 * @got_tx: Buffer to return the flag set if transmit interrupt is occured
385 * The function polls the Tx channel for the completed descriptors and calls
386 * the upper-layer driver (ULD) via supplied completion callback.
388 * Returns: XGE_HAL_OK, if the polling is completed successful.
389 * XGE_HAL_COMPLETIONS_REMAIN: There are still more completed
390 * descriptors available which are yet to be processed.
392 * See also: xge_hal_device_poll_rx_channel().
394 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e
395 xge_hal_device_poll_tx_channel(xge_hal_channel_t *channel, int *got_tx)
397 xge_hal_dtr_h first_dtrh;
398 xge_hal_device_t *hldev = (xge_hal_device_t *)channel->devh;
402 /* for each opened tx channel */
403 got_bytes = *got_tx = 0;
404 channel->poll_bytes = 0;
405 if (xge_hal_fifo_dtr_next_completed (channel, &first_dtrh,
406 &t_code) == XGE_HAL_OK) {
407 if (channel->callback(channel, first_dtrh,
408 t_code, channel->userdata) != XGE_HAL_OK) {
410 got_bytes += channel->poll_bytes + 1;
411 return XGE_HAL_COMPLETIONS_REMAIN;
414 got_bytes += channel->poll_bytes + 1;
418 hldev->irq_workload_txd[channel->post_qid] += *got_tx;
419 hldev->irq_workload_txcnt[channel->post_qid] ++;
421 hldev->irq_workload_txlen[channel->post_qid] += got_bytes;
427 * xge_hal_device_poll_rx_channels - Poll Rx channels for completed
428 * descriptors and process the same.
429 * @hldev: HAL device handle.
430 * @got_rx: Buffer to return flag set if receive is ready
432 * The function polls the Rx channels for the completed descriptors and calls
433 * the upper-layer driver (ULD) via supplied completion callback.
435 * Returns: XGE_HAL_OK, if the polling is completed successful.
436 * XGE_HAL_COMPLETIONS_REMAIN: There are still more completed
437 * descriptors available which are yet to be processed.
439 * See also: xge_hal_device_poll_tx_channels(), xge_hal_device_continue_irq().
441 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e
442 xge_hal_device_poll_rx_channels(xge_hal_device_t *hldev, int *got_rx)
445 xge_hal_channel_t *channel;
447 /* for each opened rx channel */
448 xge_list_for_each(item, &hldev->ring_channels) {
449 if (hldev->terminating)
451 channel = xge_container_of(item, xge_hal_channel_t, item);
452 (void) xge_hal_device_poll_rx_channel(channel, got_rx);
459 * xge_hal_device_poll_tx_channels - Poll Tx channels for completed
460 * descriptors and process the same.
461 * @hldev: HAL device handle.
462 * @got_tx: Buffer to return flag set if transmit is ready
464 * The function polls the Tx channels for the completed descriptors and calls
465 * the upper-layer driver (ULD) via supplied completion callback.
467 * Returns: XGE_HAL_OK, if the polling is completed successful.
468 * XGE_HAL_COMPLETIONS_REMAIN: There are still more completed
469 * descriptors available which are yet to be processed.
471 * See also: xge_hal_device_poll_rx_channels(), xge_hal_device_continue_irq().
473 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e
474 xge_hal_device_poll_tx_channels(xge_hal_device_t *hldev, int *got_tx)
477 xge_hal_channel_t *channel;
479 /* for each opened tx channel */
480 xge_list_for_each(item, &hldev->fifo_channels) {
481 if (hldev->terminating)
483 channel = xge_container_of(item, xge_hal_channel_t, item);
484 (void) xge_hal_device_poll_tx_channel(channel, got_tx);
491 * xge_hal_device_mask_tx - Mask Tx interrupts.
492 * @hldev: HAL device handle.
494 * Mask Tx device interrupts.
496 * See also: xge_hal_device_unmask_tx(), xge_hal_device_mask_rx(),
497 * xge_hal_device_clear_tx().
499 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
500 xge_hal_device_mask_tx(xge_hal_device_t *hldev)
502 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
504 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
505 0xFFFFFFFFFFFFFFFFULL,
506 &isrbar0->tx_traffic_mask);
510 * xge_hal_device_mask_rx - Mask Rx interrupts.
511 * @hldev: HAL device handle.
513 * Mask Rx device interrupts.
515 * See also: xge_hal_device_unmask_rx(), xge_hal_device_mask_tx(),
516 * xge_hal_device_clear_rx().
518 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
519 xge_hal_device_mask_rx(xge_hal_device_t *hldev)
521 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
523 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
524 0xFFFFFFFFFFFFFFFFULL,
525 &isrbar0->rx_traffic_mask);
529 * xge_hal_device_mask_all - Mask all device interrupts.
530 * @hldev: HAL device handle.
532 * Mask all device interrupts.
534 * See also: xge_hal_device_unmask_all()
536 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
537 xge_hal_device_mask_all(xge_hal_device_t *hldev)
539 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
541 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
542 0xFFFFFFFFFFFFFFFFULL,
543 &isrbar0->general_int_mask);
547 * xge_hal_device_unmask_tx - Unmask Tx interrupts.
548 * @hldev: HAL device handle.
550 * Unmask Tx device interrupts.
552 * See also: xge_hal_device_mask_tx(), xge_hal_device_clear_tx().
554 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
555 xge_hal_device_unmask_tx(xge_hal_device_t *hldev)
557 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
559 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
561 &isrbar0->tx_traffic_mask);
565 * xge_hal_device_unmask_rx - Unmask Rx interrupts.
566 * @hldev: HAL device handle.
568 * Unmask Rx device interrupts.
570 * See also: xge_hal_device_mask_rx(), xge_hal_device_clear_rx().
572 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
573 xge_hal_device_unmask_rx(xge_hal_device_t *hldev)
575 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
577 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
579 &isrbar0->rx_traffic_mask);
583 * xge_hal_device_unmask_all - Unmask all device interrupts.
584 * @hldev: HAL device handle.
586 * Unmask all device interrupts.
588 * See also: xge_hal_device_mask_all()
590 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE void
591 xge_hal_device_unmask_all(xge_hal_device_t *hldev)
593 xge_hal_pci_bar0_t *isrbar0 = (xge_hal_pci_bar0_t *)hldev->isrbar0;
595 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0,
597 &isrbar0->general_int_mask);
602 * xge_hal_device_continue_irq - Continue handling IRQ: process all
603 * completed descriptors.
604 * @hldev: HAL device handle.
606 * Process completed descriptors and unmask the device interrupts.
608 * The xge_hal_device_continue_irq() walks all open channels
609 * and calls upper-layer driver (ULD) via supplied completion
610 * callback. Note that the completion callback is specified at channel open
611 * time, see xge_hal_channel_open().
613 * Note that the xge_hal_device_continue_irq is part of the _fast_ path.
614 * To optimize the processing, the function does _not_ check for
617 * The latter is done in a polling fashion, via xge_hal_device_poll().
619 * Returns: XGE_HAL_OK.
621 * See also: xge_hal_device_handle_irq(), xge_hal_device_poll(),
622 * xge_hal_ring_dtr_next_completed(),
623 * xge_hal_fifo_dtr_next_completed(), xge_hal_channel_callback_f{}.
625 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e
626 xge_hal_device_continue_irq(xge_hal_device_t *hldev)
628 int got_rx = 1, got_tx = 1;
629 int isr_polling_cnt = hldev->config.isr_polling_cnt;
635 (void) xge_hal_device_poll_rx_channels(hldev, &got_rx);
636 if (got_tx && hldev->tti_enabled)
637 (void) xge_hal_device_poll_tx_channels(hldev, &got_tx);
639 if (!got_rx && !got_tx)
642 count += (got_rx + got_tx);
643 }while (isr_polling_cnt--);
646 hldev->stats.sw_dev_info_stats.not_traffic_intr_cnt++;
652 * xge_hal_device_handle_irq - Handle device IRQ.
653 * @hldev: HAL device handle.
655 * Perform the complete handling of the line interrupt. The function
656 * performs two calls.
657 * First it uses xge_hal_device_begin_irq() to check the reason for
658 * the interrupt and mask the device interrupts.
659 * Second, it calls xge_hal_device_continue_irq() to process all
660 * completed descriptors and re-enable the interrupts.
662 * Returns: XGE_HAL_OK - success;
663 * XGE_HAL_ERR_WRONG_IRQ - (shared) IRQ produced by other device.
665 * See also: xge_hal_device_begin_irq(), xge_hal_device_continue_irq().
667 __HAL_STATIC_DEVICE __HAL_INLINE_DEVICE xge_hal_status_e
668 xge_hal_device_handle_irq(xge_hal_device_t *hldev)
671 xge_hal_status_e status;
673 xge_hal_device_mask_all(hldev);
675 status = xge_hal_device_begin_irq(hldev, &reason);
676 if (status != XGE_HAL_OK) {
677 xge_hal_device_unmask_all(hldev);
681 if (reason & XGE_HAL_GEN_INTR_RXTRAFFIC) {
682 xge_hal_device_clear_rx(hldev);
685 status = xge_hal_device_continue_irq(hldev);
687 xge_hal_device_clear_tx(hldev);
689 xge_hal_device_unmask_all(hldev);
694 #if defined(XGE_HAL_CONFIG_LRO)
697 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL int
698 __hal_lro_check_for_session_match(lro_t *lro, tcplro_t *tcp, iplro_t *ip)
701 /* Match Source address field */
702 if ((lro->ip_hdr->saddr != ip->saddr))
705 /* Match Destination address field */
706 if ((lro->ip_hdr->daddr != ip->daddr))
709 /* Match Source Port field */
710 if ((lro->tcp_hdr->source != tcp->source))
713 /* Match Destination Port field */
714 if ((lro->tcp_hdr->dest != tcp->dest))
721 * __hal_tcp_seg_len: Find the tcp seg len.
724 * returns: Tcp seg length.
726 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL u16
727 __hal_tcp_seg_len(iplro_t *ip, tcplro_t *tcp)
731 ret = (xge_os_ntohs(ip->tot_len) -
732 ((ip->version_ihl & 0x0F)<<2) -
733 ((tcp->doff_res)>>2));
738 * __hal_ip_lro_capable: Finds whether ip is lro capable.
740 * @ext_info: descriptor info.
742 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
743 __hal_ip_lro_capable(iplro_t *ip,
744 xge_hal_dtr_info_t *ext_info)
747 #ifdef XGE_LL_DEBUG_DUMP_PKT
750 u8 ch, *iph = (u8 *)ip;
752 xge_debug_ring(XGE_TRACE, "Dump Ip:" );
753 for (i =0; i < 40; i++) {
754 ch = ntohs(*((u8 *)(iph + i)) );
755 printf("i:%d %02x, ",i,ch);
760 if (ip->version_ihl != IP_FAST_PATH_HDR_MASK) {
761 xge_debug_ring(XGE_ERR, "iphdr !=45 :%d",ip->version_ihl);
765 if (ext_info->proto & XGE_HAL_FRAME_PROTO_IP_FRAGMENTED) {
766 xge_debug_ring(XGE_ERR, "IP fragmented");
774 * __hal_tcp_lro_capable: Finds whether tcp is lro capable.
778 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
779 __hal_tcp_lro_capable(iplro_t *ip, tcplro_t *tcp, lro_t *lro, int *ts_off)
781 #ifdef XGE_LL_DEBUG_DUMP_PKT
786 xge_debug_ring(XGE_TRACE, "Dump Tcp:" );
787 for (i =0; i < 20; i++) {
788 ch = ntohs(*((u8 *)((u8 *)tcp + i)) );
789 xge_os_printf("i:%d %02x, ",i,ch);
793 if ((TCP_FAST_PATH_HDR_MASK2 != tcp->ctrl) &&
794 (TCP_FAST_PATH_HDR_MASK3 != tcp->ctrl))
798 if (TCP_FAST_PATH_HDR_MASK1 != tcp->doff_res) {
799 u16 tcp_hdr_len = tcp->doff_res >> 2; /* TCP header len */
800 u16 off = 20; /* Start of tcp options */
803 /* Does Packet can contain time stamp */
804 if (tcp_hdr_len < 32) {
806 * If the session is not opened, we can consider
807 * this packet for LRO
815 /* Ignore No-operation 0x1 */
816 while (((u8 *)tcp)[off] == 0x1)
819 /* Next option == Timestamp */
820 if (((u8 *)tcp)[off] != 0x8) {
822 * If the session ie not opened, we can consider
823 * this packet for LRO
836 * Now the session is opened. If the LRO frame doesn't
837 * have time stamp, we cannot consider current packet for
840 if (lro->ts_off == -1) {
841 xge_debug_ring(XGE_ERR, "Pkt received with time stamp after session opened with no time stamp : %02x %02x", tcp->doff_res, tcp->ctrl);
846 * If the difference is greater than three, then there are
847 * more options possible.
848 * else, there are two cases:
849 * case 1: remaining are padding bytes.
850 * case 2: remaining can contain options or padding
852 off += ((u8 *)tcp)[off+1];
853 diff = tcp_hdr_len - off;
856 * Probably contains more options.
858 xge_debug_ring(XGE_ERR, "tcphdr not fastpth : pkt received with tcp options in addition to time stamp after the session is opened %02x %02x ", tcp->doff_res, tcp->ctrl);
862 for (i = 0; i < diff; i++) {
863 u8 byte = ((u8 *)tcp)[off+i];
865 /* Ignore No-operation 0x1 */
866 if ((byte == 0x0) || (byte == 0x1))
868 xge_debug_ring(XGE_ERR, "tcphdr not fastpth : pkt received with tcp options in addition to time stamp after the session is opened %02x %02x ", tcp->doff_res, tcp->ctrl);
873 * Update the time stamp of LRO frame.
875 xge_os_memcpy(((char *)lro->tcp_hdr + lro->ts_off + 2),
876 (char *)((char *)tcp + (*ts_off) + 2), 8);
882 xge_debug_ring(XGE_TRACE, "tcphdr not fastpth %02x %02x", tcp->doff_res, tcp->ctrl);
888 * __hal_lro_capable: Finds whether frame is lro capable.
889 * @buffer: Ethernet frame.
892 * @ext_info: Descriptor info.
894 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
895 __hal_lro_capable( u8 *buffer,
898 xge_hal_dtr_info_t *ext_info)
900 u8 ip_off, ip_length;
902 if (!(ext_info->proto & XGE_HAL_FRAME_PROTO_TCP)) {
903 xge_debug_ring(XGE_ERR, "Cant do lro %d", ext_info->proto);
909 #ifdef XGE_LL_DEBUG_DUMP_PKT
914 xge_os_printf("Dump Eth:" );
915 for (i =0; i < 60; i++) {
916 ch = ntohs(*((u8 *)(buffer + i)) );
917 xge_os_printf("i:%d %02x, ",i,ch);
922 switch (ext_info->frame) {
923 case XGE_HAL_FRAME_TYPE_DIX:
924 ip_off = XGE_HAL_HEADER_ETHERNET_II_802_3_SIZE;
926 case XGE_HAL_FRAME_TYPE_LLC:
927 ip_off = (XGE_HAL_HEADER_ETHERNET_II_802_3_SIZE +
928 XGE_HAL_HEADER_802_2_SIZE);
930 case XGE_HAL_FRAME_TYPE_SNAP:
931 ip_off = (XGE_HAL_HEADER_ETHERNET_II_802_3_SIZE +
932 XGE_HAL_HEADER_SNAP_SIZE);
934 default: // XGE_HAL_FRAME_TYPE_IPX, etc.
939 if (ext_info->proto & XGE_HAL_FRAME_PROTO_VLAN_TAGGED) {
940 ip_off += XGE_HAL_HEADER_VLAN_SIZE;
943 /* Grab ip, tcp headers */
944 *ip = (iplro_t *)((char*)buffer + ip_off);
947 ip_length = (u8)((*ip)->version_ihl & 0x0F);
948 ip_length = ip_length <<2;
949 *tcp = (tcplro_t *)((char *)*ip + ip_length);
951 xge_debug_ring(XGE_TRACE, "ip_length:%d ip:"XGE_OS_LLXFMT
952 " tcp:"XGE_OS_LLXFMT"", (int)ip_length,
953 (unsigned long long)(ulong_t)*ip, (unsigned long long)(ulong_t)*tcp);
961 * __hal_open_lro_session: Open a new LRO session.
962 * @buffer: Ethernet frame.
966 * @ext_info: Descriptor info.
967 * @hldev: Hal context.
968 * @ring_lro: LRO descriptor per rx ring.
970 * @tcp_seg_len: Length of tcp segment.
971 * @ts_off: time stamp offset in the packet.
973 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL void
974 __hal_open_lro_session (u8 *buffer, iplro_t *ip, tcplro_t *tcp, lro_t **lro,
975 xge_hal_device_t *hldev, xge_hal_lro_desc_t *ring_lro, int slot,
976 u32 tcp_seg_len, int ts_off)
979 lro_t *lro_new = &ring_lro->lro_pool[slot];
982 lro_new->ll_hdr = buffer;
983 lro_new->ip_hdr = ip;
984 lro_new->tcp_hdr = tcp;
985 lro_new->tcp_next_seq_num = tcp_seg_len + xge_os_ntohl(
987 lro_new->tcp_seq_num = tcp->seq;
988 lro_new->tcp_ack_num = tcp->ack_seq;
990 lro_new->total_length = xge_os_ntohs(ip->tot_len);
991 lro_new->frags_len = 0;
992 lro_new->ts_off = ts_off;
994 hldev->stats.sw_dev_info_stats.tot_frms_lroised++;
995 hldev->stats.sw_dev_info_stats.tot_lro_sessions++;
997 *lro = ring_lro->lro_recent = lro_new;
1001 * __hal_lro_get_free_slot: Get a free LRO bucket.
1002 * @ring_lro: LRO descriptor per ring.
1004 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL int
1005 __hal_lro_get_free_slot (xge_hal_lro_desc_t *ring_lro)
1009 for (i = 0; i < XGE_HAL_LRO_MAX_BUCKETS; i++) {
1010 lro_t *lro_temp = &ring_lro->lro_pool[i];
1012 if (!lro_temp->in_use)
1019 * __hal_get_lro_session: Gets matching LRO session or creates one.
1020 * @eth_hdr: Ethernet header.
1024 * @ext_info: Descriptor info.
1025 * @hldev: Hal context.
1026 * @ring_lro: LRO descriptor per rx ring
1028 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
1029 __hal_get_lro_session (u8 *eth_hdr,
1033 xge_hal_dtr_info_t *ext_info,
1034 xge_hal_device_t *hldev,
1035 xge_hal_lro_desc_t *ring_lro,
1036 lro_t **lro_end3 /* Valid only when ret=END_3 */)
1039 int i, free_slot = -1;
1043 *lro = lro_match = NULL;
1045 * Compare the incoming frame with the lro session left from the
1046 * previous call. There is a good chance that this incoming frame
1047 * matches the lro session.
1049 if (ring_lro->lro_recent && ring_lro->lro_recent->in_use) {
1050 if (__hal_lro_check_for_session_match(ring_lro->lro_recent,
1053 lro_match = ring_lro->lro_recent;
1058 * Search in the pool of LROs for the session that matches
1059 * the incoming frame.
1061 for (i = 0; i < XGE_HAL_LRO_MAX_BUCKETS; i++) {
1062 lro_t *lro_temp = &ring_lro->lro_pool[i];
1064 if (!lro_temp->in_use) {
1065 if (free_slot == -1)
1070 if (__hal_lro_check_for_session_match(lro_temp, tcp,
1071 ip) == XGE_HAL_OK) {
1072 lro_match = lro_temp;
1081 * Matching LRO Session found
1085 if (lro_match->tcp_next_seq_num != xge_os_ntohl(tcp->seq)) {
1086 xge_debug_ring(XGE_ERR, "**retransmit **"
1088 hldev->stats.sw_dev_info_stats.lro_out_of_seq_pkt_cnt++;
1089 return XGE_HAL_INF_LRO_END_2;
1092 if (XGE_HAL_OK != __hal_ip_lro_capable(ip, ext_info))
1094 return XGE_HAL_INF_LRO_END_2;
1097 if (XGE_HAL_OK != __hal_tcp_lro_capable(ip, tcp, lro_match,
1100 * Close the current session and open a new
1101 * LRO session with this packet,
1102 * provided it has tcp payload
1104 tcp_seg_len = __hal_tcp_seg_len(ip, tcp);
1105 if (tcp_seg_len == 0)
1107 return XGE_HAL_INF_LRO_END_2;
1110 /* Get a free bucket */
1111 free_slot = __hal_lro_get_free_slot(ring_lro);
1112 if (free_slot == -1)
1114 return XGE_HAL_INF_LRO_END_2;
1118 * Open a new LRO session
1120 __hal_open_lro_session (eth_hdr, ip, tcp, lro_end3,
1121 hldev, ring_lro, free_slot, tcp_seg_len,
1124 return XGE_HAL_INF_LRO_END_3;
1128 * The frame is good, in-sequence, can be LRO-ed;
1129 * take its (latest) ACK - unless it is a dupack.
1130 * Note: to be exact need to check window size as well..
1132 if (lro_match->tcp_ack_num == tcp->ack_seq &&
1133 lro_match->tcp_seq_num == tcp->seq) {
1134 hldev->stats.sw_dev_info_stats.lro_dup_pkt_cnt++;
1135 return XGE_HAL_INF_LRO_END_2;
1138 lro_match->tcp_seq_num = tcp->seq;
1139 lro_match->tcp_ack_num = tcp->ack_seq;
1140 lro_match->frags_len += __hal_tcp_seg_len(ip, tcp);
1142 ring_lro->lro_recent = lro_match;
1144 return XGE_HAL_INF_LRO_CONT;
1147 /* ********** New Session ***************/
1148 if (free_slot == -1)
1149 return XGE_HAL_INF_LRO_UNCAPABLE;
1151 if (XGE_HAL_FAIL == __hal_ip_lro_capable(ip, ext_info))
1152 return XGE_HAL_INF_LRO_UNCAPABLE;
1154 if (XGE_HAL_FAIL == __hal_tcp_lro_capable(ip, tcp, NULL, &ts_off))
1155 return XGE_HAL_INF_LRO_UNCAPABLE;
1157 xge_debug_ring(XGE_TRACE, "Creating lro session.");
1160 * Open a LRO session, provided the packet contains payload.
1162 tcp_seg_len = __hal_tcp_seg_len(ip, tcp);
1163 if (tcp_seg_len == 0)
1164 return XGE_HAL_INF_LRO_UNCAPABLE;
1166 __hal_open_lro_session (eth_hdr, ip, tcp, lro, hldev, ring_lro, free_slot,
1167 tcp_seg_len, ts_off);
1169 return XGE_HAL_INF_LRO_BEGIN;
1173 * __hal_lro_under_optimal_thresh: Finds whether combined session is optimal.
1177 * @hldev: Hal context.
1179 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
1180 __hal_lro_under_optimal_thresh (iplro_t *ip,
1183 xge_hal_device_t *hldev)
1185 if (!lro) return XGE_HAL_FAIL;
1187 if ((lro->total_length + __hal_tcp_seg_len(ip, tcp) ) >
1188 hldev->config.lro_frm_len) {
1189 xge_debug_ring(XGE_TRACE, "Max LRO frame len exceeded:"
1190 "max length %d ", hldev->config.lro_frm_len);
1191 hldev->stats.sw_dev_info_stats.lro_frm_len_exceed_cnt++;
1192 return XGE_HAL_FAIL;
1195 if (lro->sg_num == hldev->config.lro_sg_size) {
1196 xge_debug_ring(XGE_TRACE, "Max sg count exceeded:"
1197 "max sg %d ", hldev->config.lro_sg_size);
1198 hldev->stats.sw_dev_info_stats.lro_sg_exceed_cnt++;
1199 return XGE_HAL_FAIL;
1206 * __hal_collapse_ip_hdr: Collapses ip header.
1210 * @hldev: Hal context.
1212 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
1213 __hal_collapse_ip_hdr ( iplro_t *ip,
1216 xge_hal_device_t *hldev)
1219 lro->total_length += __hal_tcp_seg_len(ip, tcp);
1221 /* May be we have to handle time stamps or more options */
1228 * __hal_collapse_tcp_hdr: Collapses tcp header.
1232 * @hldev: Hal context.
1234 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
1235 __hal_collapse_tcp_hdr ( iplro_t *ip,
1238 xge_hal_device_t *hldev)
1240 lro->tcp_next_seq_num += __hal_tcp_seg_len(ip, tcp);
1246 * __hal_append_lro: Appends new frame to existing LRO session.
1248 * @tcp: IN tcp header, OUT tcp payload.
1249 * @seg_len: tcp payload length.
1251 * @hldev: Hal context.
1253 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
1254 __hal_append_lro(iplro_t *ip,
1258 xge_hal_device_t *hldev)
1260 (void) __hal_collapse_ip_hdr(ip, *tcp, lro, hldev);
1261 (void) __hal_collapse_tcp_hdr(ip, *tcp, lro, hldev);
1262 // Update mbuf chain will be done in ll driver.
1263 // xge_hal_accumulate_large_rx on success of appending new frame to
1264 // lro will return to ll driver tcpdata pointer, and tcp payload length.
1265 // along with return code lro frame appended.
1268 *seg_len = __hal_tcp_seg_len(ip, *tcp);
1269 *tcp = (tcplro_t *)((char *)*tcp + (((*tcp)->doff_res)>>2));
1276 * __xge_hal_accumulate_large_rx: LRO a given frame
1278 * @ring: rx ring number
1279 * @eth_hdr: ethernet header.
1280 * @ip_hdr: ip header (optional)
1282 * @seglen: packet length.
1283 * @p_lro: lro pointer.
1284 * @ext_info: descriptor info, see xge_hal_dtr_info_t{}.
1285 * @hldev: HAL device.
1286 * @lro_end3: for lro_end3 output
1288 * LRO the newly received frame, i.e. attach it (if possible) to the
1289 * already accumulated (i.e., already LRO-ed) received frames (if any),
1290 * to form one super-sized frame for the subsequent processing
1293 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
1294 xge_hal_lro_process_rx(int ring, u8 *eth_hdr, u8 *ip_hdr, tcplro_t **tcp,
1295 u32 *seglen, lro_t **p_lro,
1296 xge_hal_dtr_info_t *ext_info, xge_hal_device_t *hldev,
1299 iplro_t *ip = (iplro_t *)ip_hdr;
1300 xge_hal_status_e ret;
1303 xge_debug_ring(XGE_TRACE, "Entered accumu lro. ");
1304 if (XGE_HAL_OK != __hal_lro_capable(eth_hdr, &ip, (tcplro_t **)tcp,
1306 return XGE_HAL_INF_LRO_UNCAPABLE;
1309 * This function shall get matching LRO or else
1310 * create one and return it
1312 ret = __hal_get_lro_session(eth_hdr, ip, (tcplro_t *)*tcp,
1313 p_lro, ext_info, hldev, &hldev->lro_desc[ring],
1315 xge_debug_ring(XGE_TRACE, "ret from get_lro:%d ",ret);
1317 if (XGE_HAL_INF_LRO_CONT == ret) {
1318 if (XGE_HAL_OK == __hal_lro_under_optimal_thresh(ip,
1319 (tcplro_t *)*tcp, lro, hldev)) {
1320 (void) __hal_append_lro(ip,(tcplro_t **) tcp, seglen,
1322 hldev->stats.sw_dev_info_stats.tot_frms_lroised++;
1324 if (lro->sg_num >= hldev->config.lro_sg_size) {
1325 hldev->stats.sw_dev_info_stats.lro_sg_exceed_cnt++;
1326 ret = XGE_HAL_INF_LRO_END_1;
1329 } else ret = XGE_HAL_INF_LRO_END_2;
1333 * Since its time to flush,
1334 * update ip header so that it can be sent up
1336 if ((ret == XGE_HAL_INF_LRO_END_1) ||
1337 (ret == XGE_HAL_INF_LRO_END_2) ||
1338 (ret == XGE_HAL_INF_LRO_END_3)) {
1339 lro->ip_hdr->tot_len = xge_os_htons((*p_lro)->total_length);
1340 lro->ip_hdr->check = xge_os_htons(0);
1341 lro->ip_hdr->check = XGE_LL_IP_FAST_CSUM(((u8 *)(lro->ip_hdr)),
1342 (lro->ip_hdr->version_ihl & 0x0F));
1343 lro->tcp_hdr->ack_seq = lro->tcp_ack_num;
1350 * xge_hal_accumulate_large_rx: LRO a given frame
1352 * @buffer: Ethernet frame.
1354 * @seglen: packet length.
1355 * @p_lro: lro pointer.
1356 * @ext_info: descriptor info, see xge_hal_dtr_info_t{}.
1357 * @hldev: HAL device.
1358 * @lro_end3: for lro_end3 output
1360 * LRO the newly received frame, i.e. attach it (if possible) to the
1361 * already accumulated (i.e., already LRO-ed) received frames (if any),
1362 * to form one super-sized frame for the subsequent processing
1365 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL xge_hal_status_e
1366 xge_hal_accumulate_large_rx(u8 *buffer, tcplro_t **tcp, u32 *seglen,
1367 lro_t **p_lro, xge_hal_dtr_info_t *ext_info, xge_hal_device_t *hldev,
1371 return xge_hal_lro_process_rx(ring, buffer, NULL, tcp, seglen, p_lro,
1372 ext_info, hldev, lro_end3);
1376 * xge_hal_lro_close_session: Close LRO session
1377 * @lro: LRO Session.
1378 * @hldev: HAL Context.
1380 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL void
1381 xge_hal_lro_close_session (lro_t *lro)
1387 * xge_hal_lro_next_session: Returns next LRO session in the list or NULL
1389 * @hldev: HAL Context.
1390 * @ring: rx ring number.
1392 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL lro_t *
1393 xge_hal_lro_next_session (xge_hal_device_t *hldev, int ring)
1395 xge_hal_lro_desc_t *ring_lro = &hldev->lro_desc[ring];
1397 int start_idx = ring_lro->lro_next_idx;
1399 for(i = start_idx; i < XGE_HAL_LRO_MAX_BUCKETS; i++) {
1400 lro_t *lro = &ring_lro->lro_pool[i];
1405 lro->ip_hdr->tot_len = xge_os_htons(lro->total_length);
1406 lro->ip_hdr->check = xge_os_htons(0);
1407 lro->ip_hdr->check = XGE_LL_IP_FAST_CSUM(((u8 *)(lro->ip_hdr)),
1408 (lro->ip_hdr->version_ihl & 0x0F));
1409 ring_lro->lro_next_idx = i + 1;
1413 ring_lro->lro_next_idx = 0;
1418 __HAL_STATIC_CHANNEL __HAL_INLINE_CHANNEL lro_t *
1419 xge_hal_lro_get_next_session(xge_hal_device_t *hldev)
1421 int ring = 0; /* assume default ring=0 */
1422 return xge_hal_lro_next_session(hldev, ring);