sys/dev/nxge/xgehal/xgehal-ring-fp.c

   1 /*-
   2  * Copyright (c) 2002-2007 Neterion, Inc.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  *
  26  * $FreeBSD$
  27  */
  28
  29 #ifdef XGE_DEBUG_FP
  30 #include <dev/nxge/include/xgehal-ring.h>
  31 #endif
  32
  33 __HAL_STATIC_RING __HAL_INLINE_RING xge_hal_ring_rxd_priv_t*
  34 __hal_ring_rxd_priv(xge_hal_ring_t *ring, xge_hal_dtr_h dtrh)
  35 {
  36
  37         xge_hal_ring_rxd_1_t *rxdp = (xge_hal_ring_rxd_1_t *)dtrh;
  38         xge_hal_ring_rxd_priv_t *rxd_priv;
  39
  40         xge_assert(rxdp);
  41
  42 #if defined(XGE_HAL_USE_5B_MODE)
  43         xge_assert(ring);
  44         if (ring->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5) {
  45             xge_hal_ring_rxd_5_t *rxdp_5 = (xge_hal_ring_rxd_5_t *)dtrh;
  46 #if defined (XGE_OS_PLATFORM_64BIT)
  47             int memblock_idx = rxdp_5->host_control >> 16;
  48             int i = rxdp_5->host_control & 0xFFFF;
  49             rxd_priv = (xge_hal_ring_rxd_priv_t *)
  50                 ((char*)ring->mempool->memblocks_priv_arr[memblock_idx] + ring->rxd_priv_size * i);
  51 #else
  52             /* 32-bit case */
  53             rxd_priv = (xge_hal_ring_rxd_priv_t *)rxdp_5->host_control;
  54 #endif
  55         } else
  56 #endif
  57         {
  58             rxd_priv = (xge_hal_ring_rxd_priv_t *)
  59                     (ulong_t)rxdp->host_control;
  60         }
  61
  62         xge_assert(rxd_priv);
  63         xge_assert(rxd_priv->dma_object);
  64
  65         xge_assert(rxd_priv->dma_object->handle == rxd_priv->dma_handle);
  66
  67         xge_assert(rxd_priv->dma_object->addr + rxd_priv->dma_offset ==
  68                                 rxd_priv->dma_addr);
  69
  70         return rxd_priv;
  71 }
  72
  73 __HAL_STATIC_RING __HAL_INLINE_RING int
  74 __hal_ring_block_memblock_idx(xge_hal_ring_block_t *block)
  75 {
  76            return (int)*((u64 *)(void *)((char *)block +
  77                                    XGE_HAL_RING_MEMBLOCK_IDX_OFFSET));
  78 }
  79
  80 __HAL_STATIC_RING __HAL_INLINE_RING void
  81 __hal_ring_block_memblock_idx_set(xge_hal_ring_block_t*block, int memblock_idx)
  82 {
  83            *((u64 *)(void *)((char *)block +
  84                            XGE_HAL_RING_MEMBLOCK_IDX_OFFSET)) =
  85                            memblock_idx;
  86 }
  87
  88
  89 __HAL_STATIC_RING __HAL_INLINE_RING dma_addr_t
  90 __hal_ring_block_next_pointer(xge_hal_ring_block_t *block)
  91 {
  92         return (dma_addr_t)*((u64 *)(void *)((char *)block +
  93                 XGE_HAL_RING_NEXT_BLOCK_POINTER_OFFSET));
  94 }
  95
  96 __HAL_STATIC_RING __HAL_INLINE_RING void
  97 __hal_ring_block_next_pointer_set(xge_hal_ring_block_t *block,
  98                 dma_addr_t dma_next)
  99 {
 100         *((u64 *)(void *)((char *)block +
 101                   XGE_HAL_RING_NEXT_BLOCK_POINTER_OFFSET)) = dma_next;
 102 }
 103
 104 /**
 105  * xge_hal_ring_dtr_private - Get ULD private per-descriptor data.
 106  * @channelh: Channel handle.
 107  * @dtrh: Descriptor handle.
 108  *
 109  * Returns: private ULD info associated with the descriptor.
 110  * ULD requests per-descriptor space via xge_hal_channel_open().
 111  *
 112  * See also: xge_hal_fifo_dtr_private().
 113  * Usage: See ex_rx_compl{}.
 114  */
 115 __HAL_STATIC_RING __HAL_INLINE_RING void*
 116 xge_hal_ring_dtr_private(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh)
 117 {
 118         return (char *)__hal_ring_rxd_priv((xge_hal_ring_t *) channelh, dtrh) +
 119                         sizeof(xge_hal_ring_rxd_priv_t);
 120 }
 121
 122 /**
 123  * xge_hal_ring_dtr_reserve - Reserve ring descriptor.
 124  * @channelh: Channel handle.
 125  * @dtrh: Reserved descriptor. On success HAL fills this "out" parameter
 126  *        with a valid handle.
 127  *
 128  * Reserve Rx descriptor for the subsequent filling-in (by upper layer
 129  * driver (ULD)) and posting on the corresponding channel (@channelh)
 130  * via xge_hal_ring_dtr_post().
 131  *
 132  * Returns: XGE_HAL_OK - success.
 133  * XGE_HAL_INF_OUT_OF_DESCRIPTORS - Currently no descriptors available.
 134  *
 135  * See also: xge_hal_fifo_dtr_reserve(), xge_hal_ring_dtr_free(),
 136  * xge_hal_fifo_dtr_reserve_sp(), xge_hal_status_e{}.
 137  * Usage: See ex_post_all_rx{}.
 138  */
 139 __HAL_STATIC_RING __HAL_INLINE_RING xge_hal_status_e
 140 xge_hal_ring_dtr_reserve(xge_hal_channel_h channelh, xge_hal_dtr_h *dtrh)
 141 {
 142         xge_hal_status_e status;
 143 #if defined(XGE_HAL_RX_MULTI_RESERVE_IRQ)
 144         unsigned long flags;
 145 #endif
 146
 147 #if defined(XGE_HAL_RX_MULTI_RESERVE)
 148         xge_os_spin_lock(&((xge_hal_channel_t*)channelh)->reserve_lock);
 149 #elif defined(XGE_HAL_RX_MULTI_RESERVE_IRQ)
 150         xge_os_spin_lock_irq(&((xge_hal_channel_t*)channelh)->reserve_lock,
 151         flags);
 152 #endif
 153
 154         status = __hal_channel_dtr_alloc(channelh, dtrh);
 155
 156 #if defined(XGE_HAL_RX_MULTI_RESERVE)
 157         xge_os_spin_unlock(&((xge_hal_channel_t*)channelh)->reserve_lock);
 158 #elif defined(XGE_HAL_RX_MULTI_RESERVE_IRQ)
 159         xge_os_spin_unlock_irq(&((xge_hal_channel_t*)channelh)->reserve_lock,
 160                      flags);
 161 #endif
 162
 163         if (status == XGE_HAL_OK) {
 164             xge_hal_ring_rxd_1_t *rxdp = (xge_hal_ring_rxd_1_t *)*dtrh;
 165
 166             /* instead of memset: reset this RxD */
 167             rxdp->control_1 = rxdp->control_2 = 0;
 168
 169 #if defined(XGE_OS_MEMORY_CHECK)
 170             __hal_ring_rxd_priv((xge_hal_ring_t *) channelh, rxdp)->allocated = 1;
 171 #endif
 172         }
 173
 174         return status;
 175 }
 176
 177 /**
 178  * xge_hal_ring_dtr_info_get - Get extended information associated with
 179  * a completed receive descriptor for 1b mode.
 180  * @channelh: Channel handle.
 181  * @dtrh: Descriptor handle.
 182  * @ext_info: See xge_hal_dtr_info_t{}. Returned by HAL.
 183  *
 184  * Retrieve extended information associated with a completed receive descriptor.
 185  *
 186  * See also: xge_hal_dtr_info_t{}, xge_hal_ring_dtr_1b_get(),
 187  * xge_hal_ring_dtr_5b_get().
 188  */
 189 __HAL_STATIC_RING __HAL_INLINE_RING void
 190 xge_hal_ring_dtr_info_get(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh,
 191                 xge_hal_dtr_info_t *ext_info)
 192 {
 193         /* cast to 1-buffer mode RxD: the code below relies on the fact
 194          * that control_1 and control_2 are formatted the same way.. */
 195         xge_hal_ring_rxd_1_t *rxdp = (xge_hal_ring_rxd_1_t *)dtrh;
 196
 197         ext_info->l3_cksum = XGE_HAL_RXD_GET_L3_CKSUM(rxdp->control_1);
 198         ext_info->l4_cksum = XGE_HAL_RXD_GET_L4_CKSUM(rxdp->control_1);
 199             ext_info->frame = XGE_HAL_RXD_GET_FRAME_TYPE(rxdp->control_1);
 200             ext_info->proto = XGE_HAL_RXD_GET_FRAME_PROTO(rxdp->control_1);
 201         ext_info->vlan = XGE_HAL_RXD_GET_VLAN_TAG(rxdp->control_2);
 202
 203         /* Herc only, a few extra cycles imposed on Xena and/or
 204          * when RTH is not enabled.
 205          * Alternatively, could check
 206          * xge_hal_device_check_id(), hldev->config.rth_en, queue->rth_en */
 207         ext_info->rth_it_hit = XGE_HAL_RXD_GET_RTH_IT_HIT(rxdp->control_1);
 208         ext_info->rth_spdm_hit =
 209         XGE_HAL_RXD_GET_RTH_SPDM_HIT(rxdp->control_1);
 210         ext_info->rth_hash_type =
 211         XGE_HAL_RXD_GET_RTH_HASH_TYPE(rxdp->control_1);
 212         ext_info->rth_value = XGE_HAL_RXD_1_GET_RTH_VALUE(rxdp->control_2);
 213 }
 214
 215 /**
 216  * xge_hal_ring_dtr_info_nb_get - Get extended information associated
 217  * with a completed receive descriptor for 3b or 5b
 218  * modes.
 219  * @channelh: Channel handle.
 220  * @dtrh: Descriptor handle.
 221  * @ext_info: See xge_hal_dtr_info_t{}. Returned by HAL.
 222  *
 223  * Retrieve extended information associated with a completed receive descriptor.
 224  *
 225  * See also: xge_hal_dtr_info_t{}, xge_hal_ring_dtr_1b_get(),
 226  *           xge_hal_ring_dtr_5b_get().
 227  */
 228 __HAL_STATIC_RING __HAL_INLINE_RING void
 229 xge_hal_ring_dtr_info_nb_get(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh,
 230                 xge_hal_dtr_info_t *ext_info)
 231 {
 232         /* cast to 1-buffer mode RxD: the code below relies on the fact
 233          * that control_1 and control_2 are formatted the same way.. */
 234         xge_hal_ring_rxd_1_t *rxdp = (xge_hal_ring_rxd_1_t *)dtrh;
 235
 236         ext_info->l3_cksum = XGE_HAL_RXD_GET_L3_CKSUM(rxdp->control_1);
 237         ext_info->l4_cksum = XGE_HAL_RXD_GET_L4_CKSUM(rxdp->control_1);
 238             ext_info->frame = XGE_HAL_RXD_GET_FRAME_TYPE(rxdp->control_1);
 239             ext_info->proto = XGE_HAL_RXD_GET_FRAME_PROTO(rxdp->control_1);
 240             ext_info->vlan = XGE_HAL_RXD_GET_VLAN_TAG(rxdp->control_2);
 241         /* Herc only, a few extra cycles imposed on Xena and/or
 242          * when RTH is not enabled. Same comment as above. */
 243         ext_info->rth_it_hit = XGE_HAL_RXD_GET_RTH_IT_HIT(rxdp->control_1);
 244         ext_info->rth_spdm_hit =
 245         XGE_HAL_RXD_GET_RTH_SPDM_HIT(rxdp->control_1);
 246         ext_info->rth_hash_type =
 247         XGE_HAL_RXD_GET_RTH_HASH_TYPE(rxdp->control_1);
 248         ext_info->rth_value = (u32)rxdp->buffer0_ptr;
 249 }
 250
 251 /**
 252  * xge_hal_ring_dtr_1b_set - Prepare 1-buffer-mode descriptor.
 253  * @dtrh: Descriptor handle.
 254  * @dma_pointer: DMA address of a single receive buffer this descriptor
 255  *               should carry. Note that by the time
 256  *               xge_hal_ring_dtr_1b_set
 257  *               is called, the receive buffer should be already mapped
 258  *               to the corresponding Xframe device.
 259  * @size: Size of the receive @dma_pointer buffer.
 260  *
 261  * Prepare 1-buffer-mode Rx descriptor for posting
 262  * (via xge_hal_ring_dtr_post()).
 263  *
 264  * This inline helper-function does not return any parameters and always
 265  * succeeds.
 266  *
 267  * See also: xge_hal_ring_dtr_3b_set(), xge_hal_ring_dtr_5b_set().
 268  * Usage: See ex_post_all_rx{}.
 269  */
 270 __HAL_STATIC_RING __HAL_INLINE_RING void
 271 xge_hal_ring_dtr_1b_set(xge_hal_dtr_h dtrh, dma_addr_t dma_pointer, int size)
 272 {
 273         xge_hal_ring_rxd_1_t *rxdp = (xge_hal_ring_rxd_1_t *)dtrh;
 274         rxdp->buffer0_ptr = dma_pointer;
 275         rxdp->control_2 &= (~XGE_HAL_RXD_1_MASK_BUFFER0_SIZE);
 276         rxdp->control_2 |= XGE_HAL_RXD_1_SET_BUFFER0_SIZE(size);
 277
 278         xge_debug_ring(XGE_TRACE, "xge_hal_ring_dtr_1b_set: rxdp %p control_2 %p buffer0_ptr %p",
 279                     (xge_hal_ring_rxd_1_t *)dtrh,
 280                     rxdp->control_2,
 281                     rxdp->buffer0_ptr);
 282 }
 283
 284 /**
 285  * xge_hal_ring_dtr_1b_get - Get data from the completed 1-buf
 286  * descriptor.
 287  * @channelh: Channel handle.
 288  * @dtrh: Descriptor handle.
 289  * @dma_pointer: DMA address of a single receive buffer _this_ descriptor
 290  *               carries. Returned by HAL.
 291  * @pkt_length: Length (in bytes) of the data in the buffer pointed by
 292  *              @dma_pointer. Returned by HAL.
 293  *
 294  * Retrieve protocol data from the completed 1-buffer-mode Rx descriptor.
 295  * This inline helper-function uses completed descriptor to populate receive
 296  * buffer pointer and other "out" parameters. The function always succeeds.
 297  *
 298  * See also: xge_hal_ring_dtr_3b_get(), xge_hal_ring_dtr_5b_get().
 299  * Usage: See ex_rx_compl{}.
 300  */
 301 __HAL_STATIC_RING __HAL_INLINE_RING void
 302 xge_hal_ring_dtr_1b_get(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh,
 303             dma_addr_t *dma_pointer, int *pkt_length)
 304 {
 305         xge_hal_ring_rxd_1_t *rxdp = (xge_hal_ring_rxd_1_t *)dtrh;
 306
 307         *pkt_length = XGE_HAL_RXD_1_GET_BUFFER0_SIZE(rxdp->control_2);
 308         *dma_pointer = rxdp->buffer0_ptr;
 309
 310         ((xge_hal_channel_t *)channelh)->poll_bytes += *pkt_length;
 311 }
 312
 313 /**
 314  * xge_hal_ring_dtr_3b_set - Prepare 3-buffer-mode descriptor.
 315  * @dtrh: Descriptor handle.
 316  * @dma_pointers: Array of DMA addresses. Contains exactly 3 receive buffers
 317  *               _this_ descriptor should carry.
 318  *               Note that by the time xge_hal_ring_dtr_3b_set
 319  *               is called, the receive buffers should be mapped
 320  *               to the corresponding Xframe device.
 321  * @sizes: Array of receive buffer sizes. Contains 3 sizes: one size per
 322  *         buffer from @dma_pointers.
 323  *
 324  * Prepare 3-buffer-mode Rx descriptor for posting (via
 325  * xge_hal_ring_dtr_post()).
 326  * This inline helper-function does not return any parameters and always
 327  * succeeds.
 328  *
 329  * See also: xge_hal_ring_dtr_1b_set(), xge_hal_ring_dtr_5b_set().
 330  */
 331 __HAL_STATIC_RING __HAL_INLINE_RING void
 332 xge_hal_ring_dtr_3b_set(xge_hal_dtr_h dtrh, dma_addr_t dma_pointers[],
 333                 int sizes[])
 334 {
 335         xge_hal_ring_rxd_3_t *rxdp = (xge_hal_ring_rxd_3_t *)dtrh;
 336         rxdp->buffer0_ptr = dma_pointers[0];
 337         rxdp->control_2 &= (~XGE_HAL_RXD_3_MASK_BUFFER0_SIZE);
 338         rxdp->control_2 |= XGE_HAL_RXD_3_SET_BUFFER0_SIZE(sizes[0]);
 339         rxdp->buffer1_ptr = dma_pointers[1];
 340         rxdp->control_2 &= (~XGE_HAL_RXD_3_MASK_BUFFER1_SIZE);
 341         rxdp->control_2 |= XGE_HAL_RXD_3_SET_BUFFER1_SIZE(sizes[1]);
 342         rxdp->buffer2_ptr = dma_pointers[2];
 343         rxdp->control_2 &= (~XGE_HAL_RXD_3_MASK_BUFFER2_SIZE);
 344         rxdp->control_2 |= XGE_HAL_RXD_3_SET_BUFFER2_SIZE(sizes[2]);
 345 }
 346
 347 /**
 348  * xge_hal_ring_dtr_3b_get - Get data from the completed 3-buf
 349  * descriptor.
 350  * @channelh: Channel handle.
 351  * @dtrh: Descriptor handle.
 352  * @dma_pointers: DMA addresses of the 3 receive buffers _this_ descriptor
 353  *                carries. The first two buffers contain ethernet and
 354  *                (IP + transport) headers. The 3rd buffer contains packet
 355  *                data.
 356  *                Returned by HAL.
 357  * @sizes: Array of receive buffer sizes. Contains 3 sizes: one size per
 358  * buffer from @dma_pointers. Returned by HAL.
 359  *
 360  * Retrieve protocol data from the completed 3-buffer-mode Rx descriptor.
 361  * This inline helper-function uses completed descriptor to populate receive
 362  * buffer pointer and other "out" parameters. The function always succeeds.
 363  *
 364  * See also: xge_hal_ring_dtr_3b_get(), xge_hal_ring_dtr_5b_get().
 365  */
 366 __HAL_STATIC_RING __HAL_INLINE_RING void
 367 xge_hal_ring_dtr_3b_get(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh,
 368             dma_addr_t dma_pointers[], int sizes[])
 369 {
 370         xge_hal_ring_rxd_3_t *rxdp = (xge_hal_ring_rxd_3_t *)dtrh;
 371
 372         dma_pointers[0] = rxdp->buffer0_ptr;
 373         sizes[0] = XGE_HAL_RXD_3_GET_BUFFER0_SIZE(rxdp->control_2);
 374
 375         dma_pointers[1] = rxdp->buffer1_ptr;
 376         sizes[1] = XGE_HAL_RXD_3_GET_BUFFER1_SIZE(rxdp->control_2);
 377
 378         dma_pointers[2] = rxdp->buffer2_ptr;
 379         sizes[2] = XGE_HAL_RXD_3_GET_BUFFER2_SIZE(rxdp->control_2);
 380
 381         ((xge_hal_channel_t *)channelh)->poll_bytes += sizes[0] + sizes[1] +
 382             sizes[2];
 383 }
 384
 385 /**
 386  * xge_hal_ring_dtr_5b_set - Prepare 5-buffer-mode descriptor.
 387  * @dtrh: Descriptor handle.
 388  * @dma_pointers: Array of DMA addresses. Contains exactly 5 receive buffers
 389  *               _this_ descriptor should carry.
 390  *               Note that by the time xge_hal_ring_dtr_5b_set
 391  *               is called, the receive buffers should be mapped
 392  *               to the corresponding Xframe device.
 393  * @sizes: Array of receive buffer sizes. Contains 5 sizes: one size per
 394  *         buffer from @dma_pointers.
 395  *
 396  * Prepare 3-buffer-mode Rx descriptor for posting (via
 397  * xge_hal_ring_dtr_post()).
 398  * This inline helper-function does not return any parameters and always
 399  * succeeds.
 400  *
 401  * See also: xge_hal_ring_dtr_1b_set(), xge_hal_ring_dtr_3b_set().
 402  */
 403 __HAL_STATIC_RING __HAL_INLINE_RING void
 404 xge_hal_ring_dtr_5b_set(xge_hal_dtr_h dtrh, dma_addr_t dma_pointers[],
 405                 int sizes[])
 406 {
 407         xge_hal_ring_rxd_5_t *rxdp = (xge_hal_ring_rxd_5_t *)dtrh;
 408         rxdp->buffer0_ptr = dma_pointers[0];
 409         rxdp->control_2 &= (~XGE_HAL_RXD_5_MASK_BUFFER0_SIZE);
 410         rxdp->control_2 |= XGE_HAL_RXD_5_SET_BUFFER0_SIZE(sizes[0]);
 411         rxdp->buffer1_ptr = dma_pointers[1];
 412         rxdp->control_2 &= (~XGE_HAL_RXD_5_MASK_BUFFER1_SIZE);
 413         rxdp->control_2 |= XGE_HAL_RXD_5_SET_BUFFER1_SIZE(sizes[1]);
 414         rxdp->buffer2_ptr = dma_pointers[2];
 415         rxdp->control_2 &= (~XGE_HAL_RXD_5_MASK_BUFFER2_SIZE);
 416         rxdp->control_2 |= XGE_HAL_RXD_5_SET_BUFFER2_SIZE(sizes[2]);
 417         rxdp->buffer3_ptr = dma_pointers[3];
 418         rxdp->control_3 &= (~XGE_HAL_RXD_5_MASK_BUFFER3_SIZE);
 419         rxdp->control_3 |= XGE_HAL_RXD_5_SET_BUFFER3_SIZE(sizes[3]);
 420         rxdp->buffer4_ptr = dma_pointers[4];
 421         rxdp->control_3 &= (~XGE_HAL_RXD_5_MASK_BUFFER4_SIZE);
 422         rxdp->control_3 |= XGE_HAL_RXD_5_SET_BUFFER4_SIZE(sizes[4]);
 423 }
 424
 425 /**
 426  * xge_hal_ring_dtr_5b_get - Get data from the completed 5-buf
 427  * descriptor.
 428  * @channelh: Channel handle.
 429  * @dtrh: Descriptor handle.
 430  * @dma_pointers: DMA addresses of the 5 receive buffers _this_ descriptor
 431  *                carries. The first 4 buffers contains L2 (ethernet) through
 432  *                L5 headers. The 5th buffer contain received (applicaion)
 433  *                data. Returned by HAL.
 434  * @sizes: Array of receive buffer sizes. Contains 5 sizes: one size per
 435  * buffer from @dma_pointers. Returned by HAL.
 436  *
 437  * Retrieve protocol data from the completed 5-buffer-mode Rx descriptor.
 438  * This inline helper-function uses completed descriptor to populate receive
 439  * buffer pointer and other "out" parameters. The function always succeeds.
 440  *
 441  * See also: xge_hal_ring_dtr_3b_get(), xge_hal_ring_dtr_5b_get().
 442  */
 443 __HAL_STATIC_RING __HAL_INLINE_RING void
 444 xge_hal_ring_dtr_5b_get(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh,
 445             dma_addr_t dma_pointers[], int sizes[])
 446 {
 447         xge_hal_ring_rxd_5_t *rxdp = (xge_hal_ring_rxd_5_t *)dtrh;
 448
 449         dma_pointers[0] = rxdp->buffer0_ptr;
 450         sizes[0] = XGE_HAL_RXD_5_GET_BUFFER0_SIZE(rxdp->control_2);
 451
 452         dma_pointers[1] = rxdp->buffer1_ptr;
 453         sizes[1] = XGE_HAL_RXD_5_GET_BUFFER1_SIZE(rxdp->control_2);
 454
 455         dma_pointers[2] = rxdp->buffer2_ptr;
 456         sizes[2] = XGE_HAL_RXD_5_GET_BUFFER2_SIZE(rxdp->control_2);
 457
 458         dma_pointers[3] = rxdp->buffer3_ptr;
 459         sizes[3] = XGE_HAL_RXD_5_GET_BUFFER3_SIZE(rxdp->control_3);
 460
 461         dma_pointers[4] = rxdp->buffer4_ptr;
 462         sizes[4] = XGE_HAL_RXD_5_GET_BUFFER4_SIZE(rxdp->control_3);
 463
 464         ((xge_hal_channel_t *)channelh)->poll_bytes += sizes[0] + sizes[1] +
 465             sizes[2] + sizes[3] + sizes[4];
 466 }
 467
 468
 469 /**
 470  * xge_hal_ring_dtr_pre_post - FIXME.
 471  * @channelh: Channel handle.
 472  * @dtrh: Descriptor handle.
 473  *
 474  * TBD
 475  */
 476 __HAL_STATIC_RING __HAL_INLINE_RING void
 477 xge_hal_ring_dtr_pre_post(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh)
 478 {
 479         xge_hal_ring_rxd_1_t *rxdp = (xge_hal_ring_rxd_1_t *)dtrh;
 480 #if defined(XGE_OS_DMA_REQUIRES_SYNC) && defined(XGE_HAL_DMA_DTR_STREAMING)
 481         xge_hal_ring_rxd_priv_t *priv;
 482         xge_hal_ring_t *ring = (xge_hal_ring_t *)channelh;
 483 #endif
 484 #if defined(XGE_HAL_RX_MULTI_POST_IRQ)
 485         unsigned long flags;
 486 #endif
 487
 488         rxdp->control_2 |= XGE_HAL_RXD_NOT_COMPLETED;
 489
 490 #ifdef XGE_DEBUG_ASSERT
 491             /* make sure Xena overwrites the (illegal) t_code on completion */
 492             XGE_HAL_RXD_SET_T_CODE(rxdp->control_1, XGE_HAL_RXD_T_CODE_UNUSED_C);
 493 #endif
 494
 495         xge_debug_ring(XGE_TRACE, "xge_hal_ring_dtr_pre_post: rxd 0x"XGE_OS_LLXFMT" posted %d  post_qid %d",
 496                 (unsigned long long)(ulong_t)dtrh,
 497                 ((xge_hal_ring_t *)channelh)->channel.post_index,
 498                 ((xge_hal_ring_t *)channelh)->channel.post_qid);
 499
 500 #if defined(XGE_HAL_RX_MULTI_POST)
 501         xge_os_spin_lock(&((xge_hal_channel_t*)channelh)->post_lock);
 502 #elif defined(XGE_HAL_RX_MULTI_POST_IRQ)
 503         xge_os_spin_lock_irq(&((xge_hal_channel_t*)channelh)->post_lock,
 504         flags);
 505 #endif
 506
 507 #if defined(XGE_DEBUG_ASSERT) && defined(XGE_HAL_RING_ENFORCE_ORDER)
 508         {
 509             xge_hal_channel_t *channel = (xge_hal_channel_t *)channelh;
 510
 511             if (channel->post_index != 0) {
 512                 xge_hal_dtr_h prev_dtrh;
 513                 xge_hal_ring_rxd_priv_t *rxdp_priv;
 514
 515                 rxdp_priv = __hal_ring_rxd_priv((xge_hal_ring_t*)channel, rxdp);
 516                 prev_dtrh = channel->work_arr[channel->post_index - 1];
 517
 518                 if (prev_dtrh != NULL &&
 519                     (rxdp_priv->dma_offset & (~0xFFF)) !=
 520                             rxdp_priv->dma_offset) {
 521                     xge_assert((char *)prev_dtrh +
 522                         ((xge_hal_ring_t*)channel)->rxd_size == dtrh);
 523                 }
 524             }
 525         }
 526 #endif
 527
 528         __hal_channel_dtr_post(channelh, dtrh);
 529
 530 #if defined(XGE_HAL_RX_MULTI_POST)
 531         xge_os_spin_unlock(&((xge_hal_channel_t*)channelh)->post_lock);
 532 #elif defined(XGE_HAL_RX_MULTI_POST_IRQ)
 533         xge_os_spin_unlock_irq(&((xge_hal_channel_t*)channelh)->post_lock,
 534                        flags);
 535 #endif
 536 }
 537
 538
 539 /**
 540  * xge_hal_ring_dtr_post_post - FIXME.
 541  * @channelh: Channel handle.
 542  * @dtrh: Descriptor handle.
 543  *
 544  * TBD
 545  */
 546 __HAL_STATIC_RING __HAL_INLINE_RING void
 547 xge_hal_ring_dtr_post_post(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh)
 548 {
 549         xge_hal_ring_rxd_1_t *rxdp = (xge_hal_ring_rxd_1_t *)dtrh;
 550         xge_hal_ring_t *ring = (xge_hal_ring_t *)channelh;
 551 #if defined(XGE_OS_DMA_REQUIRES_SYNC) && defined(XGE_HAL_DMA_DTR_STREAMING)
 552         xge_hal_ring_rxd_priv_t *priv;
 553 #endif
 554         /* do POST */
 555         rxdp->control_1 |= XGE_HAL_RXD_POSTED_4_XFRAME;
 556
 557 #if defined(XGE_OS_DMA_REQUIRES_SYNC) && defined(XGE_HAL_DMA_DTR_STREAMING)
 558         priv = __hal_ring_rxd_priv(ring, rxdp);
 559         xge_os_dma_sync(ring->channel.pdev,
 560                       priv->dma_handle, priv->dma_addr,
 561                   priv->dma_offset, ring->rxd_size,
 562                   XGE_OS_DMA_DIR_TODEVICE);
 563 #endif
 564
 565         xge_debug_ring(XGE_TRACE, "xge_hal_ring_dtr_post_post: rxdp %p control_1 %p",
 566                       (xge_hal_ring_rxd_1_t *)dtrh,
 567                       rxdp->control_1);
 568
 569         if (ring->channel.usage_cnt > 0)
 570             ring->channel.usage_cnt--;
 571 }
 572
 573 /**
 574  * xge_hal_ring_dtr_post_post_wmb.
 575  * @channelh: Channel handle.
 576  * @dtrh: Descriptor handle.
 577  *
 578  * Similar as xge_hal_ring_dtr_post_post, but in addition it does memory barrier.
 579  */
 580 __HAL_STATIC_RING __HAL_INLINE_RING void
 581 xge_hal_ring_dtr_post_post_wmb(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh)
 582 {
 583         xge_hal_ring_rxd_1_t *rxdp = (xge_hal_ring_rxd_1_t *)dtrh;
 584         xge_hal_ring_t *ring = (xge_hal_ring_t *)channelh;
 585 #if defined(XGE_OS_DMA_REQUIRES_SYNC) && defined(XGE_HAL_DMA_DTR_STREAMING)
 586         xge_hal_ring_rxd_priv_t *priv;
 587 #endif
 588         /* Do memory barrier before changing the ownership */
 589         xge_os_wmb();
 590
 591         /* do POST */
 592         rxdp->control_1 |= XGE_HAL_RXD_POSTED_4_XFRAME;
 593
 594 #if defined(XGE_OS_DMA_REQUIRES_SYNC) && defined(XGE_HAL_DMA_DTR_STREAMING)
 595         priv = __hal_ring_rxd_priv(ring, rxdp);
 596         xge_os_dma_sync(ring->channel.pdev,
 597                       priv->dma_handle, priv->dma_addr,
 598                   priv->dma_offset, ring->rxd_size,
 599                   XGE_OS_DMA_DIR_TODEVICE);
 600 #endif
 601
 602         if (ring->channel.usage_cnt > 0)
 603             ring->channel.usage_cnt--;
 604
 605         xge_debug_ring(XGE_TRACE, "xge_hal_ring_dtr_post_post_wmb: rxdp %p control_1 %p rxds_with_host %d",
 606                       (xge_hal_ring_rxd_1_t *)dtrh,
 607                       rxdp->control_1, ring->channel.usage_cnt);
 608
 609 }
 610
 611 /**
 612  * xge_hal_ring_dtr_post - Post descriptor on the ring channel.
 613  * @channelh: Channel handle.
 614  * @dtrh: Descriptor obtained via xge_hal_ring_dtr_reserve().
 615  *
 616  * Post descriptor on the 'ring' type channel.
 617  * Prior to posting the descriptor should be filled in accordance with
 618  * Host/Xframe interface specification for a given service (LL, etc.).
 619  *
 620  * See also: xge_hal_fifo_dtr_post_many(), xge_hal_fifo_dtr_post().
 621  * Usage: See ex_post_all_rx{}.
 622  */
 623 __HAL_STATIC_RING __HAL_INLINE_RING void
 624 xge_hal_ring_dtr_post(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh)
 625 {
 626         xge_hal_ring_dtr_pre_post(channelh, dtrh);
 627         xge_hal_ring_dtr_post_post(channelh, dtrh);
 628 }
 629
 630 /**
 631  * xge_hal_ring_dtr_next_completed - Get the _next_ completed
 632  * descriptor.
 633  * @channelh: Channel handle.
 634  * @dtrh: Descriptor handle. Returned by HAL.
 635  * @t_code: Transfer code, as per Xframe User Guide,
 636  *          Receive Descriptor Format. Returned by HAL.
 637  *
 638  * Retrieve the _next_ completed descriptor.
 639  * HAL uses channel callback (*xge_hal_channel_callback_f) to notifiy
 640  * upper-layer driver (ULD) of new completed descriptors. After that
 641  * the ULD can use xge_hal_ring_dtr_next_completed to retrieve the rest
 642  * completions (the very first completion is passed by HAL via
 643  * xge_hal_channel_callback_f).
 644  *
 645  * Implementation-wise, the upper-layer driver is free to call
 646  * xge_hal_ring_dtr_next_completed either immediately from inside the
 647  * channel callback, or in a deferred fashion and separate (from HAL)
 648  * context.
 649  *
 650  * Non-zero @t_code means failure to fill-in receive buffer(s)
 651  * of the descriptor.
 652  * For instance, parity error detected during the data transfer.
 653  * In this case Xframe will complete the descriptor and indicate
 654  * for the host that the received data is not to be used.
 655  * For details please refer to Xframe User Guide.
 656  *
 657  * Returns: XGE_HAL_OK - success.
 658  * XGE_HAL_INF_NO_MORE_COMPLETED_DESCRIPTORS - No completed descriptors
 659  * are currently available for processing.
 660  *
 661  * See also: xge_hal_channel_callback_f{},
 662  * xge_hal_fifo_dtr_next_completed(), xge_hal_status_e{}.
 663  * Usage: See ex_rx_compl{}.
 664  */
 665 __HAL_STATIC_RING __HAL_INLINE_RING xge_hal_status_e
 666 xge_hal_ring_dtr_next_completed(xge_hal_channel_h channelh, xge_hal_dtr_h *dtrh,
 667                     u8 *t_code)
 668 {
 669         xge_hal_ring_rxd_1_t *rxdp; /* doesn't matter 1, 3 or 5... */
 670         xge_hal_ring_t *ring = (xge_hal_ring_t *)channelh;
 671 #if defined(XGE_OS_DMA_REQUIRES_SYNC) && defined(XGE_HAL_DMA_DTR_STREAMING)
 672         xge_hal_ring_rxd_priv_t *priv;
 673 #endif
 674
 675         __hal_channel_dtr_try_complete(ring, dtrh);
 676         rxdp = (xge_hal_ring_rxd_1_t *)*dtrh;
 677         if (rxdp == NULL) {
 678             return XGE_HAL_INF_NO_MORE_COMPLETED_DESCRIPTORS;
 679         }
 680
 681 #if defined(XGE_OS_DMA_REQUIRES_SYNC) && defined(XGE_HAL_DMA_DTR_STREAMING)
 682         /* Note: 24 bytes at most means:
 683          *  - Control_3 in case of 5-buffer mode
 684          *  - Control_1 and Control_2
 685          *
 686          * This is the only length needs to be invalidated
 687          * type of channels.*/
 688         priv = __hal_ring_rxd_priv(ring, rxdp);
 689         xge_os_dma_sync(ring->channel.pdev,
 690                       priv->dma_handle, priv->dma_addr,
 691                   priv->dma_offset, 24,
 692                   XGE_OS_DMA_DIR_FROMDEVICE);
 693 #endif
 694
 695         /* check whether it is not the end */
 696         if (!(rxdp->control_2 & XGE_HAL_RXD_NOT_COMPLETED) &&
 697             !(rxdp->control_1 & XGE_HAL_RXD_POSTED_4_XFRAME)) {
 698 #ifndef XGE_HAL_IRQ_POLLING
 699             if (++ring->cmpl_cnt > ring->indicate_max_pkts) {
 700                 /* reset it. since we don't want to return
 701                  * garbage to the ULD */
 702                 *dtrh = 0;
 703                 return XGE_HAL_COMPLETIONS_REMAIN;
 704             }
 705 #endif
 706
 707 #ifdef XGE_DEBUG_ASSERT
 708 #if defined(XGE_HAL_USE_5B_MODE)
 709 #if !defined(XGE_OS_PLATFORM_64BIT)
 710             if (ring->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5) {
 711                 xge_assert(((xge_hal_ring_rxd_5_t *)
 712                         rxdp)->host_control!=0);
 713             }
 714 #endif
 715
 716 #else
 717             xge_assert(rxdp->host_control!=0);
 718 #endif
 719 #endif
 720
 721             __hal_channel_dtr_complete(ring);
 722
 723             *t_code = (u8)XGE_HAL_RXD_GET_T_CODE(rxdp->control_1);
 724
 725                     /* see XGE_HAL_SET_RXD_T_CODE() above.. */
 726             xge_assert(*t_code != XGE_HAL_RXD_T_CODE_UNUSED_C);
 727
 728             xge_debug_ring(XGE_TRACE,
 729                 "compl_index %d post_qid %d t_code %d rxd 0x"XGE_OS_LLXFMT,
 730                 ((xge_hal_channel_t*)ring)->compl_index,
 731                 ((xge_hal_channel_t*)ring)->post_qid, *t_code,
 732                 (unsigned long long)(ulong_t)rxdp);
 733
 734             ring->channel.usage_cnt++;
 735             if (ring->channel.stats.usage_max < ring->channel.usage_cnt)
 736                 ring->channel.stats.usage_max = ring->channel.usage_cnt;
 737
 738             return XGE_HAL_OK;
 739         }
 740
 741         /* reset it. since we don't want to return
 742          * garbage to the ULD */
 743         *dtrh = 0;
 744         return XGE_HAL_INF_NO_MORE_COMPLETED_DESCRIPTORS;
 745 }
 746
 747 /**
 748  * xge_hal_ring_dtr_free - Free descriptor.
 749  * @channelh: Channel handle.
 750  * @dtrh: Descriptor handle.
 751  *
 752  * Free the reserved descriptor. This operation is "symmetrical" to
 753  * xge_hal_ring_dtr_reserve. The "free-ing" completes the descriptor's
 754  * lifecycle.
 755  *
 756  * After free-ing (see xge_hal_ring_dtr_free()) the descriptor again can
 757  * be:
 758  *
 759  * - reserved (xge_hal_ring_dtr_reserve);
 760  *
 761  * - posted (xge_hal_ring_dtr_post);
 762  *
 763  * - completed (xge_hal_ring_dtr_next_completed);
 764  *
 765  * - and recycled again (xge_hal_ring_dtr_free).
 766  *
 767  * For alternative state transitions and more details please refer to
 768  * the design doc.
 769  *
 770  * See also: xge_hal_ring_dtr_reserve(), xge_hal_fifo_dtr_free().
 771  * Usage: See ex_rx_compl{}.
 772  */
 773 __HAL_STATIC_RING __HAL_INLINE_RING void
 774 xge_hal_ring_dtr_free(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh)
 775 {
 776 #if defined(XGE_HAL_RX_MULTI_FREE_IRQ)
 777         unsigned long flags;
 778 #endif
 779
 780 #if defined(XGE_HAL_RX_MULTI_FREE)
 781         xge_os_spin_lock(&((xge_hal_channel_t*)channelh)->free_lock);
 782 #elif defined(XGE_HAL_RX_MULTI_FREE_IRQ)
 783         xge_os_spin_lock_irq(&((xge_hal_channel_t*)channelh)->free_lock,
 784         flags);
 785 #endif
 786
 787         __hal_channel_dtr_free(channelh, dtrh);
 788 #if defined(XGE_OS_MEMORY_CHECK)
 789         __hal_ring_rxd_priv((xge_hal_ring_t * ) channelh, dtrh)->allocated = 0;
 790 #endif
 791
 792 #if defined(XGE_HAL_RX_MULTI_FREE)
 793         xge_os_spin_unlock(&((xge_hal_channel_t*)channelh)->free_lock);
 794 #elif defined(XGE_HAL_RX_MULTI_FREE_IRQ)
 795         xge_os_spin_unlock_irq(&((xge_hal_channel_t*)channelh)->free_lock,
 796         flags);
 797 #endif
 798 }
 799
 800 /**
 801  * xge_hal_ring_is_next_dtr_completed - Check if the next dtr is completed
 802  * @channelh: Channel handle.
 803  *
 804  * Checks if the _next_ completed descriptor is in host memory
 805  *
 806  * Returns: XGE_HAL_OK - success.
 807  * XGE_HAL_INF_NO_MORE_COMPLETED_DESCRIPTORS - No completed descriptors
 808  * are currently available for processing.
 809  */
 810 __HAL_STATIC_RING __HAL_INLINE_RING xge_hal_status_e
 811 xge_hal_ring_is_next_dtr_completed(xge_hal_channel_h channelh)
 812 {
 813         xge_hal_ring_rxd_1_t *rxdp; /* doesn't matter 1, 3 or 5... */
 814         xge_hal_ring_t *ring = (xge_hal_ring_t *)channelh;
 815         xge_hal_dtr_h dtrh;
 816
 817         __hal_channel_dtr_try_complete(ring, &dtrh);
 818         rxdp = (xge_hal_ring_rxd_1_t *)dtrh;
 819         if (rxdp == NULL) {
 820             return XGE_HAL_INF_NO_MORE_COMPLETED_DESCRIPTORS;
 821         }
 822
 823         /* check whether it is not the end */
 824         if (!(rxdp->control_2 & XGE_HAL_RXD_NOT_COMPLETED) &&
 825             !(rxdp->control_1 & XGE_HAL_RXD_POSTED_4_XFRAME)) {
 826
 827 #ifdef XGE_DEBUG_ASSERT
 828 #if defined(XGE_HAL_USE_5B_MODE)
 829 #if !defined(XGE_OS_PLATFORM_64BIT)
 830             if (ring->buffer_mode == XGE_HAL_RING_QUEUE_BUFFER_MODE_5) {
 831                 xge_assert(((xge_hal_ring_rxd_5_t *)
 832                         rxdp)->host_control!=0);
 833             }
 834 #endif
 835
 836 #else
 837             xge_assert(rxdp->host_control!=0);
 838 #endif
 839 #endif
 840             return XGE_HAL_OK;
 841         }
 842
 843         return XGE_HAL_INF_NO_MORE_COMPLETED_DESCRIPTORS;
 844 }