2 * Copyright (c) 2012-2015 Solarflare Communications Inc.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 * The views and conclusions contained in the software and documentation are
27 * those of the authors and should not be interpreted as representing official
28 * policies, either expressed or implied, of the FreeBSD Project.
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
38 #if EFSYS_OPT_HUNTINGTON
41 static __checkReturn efx_rc_t
45 __in uint32_t target_evq,
47 __in uint32_t instance,
48 __in efsys_mem_t *esmp,
49 __in boolean_t disable_scatter)
53 MAX(MC_CMD_INIT_RXQ_IN_LEN(EFX_RXQ_NBUFS(EFX_RXQ_MAXNDESCS)),
54 MC_CMD_INIT_RXQ_OUT_LEN)];
55 int npages = EFX_RXQ_NBUFS(size);
57 efx_qword_t *dma_addr;
61 EFSYS_ASSERT3U(size, <=, EFX_RXQ_MAXNDESCS);
63 (void) memset(payload, 0, sizeof (payload));
64 req.emr_cmd = MC_CMD_INIT_RXQ;
65 req.emr_in_buf = payload;
66 req.emr_in_length = MC_CMD_INIT_RXQ_IN_LEN(npages);
67 req.emr_out_buf = payload;
68 req.emr_out_length = MC_CMD_INIT_RXQ_OUT_LEN;
70 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_SIZE, size);
71 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_TARGET_EVQ, target_evq);
72 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_LABEL, label);
73 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_INSTANCE, instance);
74 MCDI_IN_POPULATE_DWORD_6(req, INIT_RXQ_IN_FLAGS,
75 INIT_RXQ_IN_FLAG_BUFF_MODE, 0,
76 INIT_RXQ_IN_FLAG_HDR_SPLIT, 0,
77 INIT_RXQ_IN_FLAG_TIMESTAMP, 0,
78 INIT_RXQ_IN_CRC_MODE, 0,
79 INIT_RXQ_IN_FLAG_PREFIX, 1,
80 INIT_RXQ_IN_FLAG_DISABLE_SCATTER, disable_scatter);
81 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_OWNER_ID, 0);
82 MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_PORT_ID, EVB_PORT_ID_ASSIGNED);
84 dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR);
85 addr = EFSYS_MEM_ADDR(esmp);
87 for (i = 0; i < npages; i++) {
88 EFX_POPULATE_QWORD_2(*dma_addr,
89 EFX_DWORD_1, (uint32_t)(addr >> 32),
90 EFX_DWORD_0, (uint32_t)(addr & 0xffffffff));
96 efx_mcdi_execute(enp, &req);
98 if (req.emr_rc != 0) {
106 EFSYS_PROBE1(fail1, efx_rc_t, rc);
111 static __checkReturn efx_rc_t
114 __in uint32_t instance)
117 uint8_t payload[MAX(MC_CMD_FINI_RXQ_IN_LEN,
118 MC_CMD_FINI_RXQ_OUT_LEN)];
121 (void) memset(payload, 0, sizeof (payload));
122 req.emr_cmd = MC_CMD_FINI_RXQ;
123 req.emr_in_buf = payload;
124 req.emr_in_length = MC_CMD_FINI_RXQ_IN_LEN;
125 req.emr_out_buf = payload;
126 req.emr_out_length = MC_CMD_FINI_RXQ_OUT_LEN;
128 MCDI_IN_SET_DWORD(req, FINI_RXQ_IN_INSTANCE, instance);
130 efx_mcdi_execute(enp, &req);
132 if ((req.emr_rc != 0) && (req.emr_rc != MC_CMD_ERR_EALREADY)) {
140 EFSYS_PROBE1(fail1, efx_rc_t, rc);
145 #if EFSYS_OPT_RX_SCALE
146 static __checkReturn efx_rc_t
147 efx_mcdi_rss_context_alloc(
149 __in efx_rx_scale_support_t scale_support,
150 __in uint32_t num_queues,
151 __out uint32_t *rss_contextp)
154 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN,
155 MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN)];
156 uint32_t rss_context;
157 uint32_t context_type;
160 if (num_queues > EFX_MAXRSS) {
165 switch (scale_support) {
166 case EFX_RX_SCALE_EXCLUSIVE:
167 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE;
169 case EFX_RX_SCALE_SHARED:
170 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED;
177 (void) memset(payload, 0, sizeof (payload));
178 req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC;
179 req.emr_in_buf = payload;
180 req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN;
181 req.emr_out_buf = payload;
182 req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN;
184 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID,
185 EVB_PORT_ID_ASSIGNED);
186 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type);
187 /* NUM_QUEUES is only used to validate indirection table offsets */
188 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues);
190 efx_mcdi_execute(enp, &req);
192 if (req.emr_rc != 0) {
197 if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) {
202 rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID);
203 if (rss_context == EF10_RSS_CONTEXT_INVALID) {
208 *rss_contextp = rss_context;
221 EFSYS_PROBE1(fail1, efx_rc_t, rc);
225 #endif /* EFSYS_OPT_RX_SCALE */
227 #if EFSYS_OPT_RX_SCALE
229 efx_mcdi_rss_context_free(
231 __in uint32_t rss_context)
234 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_FREE_IN_LEN,
235 MC_CMD_RSS_CONTEXT_FREE_OUT_LEN)];
238 if (rss_context == EF10_RSS_CONTEXT_INVALID) {
243 (void) memset(payload, 0, sizeof (payload));
244 req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE;
245 req.emr_in_buf = payload;
246 req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN;
247 req.emr_out_buf = payload;
248 req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN;
250 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context);
252 efx_mcdi_execute(enp, &req);
254 if (req.emr_rc != 0) {
264 EFSYS_PROBE1(fail1, efx_rc_t, rc);
268 #endif /* EFSYS_OPT_RX_SCALE */
270 #if EFSYS_OPT_RX_SCALE
272 efx_mcdi_rss_context_set_flags(
274 __in uint32_t rss_context,
275 __in efx_rx_hash_type_t type)
278 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN,
279 MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN)];
282 if (rss_context == EF10_RSS_CONTEXT_INVALID) {
287 (void) memset(payload, 0, sizeof (payload));
288 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS;
289 req.emr_in_buf = payload;
290 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN;
291 req.emr_out_buf = payload;
292 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN;
294 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID,
297 MCDI_IN_POPULATE_DWORD_4(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS,
298 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN,
299 (type & (1U << EFX_RX_HASH_IPV4)) ? 1 : 0,
300 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN,
301 (type & (1U << EFX_RX_HASH_TCPIPV4)) ? 1 : 0,
302 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN,
303 (type & (1U << EFX_RX_HASH_IPV6)) ? 1 : 0,
304 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN,
305 (type & (1U << EFX_RX_HASH_TCPIPV6)) ? 1 : 0);
307 efx_mcdi_execute(enp, &req);
309 if (req.emr_rc != 0) {
319 EFSYS_PROBE1(fail1, efx_rc_t, rc);
323 #endif /* EFSYS_OPT_RX_SCALE */
325 #if EFSYS_OPT_RX_SCALE
327 efx_mcdi_rss_context_set_key(
329 __in uint32_t rss_context,
330 __in_ecount(n) uint8_t *key,
334 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN,
335 MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN)];
338 if (rss_context == EF10_RSS_CONTEXT_INVALID) {
343 (void) memset(payload, 0, sizeof (payload));
344 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY;
345 req.emr_in_buf = payload;
346 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN;
347 req.emr_out_buf = payload;
348 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN;
350 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID,
353 EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN);
354 if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) {
359 memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY),
362 efx_mcdi_execute(enp, &req);
364 if (req.emr_rc != 0) {
376 EFSYS_PROBE1(fail1, efx_rc_t, rc);
380 #endif /* EFSYS_OPT_RX_SCALE */
382 #if EFSYS_OPT_RX_SCALE
384 efx_mcdi_rss_context_set_table(
386 __in uint32_t rss_context,
387 __in_ecount(n) unsigned int *table,
391 uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN,
392 MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN)];
396 if (rss_context == EF10_RSS_CONTEXT_INVALID) {
401 (void) memset(payload, 0, sizeof (payload));
402 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE;
403 req.emr_in_buf = payload;
404 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN;
405 req.emr_out_buf = payload;
406 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN;
408 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID,
412 MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE);
415 i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN;
417 req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0;
420 efx_mcdi_execute(enp, &req);
422 if (req.emr_rc != 0) {
432 EFSYS_PROBE1(fail1, efx_rc_t, rc);
436 #endif /* EFSYS_OPT_RX_SCALE */
439 __checkReturn efx_rc_t
443 #if EFSYS_OPT_RX_SCALE
445 if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS,
446 &enp->en_rss_context) == 0) {
448 * Allocated an exclusive RSS context, which allows both the
449 * indirection table and key to be modified.
451 enp->en_rss_support = EFX_RX_SCALE_EXCLUSIVE;
452 enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
455 * Failed to allocate an exclusive RSS context. Continue
456 * operation without support for RSS. The pseudo-header in
457 * received packets will not contain a Toeplitz hash value.
459 enp->en_rss_support = EFX_RX_SCALE_UNAVAILABLE;
460 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE;
463 #endif /* EFSYS_OPT_RX_SCALE */
468 #if EFSYS_OPT_RX_SCATTER
469 __checkReturn efx_rc_t
470 ef10_rx_scatter_enable(
472 __in unsigned int buf_size)
474 _NOTE(ARGUNUSED(enp, buf_size))
477 #endif /* EFSYS_OPT_RX_SCATTER */
479 #if EFSYS_OPT_RX_SCALE
480 __checkReturn efx_rc_t
481 ef10_rx_scale_mode_set(
483 __in efx_rx_hash_alg_t alg,
484 __in efx_rx_hash_type_t type,
485 __in boolean_t insert)
489 EFSYS_ASSERT3U(alg, ==, EFX_RX_HASHALG_TOEPLITZ);
490 EFSYS_ASSERT3U(insert, ==, B_TRUE);
492 if ((alg != EFX_RX_HASHALG_TOEPLITZ) || (insert == B_FALSE)) {
497 if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) {
502 if ((rc = efx_mcdi_rss_context_set_flags(enp,
503 enp->en_rss_context, type)) != 0)
513 EFSYS_PROBE1(fail1, efx_rc_t, rc);
517 #endif /* EFSYS_OPT_RX_SCALE */
519 #if EFSYS_OPT_RX_SCALE
520 __checkReturn efx_rc_t
521 ef10_rx_scale_key_set(
523 __in_ecount(n) uint8_t *key,
528 if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) {
533 if ((rc = efx_mcdi_rss_context_set_key(enp,
534 enp->en_rss_context, key, n)) != 0)
542 EFSYS_PROBE1(fail1, efx_rc_t, rc);
546 #endif /* EFSYS_OPT_RX_SCALE */
548 #if EFSYS_OPT_RX_SCALE
549 __checkReturn efx_rc_t
550 ef10_rx_scale_tbl_set(
552 __in_ecount(n) unsigned int *table,
557 if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) {
562 if ((rc = efx_mcdi_rss_context_set_table(enp,
563 enp->en_rss_context, table, n)) != 0)
571 EFSYS_PROBE1(fail1, efx_rc_t, rc);
575 #endif /* EFSYS_OPT_RX_SCALE */
579 * EF10 RX pseudo-header
580 * ---------------------
582 * Receive packets are prefixed by an (optional) 14 byte pseudo-header:
584 * +00: Toeplitz hash value.
585 * (32bit little-endian)
586 * +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag.
588 * +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag.
590 * +08: Packet Length. Zero if the RX datapath was in cut-through mode.
591 * (16bit little-endian)
592 * +10: MAC timestamp. Zero if timestamping is not enabled.
593 * (32bit little-endian)
595 * See "The RX Pseudo-header" in SF-109306-TC.
598 __checkReturn efx_rc_t
599 ef10_rx_prefix_pktlen(
601 __in uint8_t *buffer,
602 __out uint16_t *lengthp)
605 * The RX pseudo-header contains the packet length, excluding the
606 * pseudo-header. If the hardware receive datapath was operating in
607 * cut-through mode then the length in the RX pseudo-header will be
608 * zero, and the packet length must be obtained from the DMA length
609 * reported in the RX event.
611 *lengthp = buffer[8] | (buffer[9] << 8);
615 #if EFSYS_OPT_RX_SCALE
616 __checkReturn uint32_t
619 __in efx_rx_hash_alg_t func,
620 __in uint8_t *buffer)
623 case EFX_RX_HASHALG_TOEPLITZ:
634 #endif /* EFSYS_OPT_RX_SCALE */
639 __in_ecount(n) efsys_dma_addr_t *addrp,
642 __in unsigned int completed,
643 __in unsigned int added)
650 /* The client driver must not overfill the queue */
651 EFSYS_ASSERT3U(added - completed + n, <=,
652 EFX_RXQ_LIMIT(erp->er_mask + 1));
654 id = added & (erp->er_mask);
655 for (i = 0; i < n; i++) {
656 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
657 unsigned int, id, efsys_dma_addr_t, addrp[i],
660 EFX_POPULATE_QWORD_3(qword,
661 ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size),
662 ESF_DZ_RX_KER_BUF_ADDR_DW0,
663 (uint32_t)(addrp[i] & 0xffffffff),
664 ESF_DZ_RX_KER_BUF_ADDR_DW1,
665 (uint32_t)(addrp[i] >> 32));
667 offset = id * sizeof (efx_qword_t);
668 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
670 id = (id + 1) & (erp->er_mask);
677 __in unsigned int added,
678 __inout unsigned int *pushedp)
680 efx_nic_t *enp = erp->er_enp;
681 unsigned int pushed = *pushedp;
685 /* Hardware has alignment restriction for WPTR */
686 wptr = P2ALIGN(added, EF10_RX_WPTR_ALIGN);
692 /* Push the populated descriptors out */
693 wptr &= erp->er_mask;
695 EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr);
697 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
698 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
699 wptr, pushed & erp->er_mask);
700 EFSYS_PIO_WRITE_BARRIER();
701 EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
702 erp->er_index, &dword, B_FALSE);
705 __checkReturn efx_rc_t
709 efx_nic_t *enp = erp->er_enp;
712 if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0)
718 EFSYS_PROBE1(fail1, efx_rc_t, rc);
728 _NOTE(ARGUNUSED(erp))
732 __checkReturn efx_rc_t
735 __in unsigned int index,
736 __in unsigned int label,
737 __in efx_rxq_type_t type,
738 __in efsys_mem_t *esmp,
744 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
746 boolean_t disable_scatter;
748 _NOTE(ARGUNUSED(erp))
750 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH));
751 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
752 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
754 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MAXNDESCS));
755 EFX_STATIC_ASSERT(ISP2(EFX_RXQ_MINNDESCS));
757 if (!ISP2(n) || (n < EFX_RXQ_MINNDESCS) || (n > EFX_RXQ_MAXNDESCS)) {
761 if (index >= encp->enc_rxq_limit) {
766 /* Scatter can only be disabled if the firmware supports doing so */
767 if ((type != EFX_RXQ_TYPE_SCATTER) &&
768 enp->en_nic_cfg.enc_rx_disable_scatter_supported) {
769 disable_scatter = B_TRUE;
771 disable_scatter = B_FALSE;
774 if ((rc = efx_mcdi_init_rxq(enp, n, eep->ee_index, label, index,
775 esmp, disable_scatter)) != 0)
779 erp->er_label = label;
781 ef10_ev_rxlabel_init(eep, erp, label);
790 EFSYS_PROBE1(fail1, efx_rc_t, rc);
799 efx_nic_t *enp = erp->er_enp;
800 efx_evq_t *eep = erp->er_eep;
801 unsigned int label = erp->er_label;
803 ef10_ev_rxlabel_fini(eep, label);
805 EFSYS_ASSERT(enp->en_rx_qcount != 0);
808 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
815 #if EFSYS_OPT_RX_SCALE
816 if (enp->en_rss_support != EFX_RX_SCALE_UNAVAILABLE) {
817 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context);
819 enp->en_rss_context = 0;
820 enp->en_rss_support = EFX_RX_SCALE_UNAVAILABLE;
822 _NOTE(ARGUNUSED(enp))
823 #endif /* EFSYS_OPT_RX_SCALE */
826 #endif /* EFSYS_OPT_HUNTINGTON */