2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 2007-2016 Solarflare Communications Inc.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions are met:
10 * 1. Redistributions of source code must retain the above copyright notice,
11 * this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright notice,
13 * this list of conditions and the following disclaimer in the documentation
14 * and/or other materials provided with the distribution.
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
18 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
20 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
21 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
22 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
23 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
25 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
26 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 * The views and conclusions contained in the software and documentation are
29 * those of the authors and should not be interpreted as representing official
30 * policies, either expressed or implied, of the FreeBSD Project.
33 #include <sys/cdefs.h>
34 __FBSDID("$FreeBSD$");
40 #define EFX_TX_QSTAT_INCR(_etp, _stat) \
42 (_etp)->et_stat[_stat]++; \
43 _NOTE(CONSTANTCONDITION) \
46 #define EFX_TX_QSTAT_INCR(_etp, _stat)
51 static __checkReturn efx_rc_t
59 static __checkReturn efx_rc_t
62 __in unsigned int index,
63 __in unsigned int label,
64 __in efsys_mem_t *esmp,
70 __out unsigned int *addedp);
76 static __checkReturn efx_rc_t
79 __in_ecount(n) efx_buffer_t *eb,
81 __in unsigned int completed,
82 __inout unsigned int *addedp);
87 __in unsigned int added,
88 __in unsigned int pushed);
90 static __checkReturn efx_rc_t
93 __in unsigned int ns);
95 static __checkReturn efx_rc_t
101 __in efx_txq_t *etp);
103 __checkReturn efx_rc_t
106 __in_ecount(n) efx_desc_t *ed,
108 __in unsigned int completed,
109 __inout unsigned int *addedp);
112 siena_tx_qdesc_dma_create(
114 __in efsys_dma_addr_t addr,
117 __out efx_desc_t *edp);
121 siena_tx_qstats_update(
123 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat);
126 #endif /* EFSYS_OPT_SIENA */
130 static const efx_tx_ops_t __efx_tx_siena_ops = {
131 siena_tx_init, /* etxo_init */
132 siena_tx_fini, /* etxo_fini */
133 siena_tx_qcreate, /* etxo_qcreate */
134 siena_tx_qdestroy, /* etxo_qdestroy */
135 siena_tx_qpost, /* etxo_qpost */
136 siena_tx_qpush, /* etxo_qpush */
137 siena_tx_qpace, /* etxo_qpace */
138 siena_tx_qflush, /* etxo_qflush */
139 siena_tx_qenable, /* etxo_qenable */
140 NULL, /* etxo_qpio_enable */
141 NULL, /* etxo_qpio_disable */
142 NULL, /* etxo_qpio_write */
143 NULL, /* etxo_qpio_post */
144 siena_tx_qdesc_post, /* etxo_qdesc_post */
145 siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
146 NULL, /* etxo_qdesc_tso_create */
147 NULL, /* etxo_qdesc_tso2_create */
148 NULL, /* etxo_qdesc_vlantci_create */
150 siena_tx_qstats_update, /* etxo_qstats_update */
153 #endif /* EFSYS_OPT_SIENA */
155 #if EFSYS_OPT_HUNTINGTON
156 static const efx_tx_ops_t __efx_tx_hunt_ops = {
157 ef10_tx_init, /* etxo_init */
158 ef10_tx_fini, /* etxo_fini */
159 ef10_tx_qcreate, /* etxo_qcreate */
160 ef10_tx_qdestroy, /* etxo_qdestroy */
161 ef10_tx_qpost, /* etxo_qpost */
162 ef10_tx_qpush, /* etxo_qpush */
163 ef10_tx_qpace, /* etxo_qpace */
164 ef10_tx_qflush, /* etxo_qflush */
165 ef10_tx_qenable, /* etxo_qenable */
166 ef10_tx_qpio_enable, /* etxo_qpio_enable */
167 ef10_tx_qpio_disable, /* etxo_qpio_disable */
168 ef10_tx_qpio_write, /* etxo_qpio_write */
169 ef10_tx_qpio_post, /* etxo_qpio_post */
170 ef10_tx_qdesc_post, /* etxo_qdesc_post */
171 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
172 ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */
173 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
174 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
176 ef10_tx_qstats_update, /* etxo_qstats_update */
179 #endif /* EFSYS_OPT_HUNTINGTON */
181 #if EFSYS_OPT_MEDFORD
182 static const efx_tx_ops_t __efx_tx_medford_ops = {
183 ef10_tx_init, /* etxo_init */
184 ef10_tx_fini, /* etxo_fini */
185 ef10_tx_qcreate, /* etxo_qcreate */
186 ef10_tx_qdestroy, /* etxo_qdestroy */
187 ef10_tx_qpost, /* etxo_qpost */
188 ef10_tx_qpush, /* etxo_qpush */
189 ef10_tx_qpace, /* etxo_qpace */
190 ef10_tx_qflush, /* etxo_qflush */
191 ef10_tx_qenable, /* etxo_qenable */
192 ef10_tx_qpio_enable, /* etxo_qpio_enable */
193 ef10_tx_qpio_disable, /* etxo_qpio_disable */
194 ef10_tx_qpio_write, /* etxo_qpio_write */
195 ef10_tx_qpio_post, /* etxo_qpio_post */
196 ef10_tx_qdesc_post, /* etxo_qdesc_post */
197 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
198 NULL, /* etxo_qdesc_tso_create */
199 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
200 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
202 ef10_tx_qstats_update, /* etxo_qstats_update */
205 #endif /* EFSYS_OPT_MEDFORD */
207 __checkReturn efx_rc_t
211 const efx_tx_ops_t *etxop;
214 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
215 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
217 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
222 if (enp->en_mod_flags & EFX_MOD_TX) {
227 switch (enp->en_family) {
229 case EFX_FAMILY_SIENA:
230 etxop = &__efx_tx_siena_ops;
232 #endif /* EFSYS_OPT_SIENA */
234 #if EFSYS_OPT_HUNTINGTON
235 case EFX_FAMILY_HUNTINGTON:
236 etxop = &__efx_tx_hunt_ops;
238 #endif /* EFSYS_OPT_HUNTINGTON */
240 #if EFSYS_OPT_MEDFORD
241 case EFX_FAMILY_MEDFORD:
242 etxop = &__efx_tx_medford_ops;
244 #endif /* EFSYS_OPT_MEDFORD */
252 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
254 if ((rc = etxop->etxo_init(enp)) != 0)
257 enp->en_etxop = etxop;
258 enp->en_mod_flags |= EFX_MOD_TX;
268 EFSYS_PROBE1(fail1, efx_rc_t, rc);
270 enp->en_etxop = NULL;
271 enp->en_mod_flags &= ~EFX_MOD_TX;
279 const efx_tx_ops_t *etxop = enp->en_etxop;
281 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
282 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
283 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
284 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
286 etxop->etxo_fini(enp);
288 enp->en_etxop = NULL;
289 enp->en_mod_flags &= ~EFX_MOD_TX;
292 __checkReturn efx_rc_t
295 __in unsigned int index,
296 __in unsigned int label,
297 __in efsys_mem_t *esmp,
302 __deref_out efx_txq_t **etpp,
303 __out unsigned int *addedp)
305 const efx_tx_ops_t *etxop = enp->en_etxop;
306 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
310 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
311 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
313 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit);
315 /* Allocate an TXQ object */
316 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
323 etp->et_magic = EFX_TXQ_MAGIC;
325 etp->et_index = index;
326 etp->et_mask = n - 1;
329 /* Initial descriptor index may be modified by etxo_qcreate */
332 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp,
333 n, id, flags, eep, etp, addedp)) != 0)
343 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
345 EFSYS_PROBE1(fail1, efx_rc_t, rc);
353 efx_nic_t *enp = etp->et_enp;
354 const efx_tx_ops_t *etxop = enp->en_etxop;
356 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
358 EFSYS_ASSERT(enp->en_tx_qcount != 0);
361 etxop->etxo_qdestroy(etp);
363 /* Free the TXQ object */
364 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
367 __checkReturn efx_rc_t
370 __in_ecount(n) efx_buffer_t *eb,
372 __in unsigned int completed,
373 __inout unsigned int *addedp)
375 efx_nic_t *enp = etp->et_enp;
376 const efx_tx_ops_t *etxop = enp->en_etxop;
379 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
381 if ((rc = etxop->etxo_qpost(etp, eb,
382 n, completed, addedp)) != 0)
388 EFSYS_PROBE1(fail1, efx_rc_t, rc);
395 __in unsigned int added,
396 __in unsigned int pushed)
398 efx_nic_t *enp = etp->et_enp;
399 const efx_tx_ops_t *etxop = enp->en_etxop;
401 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
403 etxop->etxo_qpush(etp, added, pushed);
406 __checkReturn efx_rc_t
409 __in unsigned int ns)
411 efx_nic_t *enp = etp->et_enp;
412 const efx_tx_ops_t *etxop = enp->en_etxop;
415 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
417 if ((rc = etxop->etxo_qpace(etp, ns)) != 0)
423 EFSYS_PROBE1(fail1, efx_rc_t, rc);
427 __checkReturn efx_rc_t
431 efx_nic_t *enp = etp->et_enp;
432 const efx_tx_ops_t *etxop = enp->en_etxop;
435 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
437 if ((rc = etxop->etxo_qflush(etp)) != 0)
443 EFSYS_PROBE1(fail1, efx_rc_t, rc);
451 efx_nic_t *enp = etp->et_enp;
452 const efx_tx_ops_t *etxop = enp->en_etxop;
454 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
456 etxop->etxo_qenable(etp);
459 __checkReturn efx_rc_t
463 efx_nic_t *enp = etp->et_enp;
464 const efx_tx_ops_t *etxop = enp->en_etxop;
467 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
469 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) {
473 if (etxop->etxo_qpio_enable == NULL) {
477 if ((rc = etxop->etxo_qpio_enable(etp)) != 0)
487 EFSYS_PROBE1(fail1, efx_rc_t, rc);
495 efx_nic_t *enp = etp->et_enp;
496 const efx_tx_ops_t *etxop = enp->en_etxop;
498 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
500 if (etxop->etxo_qpio_disable != NULL)
501 etxop->etxo_qpio_disable(etp);
504 __checkReturn efx_rc_t
507 __in_ecount(buf_length) uint8_t *buffer,
508 __in size_t buf_length,
509 __in size_t pio_buf_offset)
511 efx_nic_t *enp = etp->et_enp;
512 const efx_tx_ops_t *etxop = enp->en_etxop;
515 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
517 if (etxop->etxo_qpio_write != NULL) {
518 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length,
519 pio_buf_offset)) != 0)
527 EFSYS_PROBE1(fail1, efx_rc_t, rc);
531 __checkReturn efx_rc_t
534 __in size_t pkt_length,
535 __in unsigned int completed,
536 __inout unsigned int *addedp)
538 efx_nic_t *enp = etp->et_enp;
539 const efx_tx_ops_t *etxop = enp->en_etxop;
542 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
544 if (etxop->etxo_qpio_post != NULL) {
545 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed,
554 EFSYS_PROBE1(fail1, efx_rc_t, rc);
558 __checkReturn efx_rc_t
561 __in_ecount(n) efx_desc_t *ed,
563 __in unsigned int completed,
564 __inout unsigned int *addedp)
566 efx_nic_t *enp = etp->et_enp;
567 const efx_tx_ops_t *etxop = enp->en_etxop;
570 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
572 if ((rc = etxop->etxo_qdesc_post(etp, ed,
573 n, completed, addedp)) != 0)
579 EFSYS_PROBE1(fail1, efx_rc_t, rc);
584 efx_tx_qdesc_dma_create(
586 __in efsys_dma_addr_t addr,
589 __out efx_desc_t *edp)
591 efx_nic_t *enp = etp->et_enp;
592 const efx_tx_ops_t *etxop = enp->en_etxop;
594 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
595 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL);
597 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp);
601 efx_tx_qdesc_tso_create(
603 __in uint16_t ipv4_id,
604 __in uint32_t tcp_seq,
605 __in uint8_t tcp_flags,
606 __out efx_desc_t *edp)
608 efx_nic_t *enp = etp->et_enp;
609 const efx_tx_ops_t *etxop = enp->en_etxop;
611 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
612 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL);
614 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp);
618 efx_tx_qdesc_tso2_create(
620 __in uint16_t ipv4_id,
621 __in uint32_t tcp_seq,
623 __out_ecount(count) efx_desc_t *edp,
626 efx_nic_t *enp = etp->et_enp;
627 const efx_tx_ops_t *etxop = enp->en_etxop;
629 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
630 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL);
632 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, tcp_seq, mss, edp, count);
636 efx_tx_qdesc_vlantci_create(
639 __out efx_desc_t *edp)
641 efx_nic_t *enp = etp->et_enp;
642 const efx_tx_ops_t *etxop = enp->en_etxop;
644 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
645 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL);
647 etxop->etxo_qdesc_vlantci_create(etp, tci, edp);
653 efx_tx_qstats_update(
655 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
657 efx_nic_t *enp = etp->et_enp;
658 const efx_tx_ops_t *etxop = enp->en_etxop;
660 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
662 etxop->etxo_qstats_update(etp, stat);
669 static __checkReturn efx_rc_t
676 * Disable the timer-based TX DMA backoff and allow TX DMA to be
677 * controlled by the RX FIFO fill level (although always allow a
680 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
681 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
682 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
683 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
684 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
685 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
686 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
687 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
690 * Filter all packets less than 14 bytes to avoid parsing
693 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
694 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);
697 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
698 * descriptors (which is bad).
700 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
701 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
702 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
707 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \
713 id = (_added)++ & (_etp)->et_mask; \
714 offset = id * sizeof (efx_qword_t); \
716 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \
717 unsigned int, id, efsys_dma_addr_t, (_addr), \
718 size_t, (_size), boolean_t, (_eop)); \
720 EFX_POPULATE_QWORD_4(qword, \
721 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \
722 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \
723 FSF_AZ_TX_KER_BUF_ADDR_DW0, \
724 (uint32_t)((_addr) & 0xffffffff), \
725 FSF_AZ_TX_KER_BUF_ADDR_DW1, \
726 (uint32_t)((_addr) >> 32)); \
727 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \
729 _NOTE(CONSTANTCONDITION) \
732 static __checkReturn efx_rc_t
735 __in_ecount(n) efx_buffer_t *eb,
737 __in unsigned int completed,
738 __inout unsigned int *addedp)
740 unsigned int added = *addedp;
744 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1))
747 for (i = 0; i < n; i++) {
748 efx_buffer_t *ebp = &eb[i];
749 efsys_dma_addr_t start = ebp->eb_addr;
750 size_t size = ebp->eb_size;
751 efsys_dma_addr_t end = start + size;
754 * Fragments must not span 4k boundaries.
755 * Here it is a stricter requirement than the maximum length.
757 EFSYS_ASSERT(P2ROUNDUP(start + 1,
758 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= end);
760 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
763 EFX_TX_QSTAT_INCR(etp, TX_POST);
769 EFSYS_PROBE1(fail1, efx_rc_t, rc);
777 __in unsigned int added,
778 __in unsigned int pushed)
780 efx_nic_t *enp = etp->et_enp;
785 /* Push the populated descriptors out */
786 wptr = added & etp->et_mask;
788 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);
790 /* Only write the third DWORD */
791 EFX_POPULATE_DWORD_1(dword,
792 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
794 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
795 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
796 wptr, pushed & etp->et_mask);
797 EFSYS_PIO_WRITE_BARRIER();
798 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
799 etp->et_index, &dword, B_FALSE);
802 #define EFX_MAX_PACE_VALUE 20
803 #define EFX_TX_PACE_CLOCK_BASE 104
805 static __checkReturn efx_rc_t
808 __in unsigned int ns)
810 efx_nic_t *enp = etp->et_enp;
811 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
813 unsigned int pace_val;
814 unsigned int timer_period;
821 * The pace_val to write into the table is s.t
822 * ns <= timer_period * (2 ^ pace_val)
824 timer_period = EFX_TX_PACE_CLOCK_BASE / encp->enc_clk_mult;
825 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) {
826 if ((timer_period << pace_val) >= ns)
830 if (pace_val > EFX_MAX_PACE_VALUE) {
835 /* Update the pacing table */
836 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val);
837 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index,
843 EFSYS_PROBE1(fail1, efx_rc_t, rc);
848 static __checkReturn efx_rc_t
852 efx_nic_t *enp = etp->et_enp;
856 efx_tx_qpace(etp, 0);
858 label = etp->et_index;
860 /* Flush the queue */
861 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
862 FRF_AZ_TX_FLUSH_DESCQ, label);
863 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
872 efx_nic_t *enp = etp->et_enp;
875 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
876 etp->et_index, &oword, B_TRUE);
878 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
879 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
880 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
881 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
882 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));
884 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
885 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
886 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
888 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
889 etp->et_index, &oword, B_TRUE);
892 static __checkReturn efx_rc_t
895 __in unsigned int index,
896 __in unsigned int label,
897 __in efsys_mem_t *esmp,
903 __out unsigned int *addedp)
905 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
910 _NOTE(ARGUNUSED(esmp))
912 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS ==
913 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
914 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);
916 EFSYS_ASSERT(ISP2(EFX_TXQ_MAXNDESCS(encp)));
917 EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS));
919 if (!ISP2(n) || (n < EFX_TXQ_MINNDESCS) || (n > EFX_EVQ_MAXNEVS)) {
923 if (index >= encp->enc_txq_limit) {
928 (1 << size) <= (EFX_TXQ_MAXNDESCS(encp) / EFX_TXQ_MINNDESCS);
930 if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS))
932 if (id + (1 << size) >= encp->enc_buftbl_limit) {
937 /* Set up the new descriptor queue */
940 EFX_POPULATE_OWORD_6(oword,
941 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
942 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
943 FRF_AZ_TX_DESCQ_OWNER_ID, 0,
944 FRF_AZ_TX_DESCQ_LABEL, label,
945 FRF_AZ_TX_DESCQ_SIZE, size,
946 FRF_AZ_TX_DESCQ_TYPE, 0);
948 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
949 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
950 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1);
951 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
952 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1);
954 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
955 etp->et_index, &oword, B_TRUE);
964 EFSYS_PROBE1(fail1, efx_rc_t, rc);
969 __checkReturn efx_rc_t
972 __in_ecount(n) efx_desc_t *ed,
974 __in unsigned int completed,
975 __inout unsigned int *addedp)
977 unsigned int added = *addedp;
981 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
986 for (i = 0; i < n; i++) {
987 efx_desc_t *edp = &ed[i];
991 id = added++ & etp->et_mask;
992 offset = id * sizeof (efx_desc_t);
994 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq);
997 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index,
998 unsigned int, added, unsigned int, n);
1000 EFX_TX_QSTAT_INCR(etp, TX_POST);
1006 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1011 siena_tx_qdesc_dma_create(
1012 __in efx_txq_t *etp,
1013 __in efsys_dma_addr_t addr,
1016 __out efx_desc_t *edp)
1019 * Fragments must not span 4k boundaries.
1020 * Here it is a stricter requirement than the maximum length.
1022 EFSYS_ASSERT(P2ROUNDUP(addr + 1,
1023 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= addr + size);
1025 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index,
1026 efsys_dma_addr_t, addr,
1027 size_t, size, boolean_t, eop);
1029 EFX_POPULATE_QWORD_4(edp->ed_eq,
1030 FSF_AZ_TX_KER_CONT, eop ? 0 : 1,
1031 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size,
1032 FSF_AZ_TX_KER_BUF_ADDR_DW0,
1033 (uint32_t)(addr & 0xffffffff),
1034 FSF_AZ_TX_KER_BUF_ADDR_DW1,
1035 (uint32_t)(addr >> 32));
1038 #endif /* EFSYS_OPT_SIENA */
1040 #if EFSYS_OPT_QSTATS
1042 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 2866874ecd7a363b */
1043 static const char * const __efx_tx_qstat_name[] = {
1047 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */
1051 __in efx_nic_t *enp,
1052 __in unsigned int id)
1054 _NOTE(ARGUNUSED(enp))
1055 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1056 EFSYS_ASSERT3U(id, <, TX_NQSTATS);
1058 return (__efx_tx_qstat_name[id]);
1060 #endif /* EFSYS_OPT_NAMES */
1061 #endif /* EFSYS_OPT_QSTATS */
1065 #if EFSYS_OPT_QSTATS
1067 siena_tx_qstats_update(
1068 __in efx_txq_t *etp,
1069 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
1073 for (id = 0; id < TX_NQSTATS; id++) {
1074 efsys_stat_t *essp = &stat[id];
1076 EFSYS_STAT_INCR(essp, etp->et_stat[id]);
1077 etp->et_stat[id] = 0;
1080 #endif /* EFSYS_OPT_QSTATS */
1084 __in efx_txq_t *etp)
1086 efx_nic_t *enp = etp->et_enp;
1089 /* Purge descriptor queue */
1090 EFX_ZERO_OWORD(oword);
1092 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
1093 etp->et_index, &oword, B_TRUE);
1098 __in efx_nic_t *enp)
1100 _NOTE(ARGUNUSED(enp))
1103 #endif /* EFSYS_OPT_SIENA */