2 * Copyright (c) 2007-2016 Solarflare Communications Inc.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 * The views and conclusions contained in the software and documentation are
27 * those of the authors and should not be interpreted as representing official
28 * policies, either expressed or implied, of the FreeBSD Project.
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
38 #define EFX_TX_QSTAT_INCR(_etp, _stat) \
40 (_etp)->et_stat[_stat]++; \
41 _NOTE(CONSTANTCONDITION) \
44 #define EFX_TX_QSTAT_INCR(_etp, _stat)
49 static __checkReturn efx_rc_t
57 static __checkReturn efx_rc_t
60 __in unsigned int index,
61 __in unsigned int label,
62 __in efsys_mem_t *esmp,
68 __out unsigned int *addedp);
74 static __checkReturn efx_rc_t
77 __in_ecount(n) efx_buffer_t *eb,
79 __in unsigned int completed,
80 __inout unsigned int *addedp);
85 __in unsigned int added,
86 __in unsigned int pushed);
88 static __checkReturn efx_rc_t
91 __in unsigned int ns);
93 static __checkReturn efx_rc_t
101 __checkReturn efx_rc_t
104 __in_ecount(n) efx_desc_t *ed,
106 __in unsigned int completed,
107 __inout unsigned int *addedp);
110 siena_tx_qdesc_dma_create(
112 __in efsys_dma_addr_t addr,
115 __out efx_desc_t *edp);
119 siena_tx_qstats_update(
121 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat);
124 #endif /* EFSYS_OPT_SIENA */
128 static const efx_tx_ops_t __efx_tx_siena_ops = {
129 siena_tx_init, /* etxo_init */
130 siena_tx_fini, /* etxo_fini */
131 siena_tx_qcreate, /* etxo_qcreate */
132 siena_tx_qdestroy, /* etxo_qdestroy */
133 siena_tx_qpost, /* etxo_qpost */
134 siena_tx_qpush, /* etxo_qpush */
135 siena_tx_qpace, /* etxo_qpace */
136 siena_tx_qflush, /* etxo_qflush */
137 siena_tx_qenable, /* etxo_qenable */
138 NULL, /* etxo_qpio_enable */
139 NULL, /* etxo_qpio_disable */
140 NULL, /* etxo_qpio_write */
141 NULL, /* etxo_qpio_post */
142 siena_tx_qdesc_post, /* etxo_qdesc_post */
143 siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
144 NULL, /* etxo_qdesc_tso_create */
145 NULL, /* etxo_qdesc_tso2_create */
146 NULL, /* etxo_qdesc_vlantci_create */
148 siena_tx_qstats_update, /* etxo_qstats_update */
151 #endif /* EFSYS_OPT_SIENA */
153 #if EFSYS_OPT_HUNTINGTON
154 static const efx_tx_ops_t __efx_tx_hunt_ops = {
155 ef10_tx_init, /* etxo_init */
156 ef10_tx_fini, /* etxo_fini */
157 ef10_tx_qcreate, /* etxo_qcreate */
158 ef10_tx_qdestroy, /* etxo_qdestroy */
159 ef10_tx_qpost, /* etxo_qpost */
160 ef10_tx_qpush, /* etxo_qpush */
161 ef10_tx_qpace, /* etxo_qpace */
162 ef10_tx_qflush, /* etxo_qflush */
163 ef10_tx_qenable, /* etxo_qenable */
164 ef10_tx_qpio_enable, /* etxo_qpio_enable */
165 ef10_tx_qpio_disable, /* etxo_qpio_disable */
166 ef10_tx_qpio_write, /* etxo_qpio_write */
167 ef10_tx_qpio_post, /* etxo_qpio_post */
168 ef10_tx_qdesc_post, /* etxo_qdesc_post */
169 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
170 ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */
171 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
172 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
174 ef10_tx_qstats_update, /* etxo_qstats_update */
177 #endif /* EFSYS_OPT_HUNTINGTON */
179 #if EFSYS_OPT_MEDFORD
180 static const efx_tx_ops_t __efx_tx_medford_ops = {
181 ef10_tx_init, /* etxo_init */
182 ef10_tx_fini, /* etxo_fini */
183 ef10_tx_qcreate, /* etxo_qcreate */
184 ef10_tx_qdestroy, /* etxo_qdestroy */
185 ef10_tx_qpost, /* etxo_qpost */
186 ef10_tx_qpush, /* etxo_qpush */
187 ef10_tx_qpace, /* etxo_qpace */
188 ef10_tx_qflush, /* etxo_qflush */
189 ef10_tx_qenable, /* etxo_qenable */
190 ef10_tx_qpio_enable, /* etxo_qpio_enable */
191 ef10_tx_qpio_disable, /* etxo_qpio_disable */
192 ef10_tx_qpio_write, /* etxo_qpio_write */
193 ef10_tx_qpio_post, /* etxo_qpio_post */
194 ef10_tx_qdesc_post, /* etxo_qdesc_post */
195 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
196 NULL, /* etxo_qdesc_tso_create */
197 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
198 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
200 ef10_tx_qstats_update, /* etxo_qstats_update */
203 #endif /* EFSYS_OPT_MEDFORD */
205 __checkReturn efx_rc_t
209 const efx_tx_ops_t *etxop;
212 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
213 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
215 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
220 if (enp->en_mod_flags & EFX_MOD_TX) {
225 switch (enp->en_family) {
227 case EFX_FAMILY_SIENA:
228 etxop = &__efx_tx_siena_ops;
230 #endif /* EFSYS_OPT_SIENA */
232 #if EFSYS_OPT_HUNTINGTON
233 case EFX_FAMILY_HUNTINGTON:
234 etxop = &__efx_tx_hunt_ops;
236 #endif /* EFSYS_OPT_HUNTINGTON */
238 #if EFSYS_OPT_MEDFORD
239 case EFX_FAMILY_MEDFORD:
240 etxop = &__efx_tx_medford_ops;
242 #endif /* EFSYS_OPT_MEDFORD */
250 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
252 if ((rc = etxop->etxo_init(enp)) != 0)
255 enp->en_etxop = etxop;
256 enp->en_mod_flags |= EFX_MOD_TX;
266 EFSYS_PROBE1(fail1, efx_rc_t, rc);
268 enp->en_etxop = NULL;
269 enp->en_mod_flags &= ~EFX_MOD_TX;
277 const efx_tx_ops_t *etxop = enp->en_etxop;
279 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
280 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
281 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
282 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
284 etxop->etxo_fini(enp);
286 enp->en_etxop = NULL;
287 enp->en_mod_flags &= ~EFX_MOD_TX;
290 __checkReturn efx_rc_t
293 __in unsigned int index,
294 __in unsigned int label,
295 __in efsys_mem_t *esmp,
300 __deref_out efx_txq_t **etpp,
301 __out unsigned int *addedp)
303 const efx_tx_ops_t *etxop = enp->en_etxop;
304 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
308 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
309 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
311 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit);
313 /* Allocate an TXQ object */
314 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
321 etp->et_magic = EFX_TXQ_MAGIC;
323 etp->et_index = index;
324 etp->et_mask = n - 1;
327 /* Initial descriptor index may be modified by etxo_qcreate */
330 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp,
331 n, id, flags, eep, etp, addedp)) != 0)
341 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
343 EFSYS_PROBE1(fail1, efx_rc_t, rc);
351 efx_nic_t *enp = etp->et_enp;
352 const efx_tx_ops_t *etxop = enp->en_etxop;
354 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
356 EFSYS_ASSERT(enp->en_tx_qcount != 0);
359 etxop->etxo_qdestroy(etp);
361 /* Free the TXQ object */
362 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
365 __checkReturn efx_rc_t
368 __in_ecount(n) efx_buffer_t *eb,
370 __in unsigned int completed,
371 __inout unsigned int *addedp)
373 efx_nic_t *enp = etp->et_enp;
374 const efx_tx_ops_t *etxop = enp->en_etxop;
377 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
379 if ((rc = etxop->etxo_qpost(etp, eb,
380 n, completed, addedp)) != 0)
386 EFSYS_PROBE1(fail1, efx_rc_t, rc);
393 __in unsigned int added,
394 __in unsigned int pushed)
396 efx_nic_t *enp = etp->et_enp;
397 const efx_tx_ops_t *etxop = enp->en_etxop;
399 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
401 etxop->etxo_qpush(etp, added, pushed);
404 __checkReturn efx_rc_t
407 __in unsigned int ns)
409 efx_nic_t *enp = etp->et_enp;
410 const efx_tx_ops_t *etxop = enp->en_etxop;
413 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
415 if ((rc = etxop->etxo_qpace(etp, ns)) != 0)
421 EFSYS_PROBE1(fail1, efx_rc_t, rc);
425 __checkReturn efx_rc_t
429 efx_nic_t *enp = etp->et_enp;
430 const efx_tx_ops_t *etxop = enp->en_etxop;
433 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
435 if ((rc = etxop->etxo_qflush(etp)) != 0)
441 EFSYS_PROBE1(fail1, efx_rc_t, rc);
449 efx_nic_t *enp = etp->et_enp;
450 const efx_tx_ops_t *etxop = enp->en_etxop;
452 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
454 etxop->etxo_qenable(etp);
457 __checkReturn efx_rc_t
461 efx_nic_t *enp = etp->et_enp;
462 const efx_tx_ops_t *etxop = enp->en_etxop;
465 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
467 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) {
471 if (etxop->etxo_qpio_enable == NULL) {
475 if ((rc = etxop->etxo_qpio_enable(etp)) != 0)
485 EFSYS_PROBE1(fail1, efx_rc_t, rc);
493 efx_nic_t *enp = etp->et_enp;
494 const efx_tx_ops_t *etxop = enp->en_etxop;
496 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
498 if (etxop->etxo_qpio_disable != NULL)
499 etxop->etxo_qpio_disable(etp);
502 __checkReturn efx_rc_t
505 __in_ecount(buf_length) uint8_t *buffer,
506 __in size_t buf_length,
507 __in size_t pio_buf_offset)
509 efx_nic_t *enp = etp->et_enp;
510 const efx_tx_ops_t *etxop = enp->en_etxop;
513 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
515 if (etxop->etxo_qpio_write != NULL) {
516 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length,
517 pio_buf_offset)) != 0)
525 EFSYS_PROBE1(fail1, efx_rc_t, rc);
529 __checkReturn efx_rc_t
532 __in size_t pkt_length,
533 __in unsigned int completed,
534 __inout unsigned int *addedp)
536 efx_nic_t *enp = etp->et_enp;
537 const efx_tx_ops_t *etxop = enp->en_etxop;
540 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
542 if (etxop->etxo_qpio_post != NULL) {
543 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed,
552 EFSYS_PROBE1(fail1, efx_rc_t, rc);
556 __checkReturn efx_rc_t
559 __in_ecount(n) efx_desc_t *ed,
561 __in unsigned int completed,
562 __inout unsigned int *addedp)
564 efx_nic_t *enp = etp->et_enp;
565 const efx_tx_ops_t *etxop = enp->en_etxop;
568 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
570 if ((rc = etxop->etxo_qdesc_post(etp, ed,
571 n, completed, addedp)) != 0)
577 EFSYS_PROBE1(fail1, efx_rc_t, rc);
582 efx_tx_qdesc_dma_create(
584 __in efsys_dma_addr_t addr,
587 __out efx_desc_t *edp)
589 efx_nic_t *enp = etp->et_enp;
590 const efx_tx_ops_t *etxop = enp->en_etxop;
592 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
593 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL);
595 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp);
599 efx_tx_qdesc_tso_create(
601 __in uint16_t ipv4_id,
602 __in uint32_t tcp_seq,
603 __in uint8_t tcp_flags,
604 __out efx_desc_t *edp)
606 efx_nic_t *enp = etp->et_enp;
607 const efx_tx_ops_t *etxop = enp->en_etxop;
609 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
610 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL);
612 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp);
616 efx_tx_qdesc_tso2_create(
618 __in uint16_t ipv4_id,
619 __in uint32_t tcp_seq,
621 __out_ecount(count) efx_desc_t *edp,
624 efx_nic_t *enp = etp->et_enp;
625 const efx_tx_ops_t *etxop = enp->en_etxop;
627 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
628 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL);
630 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, tcp_seq, mss, edp, count);
634 efx_tx_qdesc_vlantci_create(
637 __out efx_desc_t *edp)
639 efx_nic_t *enp = etp->et_enp;
640 const efx_tx_ops_t *etxop = enp->en_etxop;
642 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
643 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL);
645 etxop->etxo_qdesc_vlantci_create(etp, tci, edp);
651 efx_tx_qstats_update(
653 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
655 efx_nic_t *enp = etp->et_enp;
656 const efx_tx_ops_t *etxop = enp->en_etxop;
658 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
660 etxop->etxo_qstats_update(etp, stat);
667 static __checkReturn efx_rc_t
674 * Disable the timer-based TX DMA backoff and allow TX DMA to be
675 * controlled by the RX FIFO fill level (although always allow a
678 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
679 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
680 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
681 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
682 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
683 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
684 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
685 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
688 * Filter all packets less than 14 bytes to avoid parsing
691 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
692 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);
695 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
696 * descriptors (which is bad).
698 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
699 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
700 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
705 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \
711 id = (_added)++ & (_etp)->et_mask; \
712 offset = id * sizeof (efx_qword_t); \
714 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \
715 unsigned int, id, efsys_dma_addr_t, (_addr), \
716 size_t, (_size), boolean_t, (_eop)); \
718 EFX_POPULATE_QWORD_4(qword, \
719 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \
720 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \
721 FSF_AZ_TX_KER_BUF_ADDR_DW0, \
722 (uint32_t)((_addr) & 0xffffffff), \
723 FSF_AZ_TX_KER_BUF_ADDR_DW1, \
724 (uint32_t)((_addr) >> 32)); \
725 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \
727 _NOTE(CONSTANTCONDITION) \
730 static __checkReturn efx_rc_t
733 __in_ecount(n) efx_buffer_t *eb,
735 __in unsigned int completed,
736 __inout unsigned int *addedp)
738 unsigned int added = *addedp;
742 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1))
745 for (i = 0; i < n; i++) {
746 efx_buffer_t *ebp = &eb[i];
747 efsys_dma_addr_t start = ebp->eb_addr;
748 size_t size = ebp->eb_size;
749 efsys_dma_addr_t end = start + size;
752 * Fragments must not span 4k boundaries.
753 * Here it is a stricter requirement than the maximum length.
755 EFSYS_ASSERT(P2ROUNDUP(start + 1,
756 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= end);
758 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
761 EFX_TX_QSTAT_INCR(etp, TX_POST);
767 EFSYS_PROBE1(fail1, efx_rc_t, rc);
775 __in unsigned int added,
776 __in unsigned int pushed)
778 efx_nic_t *enp = etp->et_enp;
783 /* Push the populated descriptors out */
784 wptr = added & etp->et_mask;
786 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);
788 /* Only write the third DWORD */
789 EFX_POPULATE_DWORD_1(dword,
790 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
792 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
793 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
794 wptr, pushed & etp->et_mask);
795 EFSYS_PIO_WRITE_BARRIER();
796 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
797 etp->et_index, &dword, B_FALSE);
800 #define EFX_MAX_PACE_VALUE 20
801 #define EFX_TX_PACE_CLOCK_BASE 104
803 static __checkReturn efx_rc_t
806 __in unsigned int ns)
808 efx_nic_t *enp = etp->et_enp;
809 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
811 unsigned int pace_val;
812 unsigned int timer_period;
819 * The pace_val to write into the table is s.t
820 * ns <= timer_period * (2 ^ pace_val)
822 timer_period = EFX_TX_PACE_CLOCK_BASE / encp->enc_clk_mult;
823 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) {
824 if ((timer_period << pace_val) >= ns)
828 if (pace_val > EFX_MAX_PACE_VALUE) {
833 /* Update the pacing table */
834 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val);
835 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index,
841 EFSYS_PROBE1(fail1, efx_rc_t, rc);
846 static __checkReturn efx_rc_t
850 efx_nic_t *enp = etp->et_enp;
854 efx_tx_qpace(etp, 0);
856 label = etp->et_index;
858 /* Flush the queue */
859 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
860 FRF_AZ_TX_FLUSH_DESCQ, label);
861 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
870 efx_nic_t *enp = etp->et_enp;
873 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
874 etp->et_index, &oword, B_TRUE);
876 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
877 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
878 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
879 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
880 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));
882 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
883 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
884 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
886 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
887 etp->et_index, &oword, B_TRUE);
890 static __checkReturn efx_rc_t
893 __in unsigned int index,
894 __in unsigned int label,
895 __in efsys_mem_t *esmp,
901 __out unsigned int *addedp)
903 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
908 _NOTE(ARGUNUSED(esmp))
910 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS ==
911 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
912 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);
914 EFSYS_ASSERT(ISP2(EFX_TXQ_MAXNDESCS(encp)));
915 EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS));
917 if (!ISP2(n) || (n < EFX_TXQ_MINNDESCS) || (n > EFX_EVQ_MAXNEVS)) {
921 if (index >= encp->enc_txq_limit) {
926 (1 << size) <= (EFX_TXQ_MAXNDESCS(encp) / EFX_TXQ_MINNDESCS);
928 if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS))
930 if (id + (1 << size) >= encp->enc_buftbl_limit) {
935 /* Set up the new descriptor queue */
938 EFX_POPULATE_OWORD_6(oword,
939 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
940 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
941 FRF_AZ_TX_DESCQ_OWNER_ID, 0,
942 FRF_AZ_TX_DESCQ_LABEL, label,
943 FRF_AZ_TX_DESCQ_SIZE, size,
944 FRF_AZ_TX_DESCQ_TYPE, 0);
946 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
947 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
948 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1);
949 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
950 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1);
952 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
953 etp->et_index, &oword, B_TRUE);
962 EFSYS_PROBE1(fail1, efx_rc_t, rc);
967 __checkReturn efx_rc_t
970 __in_ecount(n) efx_desc_t *ed,
972 __in unsigned int completed,
973 __inout unsigned int *addedp)
975 unsigned int added = *addedp;
979 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
984 for (i = 0; i < n; i++) {
985 efx_desc_t *edp = &ed[i];
989 id = added++ & etp->et_mask;
990 offset = id * sizeof (efx_desc_t);
992 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq);
995 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index,
996 unsigned int, added, unsigned int, n);
998 EFX_TX_QSTAT_INCR(etp, TX_POST);
1004 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1009 siena_tx_qdesc_dma_create(
1010 __in efx_txq_t *etp,
1011 __in efsys_dma_addr_t addr,
1014 __out efx_desc_t *edp)
1017 * Fragments must not span 4k boundaries.
1018 * Here it is a stricter requirement than the maximum length.
1020 EFSYS_ASSERT(P2ROUNDUP(addr + 1,
1021 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= addr + size);
1023 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index,
1024 efsys_dma_addr_t, addr,
1025 size_t, size, boolean_t, eop);
1027 EFX_POPULATE_QWORD_4(edp->ed_eq,
1028 FSF_AZ_TX_KER_CONT, eop ? 0 : 1,
1029 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size,
1030 FSF_AZ_TX_KER_BUF_ADDR_DW0,
1031 (uint32_t)(addr & 0xffffffff),
1032 FSF_AZ_TX_KER_BUF_ADDR_DW1,
1033 (uint32_t)(addr >> 32));
1036 #endif /* EFSYS_OPT_SIENA */
1038 #if EFSYS_OPT_QSTATS
1040 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 2866874ecd7a363b */
1041 static const char * const __efx_tx_qstat_name[] = {
1045 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */
1049 __in efx_nic_t *enp,
1050 __in unsigned int id)
1052 _NOTE(ARGUNUSED(enp))
1053 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1054 EFSYS_ASSERT3U(id, <, TX_NQSTATS);
1056 return (__efx_tx_qstat_name[id]);
1058 #endif /* EFSYS_OPT_NAMES */
1059 #endif /* EFSYS_OPT_QSTATS */
1063 #if EFSYS_OPT_QSTATS
1065 siena_tx_qstats_update(
1066 __in efx_txq_t *etp,
1067 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
1071 for (id = 0; id < TX_NQSTATS; id++) {
1072 efsys_stat_t *essp = &stat[id];
1074 EFSYS_STAT_INCR(essp, etp->et_stat[id]);
1075 etp->et_stat[id] = 0;
1078 #endif /* EFSYS_OPT_QSTATS */
1082 __in efx_txq_t *etp)
1084 efx_nic_t *enp = etp->et_enp;
1087 /* Purge descriptor queue */
1088 EFX_ZERO_OWORD(oword);
1090 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
1091 etp->et_index, &oword, B_TRUE);
1096 __in efx_nic_t *enp)
1098 _NOTE(ARGUNUSED(enp))
1101 #endif /* EFSYS_OPT_SIENA */