2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 2017-2018 Chelsio Communications, Inc.
6 * Written by: John Baldwin <jhb@FreeBSD.org>
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 #include <sys/cdefs.h>
33 __FBSDID("$FreeBSD$");
35 #include <sys/param.h>
36 #include <sys/sglist.h>
37 #include <sys/socket.h>
38 #include <sys/socketvar.h>
39 #include <sys/systm.h>
40 #include <netinet/in.h>
41 #include <netinet/in_pcb.h>
42 #include <netinet/tcp_var.h>
43 #include <netinet/toecore.h>
46 #include "common/common.h"
47 #include "common/t4_tcb.h"
48 #include "tom/t4_tom_l2t.h"
49 #include "tom/t4_tom.h"
52 * The TCP sequence number of a CPL_TLS_DATA mbuf is saved here while
53 * the mbuf is in the ulp_pdu_reclaimq.
55 #define tls_tcp_seq PH_loc.thirtytwo[0]
58 * Handshake lock used for the handshake timer. Having a global lock
59 * is perhaps not ideal, but it avoids having to use callout_drain()
60 * in tls_uninit_toep() which can't block. Also, the timer shouldn't
61 * actually fire for most connections.
63 static struct mtx tls_handshake_lock;
66 t4_set_tls_tcb_field(struct toepcb *toep, uint16_t word, uint64_t mask,
69 struct adapter *sc = td_adapter(toep->td);
71 t4_set_tcb_field(sc, toep->ofld_txq, toep, word, mask, val, 0, 0);
74 /* TLS and DTLS common routines */
76 can_tls_offload(struct adapter *sc)
79 return (sc->tt.tls && sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS);
83 tls_tx_key(struct toepcb *toep)
85 struct tls_ofld_info *tls_ofld = &toep->tls;
87 return (tls_ofld->tx_key_addr >= 0);
91 tls_rx_key(struct toepcb *toep)
93 struct tls_ofld_info *tls_ofld = &toep->tls;
95 return (tls_ofld->rx_key_addr >= 0);
99 key_size(struct toepcb *toep)
101 struct tls_ofld_info *tls_ofld = &toep->tls;
103 return ((tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_IMMEDIATE) ?
104 tls_ofld->k_ctx.tx_key_info_size : KEY_IN_DDR_SIZE);
107 /* Set TLS Key-Id in TCB */
109 t4_set_tls_keyid(struct toepcb *toep, unsigned int key_id)
112 t4_set_tls_tcb_field(toep, W_TCB_RX_TLS_KEY_TAG,
113 V_TCB_RX_TLS_KEY_TAG(M_TCB_RX_TLS_BUF_TAG),
114 V_TCB_RX_TLS_KEY_TAG(key_id));
117 /* Clear TF_RX_QUIESCE to re-enable receive. */
119 t4_clear_rx_quiesce(struct toepcb *toep)
122 t4_set_tls_tcb_field(toep, W_TCB_T_FLAGS, V_TF_RX_QUIESCE(1), 0);
126 tls_clr_ofld_mode(struct toepcb *toep)
129 tls_stop_handshake_timer(toep);
131 /* Operate in PDU extraction mode only. */
132 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW,
133 V_TCB_ULP_RAW(M_TCB_ULP_RAW),
134 V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1)));
135 t4_clear_rx_quiesce(toep);
139 tls_clr_quiesce(struct toepcb *toep)
142 tls_stop_handshake_timer(toep);
143 t4_clear_rx_quiesce(toep);
147 * Calculate the TLS data expansion size
150 tls_expansion_size(struct toepcb *toep, int data_len, int full_pdus_only,
151 unsigned short *pdus_per_ulp)
153 struct tls_ofld_info *tls_ofld = &toep->tls;
154 struct tls_scmd *scmd = &tls_ofld->scmd0;
155 int expn_size = 0, frag_count = 0, pad_per_pdu = 0,
156 pad_last_pdu = 0, last_frag_size = 0, max_frag_size = 0;
158 int hdr_len = TLS_HEADER_LENGTH;
161 max_frag_size = tls_ofld->k_ctx.frag_size;
162 if (G_SCMD_CIPH_MODE(scmd->seqno_numivs) ==
163 SCMD_CIPH_MODE_AES_GCM) {
164 frag_count = (data_len / max_frag_size);
165 exp_per_pdu = GCM_TAG_SIZE + AEAD_EXPLICIT_DATA_SIZE +
167 expn_size = frag_count * exp_per_pdu;
168 if (full_pdus_only) {
169 *pdus_per_ulp = data_len / (exp_per_pdu +
171 if (*pdus_per_ulp > 32)
173 else if(!*pdus_per_ulp)
175 expn_size = (*pdus_per_ulp) * exp_per_pdu;
178 if ((last_frag_size = data_len % max_frag_size) > 0) {
180 expn_size += exp_per_pdu;
183 } else if (G_SCMD_CIPH_MODE(scmd->seqno_numivs) !=
184 SCMD_CIPH_MODE_NOP) {
185 /* Calculate the number of fragments we can make */
186 frag_count = (data_len / max_frag_size);
187 if (frag_count > 0) {
188 pad_per_pdu = (((howmany((max_frag_size +
189 tls_ofld->mac_length),
190 CIPHER_BLOCK_SIZE)) *
193 tls_ofld->mac_length));
195 pad_per_pdu = CIPHER_BLOCK_SIZE;
196 exp_per_pdu = pad_per_pdu +
197 tls_ofld->mac_length +
198 hdr_len + CIPHER_BLOCK_SIZE;
199 expn_size = frag_count * exp_per_pdu;
201 if (full_pdus_only) {
202 *pdus_per_ulp = data_len / (exp_per_pdu +
204 if (*pdus_per_ulp > 32)
206 else if (!*pdus_per_ulp)
208 expn_size = (*pdus_per_ulp) * exp_per_pdu;
211 /* Consider the last fragment */
212 if ((last_frag_size = data_len % max_frag_size) > 0) {
213 pad_last_pdu = (((howmany((last_frag_size +
214 tls_ofld->mac_length),
215 CIPHER_BLOCK_SIZE)) *
218 tls_ofld->mac_length));
220 pad_last_pdu = CIPHER_BLOCK_SIZE;
221 expn_size += (pad_last_pdu +
222 tls_ofld->mac_length + hdr_len +
233 tls_copy_tx_key(struct toepcb *toep, void *dst)
235 struct tls_ofld_info *tls_ofld = &toep->tls;
236 struct ulptx_sc_memrd *sc_memrd;
237 struct ulptx_idata *sc;
239 if (tls_ofld->k_ctx.tx_key_info_size <= 0)
242 if (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_DDR) {
244 sc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
245 sc->len = htobe32(0);
246 sc_memrd = (struct ulptx_sc_memrd *)(sc + 1);
247 sc_memrd->cmd_to_len = htobe32(V_ULPTX_CMD(ULP_TX_SC_MEMRD) |
248 V_ULP_TX_SC_MORE(1) |
249 V_ULPTX_LEN16(tls_ofld->k_ctx.tx_key_info_size >> 4));
250 sc_memrd->addr = htobe32(tls_ofld->tx_key_addr >> 5);
251 } else if (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_IMMEDIATE) {
252 memcpy(dst, &tls_ofld->k_ctx.tx,
253 tls_ofld->k_ctx.tx_key_info_size);
257 /* TLS/DTLS content type for CPL SFO */
258 static inline unsigned char
259 tls_content_type(unsigned char content_type)
262 * XXX: Shouldn't this map CONTENT_TYPE_APP_DATA to DATA and
263 * default to "CUSTOM" for all other types including
266 switch (content_type) {
267 case CONTENT_TYPE_CCS:
268 return CPL_TX_TLS_SFO_TYPE_CCS;
269 case CONTENT_TYPE_ALERT:
270 return CPL_TX_TLS_SFO_TYPE_ALERT;
271 case CONTENT_TYPE_HANDSHAKE:
272 return CPL_TX_TLS_SFO_TYPE_HANDSHAKE;
273 case CONTENT_TYPE_HEARTBEAT:
274 return CPL_TX_TLS_SFO_TYPE_HEARTBEAT;
276 return CPL_TX_TLS_SFO_TYPE_DATA;
280 get_cipher_key_size(unsigned int ck_size)
283 case AES_NOP: /* NOP */
285 case AES_128: /* AES128 */
286 return CH_CK_SIZE_128;
287 case AES_192: /* AES192 */
288 return CH_CK_SIZE_192;
289 case AES_256: /* AES256 */
290 return CH_CK_SIZE_256;
292 return CH_CK_SIZE_256;
297 get_mac_key_size(unsigned int mk_size)
300 case SHA_NOP: /* NOP */
301 return CH_MK_SIZE_128;
302 case SHA_GHASH: /* GHASH */
303 case SHA_512: /* SHA512 */
304 return CH_MK_SIZE_512;
305 case SHA_224: /* SHA2-224 */
306 return CH_MK_SIZE_192;
307 case SHA_256: /* SHA2-256*/
308 return CH_MK_SIZE_256;
309 case SHA_384: /* SHA384 */
310 return CH_MK_SIZE_512;
311 case SHA1: /* SHA1 */
313 return CH_MK_SIZE_160;
318 get_proto_ver(int proto_ver)
322 return TLS_1_2_VERSION;
324 return TLS_1_1_VERSION;
325 case DTLS1_2_VERSION:
326 return DTLS_1_2_VERSION;
328 return TLS_VERSION_MAX;
333 tls_rxkey_flit1(struct tls_keyctx *kwr, struct tls_key_context *kctx)
336 if (kctx->state.enc_mode == CH_EVP_CIPH_GCM_MODE) {
337 kwr->u.rxhdr.ivinsert_to_authinsrt =
338 htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) |
339 V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) |
340 V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) |
341 V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(14ULL) |
342 V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(16ULL) |
343 V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(14ULL) |
344 V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) |
345 V_TLS_KEYCTX_TX_WR_AUTHINSRT(16ULL));
346 kwr->u.rxhdr.ivpresent_to_rxmk_size &=
347 ~(V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1));
348 kwr->u.rxhdr.authmode_to_rxvalid &=
349 ~(V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1));
351 kwr->u.rxhdr.ivinsert_to_authinsrt =
352 htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) |
353 V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) |
354 V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) |
355 V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(22ULL) |
356 V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(0ULL) |
357 V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(22ULL) |
358 V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) |
359 V_TLS_KEYCTX_TX_WR_AUTHINSRT(0ULL));
365 prepare_rxkey_wr(struct tls_keyctx *kwr, struct tls_key_context *kctx)
367 unsigned int ck_size = kctx->cipher_secret_size;
368 unsigned int mk_size = kctx->mac_secret_size;
369 int proto_ver = kctx->proto_ver;
371 kwr->u.rxhdr.flitcnt_hmacctrl =
372 ((kctx->tx_key_info_size >> 4) << 3) | kctx->hmac_ctrl;
374 kwr->u.rxhdr.protover_ciphmode =
375 V_TLS_KEYCTX_TX_WR_PROTOVER(get_proto_ver(proto_ver)) |
376 V_TLS_KEYCTX_TX_WR_CIPHMODE(kctx->state.enc_mode);
378 kwr->u.rxhdr.authmode_to_rxvalid =
379 V_TLS_KEYCTX_TX_WR_AUTHMODE(kctx->state.auth_mode) |
380 V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1) |
381 V_TLS_KEYCTX_TX_WR_SEQNUMCTRL(3) |
382 V_TLS_KEYCTX_TX_WR_RXVALID(1);
384 kwr->u.rxhdr.ivpresent_to_rxmk_size =
385 V_TLS_KEYCTX_TX_WR_IVPRESENT(0) |
386 V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1) |
387 V_TLS_KEYCTX_TX_WR_RXCK_SIZE(get_cipher_key_size(ck_size)) |
388 V_TLS_KEYCTX_TX_WR_RXMK_SIZE(get_mac_key_size(mk_size));
390 tls_rxkey_flit1(kwr, kctx);
392 /* No key reversal for GCM */
393 if (kctx->state.enc_mode != CH_EVP_CIPH_GCM_MODE) {
394 t4_aes_getdeckey(kwr->keys.edkey, kctx->rx.key,
395 (kctx->cipher_secret_size << 3));
396 memcpy(kwr->keys.edkey + kctx->cipher_secret_size,
397 kctx->rx.key + kctx->cipher_secret_size,
398 (IPAD_SIZE + OPAD_SIZE));
400 memcpy(kwr->keys.edkey, kctx->rx.key,
401 (kctx->tx_key_info_size - SALT_SIZE));
402 memcpy(kwr->u.rxhdr.rxsalt, kctx->rx.salt, SALT_SIZE);
408 prepare_txkey_wr(struct tls_keyctx *kwr, struct tls_key_context *kctx)
410 unsigned int ck_size = kctx->cipher_secret_size;
411 unsigned int mk_size = kctx->mac_secret_size;
413 kwr->u.txhdr.ctxlen =
414 (kctx->tx_key_info_size >> 4);
415 kwr->u.txhdr.dualck_to_txvalid =
416 V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1) |
417 V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1) |
418 V_TLS_KEYCTX_TX_WR_TXCK_SIZE(get_cipher_key_size(ck_size)) |
419 V_TLS_KEYCTX_TX_WR_TXMK_SIZE(get_mac_key_size(mk_size)) |
420 V_TLS_KEYCTX_TX_WR_TXVALID(1);
422 memcpy(kwr->keys.edkey, kctx->tx.key, HDR_KCTX_SIZE);
423 if (kctx->state.enc_mode == CH_EVP_CIPH_GCM_MODE) {
424 memcpy(kwr->u.txhdr.txsalt, kctx->tx.salt, SALT_SIZE);
425 kwr->u.txhdr.dualck_to_txvalid &=
426 ~(V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1));
428 kwr->u.txhdr.dualck_to_txvalid = htons(kwr->u.txhdr.dualck_to_txvalid);
431 /* TLS Key memory management */
433 tls_init_kmap(struct adapter *sc, struct tom_data *td)
436 td->key_map = vmem_create("T4TLS key map", sc->vres.key.start,
437 sc->vres.key.size, 8, 0, M_FIRSTFIT | M_NOWAIT);
438 if (td->key_map == NULL)
444 tls_free_kmap(struct tom_data *td)
447 if (td->key_map != NULL)
448 vmem_destroy(td->key_map);
452 get_new_keyid(struct toepcb *toep, struct tls_key_context *k_ctx)
454 struct tom_data *td = toep->td;
457 if (vmem_alloc(td->key_map, TLS_KEY_CONTEXT_SZ, M_NOWAIT | M_FIRSTFIT,
465 free_keyid(struct toepcb *toep, int keyid)
467 struct tom_data *td = toep->td;
469 vmem_free(td->key_map, keyid, TLS_KEY_CONTEXT_SZ);
473 clear_tls_keyid(struct toepcb *toep)
475 struct tls_ofld_info *tls_ofld = &toep->tls;
477 if (tls_ofld->rx_key_addr >= 0) {
478 free_keyid(toep, tls_ofld->rx_key_addr);
479 tls_ofld->rx_key_addr = -1;
481 if (tls_ofld->tx_key_addr >= 0) {
482 free_keyid(toep, tls_ofld->tx_key_addr);
483 tls_ofld->tx_key_addr = -1;
488 get_keyid(struct tls_ofld_info *tls_ofld, unsigned int ops)
490 return (ops & KEY_WRITE_RX ? tls_ofld->rx_key_addr :
491 ((ops & KEY_WRITE_TX) ? tls_ofld->tx_key_addr : -1));
495 get_tp_plen_max(struct tls_ofld_info *tls_ofld)
497 int plen = ((min(3*4096, TP_TX_PG_SZ))/1448) * 1448;
499 return (tls_ofld->k_ctx.frag_size <= 8192 ? plen : FC_TP_PLEN_MAX);
502 /* Send request to get the key-id */
504 tls_program_key_id(struct toepcb *toep, struct tls_key_context *k_ctx)
506 struct tls_ofld_info *tls_ofld = &toep->tls;
507 struct adapter *sc = td_adapter(toep->td);
508 struct ofld_tx_sdesc *txsd;
509 int kwrlen, kctxlen, keyid, len;
511 struct tls_key_req *kwr;
512 struct tls_keyctx *kctx;
514 kwrlen = roundup2(sizeof(*kwr), 16);
515 kctxlen = roundup2(sizeof(*kctx), 32);
516 len = kwrlen + kctxlen;
518 if (toep->txsd_avail == 0)
521 /* Dont initialize key for re-neg */
522 if (!G_KEY_CLR_LOC(k_ctx->l_p_key)) {
523 if ((keyid = get_new_keyid(toep, k_ctx)) < 0) {
527 keyid = get_keyid(tls_ofld, k_ctx->l_p_key);
530 wr = alloc_wrqe(len, toep->ofld_txq);
532 free_keyid(toep, keyid);
536 memset(kwr, 0, kwrlen);
538 kwr->wr_hi = htobe32(V_FW_WR_OP(FW_ULPTX_WR) | F_FW_WR_COMPL |
540 kwr->wr_mid = htobe32(V_FW_WR_LEN16(DIV_ROUND_UP(len, 16)) |
541 V_FW_WR_FLOWID(toep->tid));
542 kwr->protocol = get_proto_ver(k_ctx->proto_ver);
543 kwr->mfs = htons(k_ctx->frag_size);
544 kwr->reneg_to_write_rx = k_ctx->l_p_key;
547 kwr->cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE) |
548 V_T5_ULP_MEMIO_ORDER(1) | V_T5_ULP_MEMIO_IMM(1));
549 kwr->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(kctxlen >> 5));
550 kwr->len16 = htobe32((toep->tid << 8) |
551 DIV_ROUND_UP(len - sizeof(struct work_request_hdr), 16));
552 kwr->kaddr = htobe32(V_ULP_MEMIO_ADDR(keyid >> 5));
555 kwr->sc_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
556 kwr->sc_len = htobe32(kctxlen);
558 /* XXX: This assumes that kwrlen == sizeof(*kwr). */
559 kctx = (struct tls_keyctx *)(kwr + 1);
560 memset(kctx, 0, kctxlen);
562 if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_TX) {
563 tls_ofld->tx_key_addr = keyid;
564 prepare_txkey_wr(kctx, k_ctx);
565 } else if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) {
566 tls_ofld->rx_key_addr = keyid;
567 prepare_rxkey_wr(kctx, k_ctx);
570 txsd = &toep->txsd[toep->txsd_pidx];
571 txsd->tx_credits = DIV_ROUND_UP(len, 16);
573 toep->tx_credits -= txsd->tx_credits;
574 if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
583 /* Store a key received from SSL in DDR. */
585 program_key_context(struct tcpcb *tp, struct toepcb *toep,
586 struct tls_key_context *uk_ctx)
588 struct adapter *sc = td_adapter(toep->td);
589 struct tls_ofld_info *tls_ofld = &toep->tls;
590 struct tls_key_context *k_ctx;
591 int error, key_offset;
593 if (tp->t_state != TCPS_ESTABLISHED) {
595 * XXX: Matches Linux driver, but not sure this is a
596 * very appropriate error.
601 /* Stop timer on handshake completion */
602 tls_stop_handshake_timer(toep);
604 toep->flags &= ~TPF_FORCE_CREDITS;
606 CTR4(KTR_CXGBE, "%s: tid %d %s proto_ver %#x", __func__, toep->tid,
607 G_KEY_GET_LOC(uk_ctx->l_p_key) == KEY_WRITE_RX ? "KEY_WRITE_RX" :
608 "KEY_WRITE_TX", uk_ctx->proto_ver);
610 if (G_KEY_GET_LOC(uk_ctx->l_p_key) == KEY_WRITE_RX &&
611 toep->ulp_mode != ULP_MODE_TLS)
614 /* Don't copy the 'tx' and 'rx' fields. */
615 k_ctx = &tls_ofld->k_ctx;
616 memcpy(&k_ctx->l_p_key, &uk_ctx->l_p_key,
617 sizeof(*k_ctx) - offsetof(struct tls_key_context, l_p_key));
619 /* TLS version != 1.1 and !1.2 OR DTLS != 1.2 */
620 if (get_proto_ver(k_ctx->proto_ver) > DTLS_1_2_VERSION) {
621 if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) {
622 tls_ofld->rx_key_addr = -1;
623 t4_clear_rx_quiesce(toep);
625 tls_ofld->tx_key_addr = -1;
630 if (k_ctx->state.enc_mode == CH_EVP_CIPH_GCM_MODE) {
632 k_ctx->mac_first = 0;
633 k_ctx->hmac_ctrl = 0;
635 k_ctx->iv_size = 8; /* for CBC, iv is 16B, unit of 2B */
636 k_ctx->mac_first = 1;
639 tls_ofld->scmd0.seqno_numivs =
640 (V_SCMD_SEQ_NO_CTRL(3) |
641 V_SCMD_PROTO_VERSION(get_proto_ver(k_ctx->proto_ver)) |
642 V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) |
643 V_SCMD_CIPH_AUTH_SEQ_CTRL((k_ctx->mac_first == 0)) |
644 V_SCMD_CIPH_MODE(k_ctx->state.enc_mode) |
645 V_SCMD_AUTH_MODE(k_ctx->state.auth_mode) |
646 V_SCMD_HMAC_CTRL(k_ctx->hmac_ctrl) |
647 V_SCMD_IV_SIZE(k_ctx->iv_size));
649 tls_ofld->scmd0.ivgen_hdrlen =
650 (V_SCMD_IV_GEN_CTRL(k_ctx->iv_ctrl) |
651 V_SCMD_KEY_CTX_INLINE(0) |
652 V_SCMD_TLS_FRAG_ENABLE(1));
654 tls_ofld->mac_length = k_ctx->mac_secret_size;
656 if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) {
657 k_ctx->rx = uk_ctx->rx;
658 /* Dont initialize key for re-neg */
659 if (!G_KEY_CLR_LOC(k_ctx->l_p_key))
660 tls_ofld->rx_key_addr = -1;
662 k_ctx->tx = uk_ctx->tx;
663 /* Dont initialize key for re-neg */
664 if (!G_KEY_CLR_LOC(k_ctx->l_p_key))
665 tls_ofld->tx_key_addr = -1;
668 /* Flush pending data before new Tx key becomes active */
669 if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_TX) {
672 /* XXX: This might not drain everything. */
673 t4_push_frames(sc, toep, 0);
674 sb = &toep->inp->inp_socket->so_snd;
677 /* XXX: This asserts that everything has been pushed. */
678 MPASS(sb->sb_sndptr == NULL || sb->sb_sndptr->m_next == NULL);
679 sb->sb_sndptr = NULL;
680 tls_ofld->sb_off = sbavail(sb);
682 tls_ofld->tx_seq_no = 0;
685 if ((G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) ||
686 (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_DDR)) {
687 error = tls_program_key_id(toep, k_ctx);
689 /* XXX: Only clear quiesce for KEY_WRITE_RX? */
690 t4_clear_rx_quiesce(toep);
695 if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) {
697 * RX key tags are an index into the key portion of MA
698 * memory stored as an offset from the base address in
701 key_offset = tls_ofld->rx_key_addr - sc->vres.key.start;
702 t4_set_tls_keyid(toep, key_offset / 64);
703 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW,
704 V_TCB_ULP_RAW(M_TCB_ULP_RAW),
705 V_TCB_ULP_RAW((V_TF_TLS_KEY_SIZE(3) |
706 V_TF_TLS_CONTROL(1) |
708 V_TF_TLS_ENABLE(1))));
709 t4_set_tls_tcb_field(toep, W_TCB_TLS_SEQ,
710 V_TCB_TLS_SEQ(M_TCB_TLS_SEQ),
712 t4_clear_rx_quiesce(toep);
714 unsigned short pdus_per_ulp;
716 if (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_IMMEDIATE)
717 tls_ofld->tx_key_addr = 1;
719 tls_ofld->fcplenmax = get_tp_plen_max(tls_ofld);
720 tls_ofld->expn_per_ulp = tls_expansion_size(toep,
721 tls_ofld->fcplenmax, 1, &pdus_per_ulp);
722 tls_ofld->pdus_per_ulp = pdus_per_ulp;
723 tls_ofld->adjusted_plen = tls_ofld->pdus_per_ulp *
724 ((tls_ofld->expn_per_ulp/tls_ofld->pdus_per_ulp) +
725 tls_ofld->k_ctx.frag_size);
732 * In some cases a client connection can hang without sending the
733 * ServerHelloDone message from the NIC to the host. Send a dummy
734 * RX_DATA_ACK with RX_MODULATE to unstick the connection.
737 tls_send_handshake_ack(void *arg)
739 struct toepcb *toep = arg;
740 struct tls_ofld_info *tls_ofld = &toep->tls;
741 struct adapter *sc = td_adapter(toep->td);
744 * XXX: Does not have the t4_get_tcb() checks to refine the
747 callout_schedule(&tls_ofld->handshake_timer, TLS_SRV_HELLO_RD_TM * hz);
749 CTR2(KTR_CXGBE, "%s: tid %d sending RX_DATA_ACK", __func__, toep->tid);
750 send_rx_modulate(sc, toep);
754 tls_start_handshake_timer(struct toepcb *toep)
756 struct tls_ofld_info *tls_ofld = &toep->tls;
758 mtx_lock(&tls_handshake_lock);
759 callout_reset(&tls_ofld->handshake_timer, TLS_SRV_HELLO_BKOFF_TM * hz,
760 tls_send_handshake_ack, toep);
761 mtx_unlock(&tls_handshake_lock);
765 tls_stop_handshake_timer(struct toepcb *toep)
767 struct tls_ofld_info *tls_ofld = &toep->tls;
769 mtx_lock(&tls_handshake_lock);
770 callout_stop(&tls_ofld->handshake_timer);
771 mtx_unlock(&tls_handshake_lock);
775 t4_ctloutput_tls(struct socket *so, struct sockopt *sopt)
777 struct tls_key_context uk_ctx;
784 if (sopt->sopt_dir == SOPT_SET &&
785 sopt->sopt_name == TCP_TLSOM_SET_TLS_CONTEXT) {
786 error = sooptcopyin(sopt, &uk_ctx, sizeof(uk_ctx),
793 KASSERT(inp != NULL, ("tcp_ctloutput: inp == NULL"));
795 if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
801 switch (sopt->sopt_dir) {
803 switch (sopt->sopt_name) {
804 case TCP_TLSOM_SET_TLS_CONTEXT:
805 error = program_key_context(tp, toep, &uk_ctx);
808 case TCP_TLSOM_CLR_TLS_TOM:
809 if (toep->ulp_mode == ULP_MODE_TLS) {
810 CTR2(KTR_CXGBE, "%s: tid %d CLR_TLS_TOM",
811 __func__, toep->tid);
812 tls_clr_ofld_mode(toep);
817 case TCP_TLSOM_CLR_QUIES:
818 if (toep->ulp_mode == ULP_MODE_TLS) {
819 CTR2(KTR_CXGBE, "%s: tid %d CLR_QUIES",
820 __func__, toep->tid);
821 tls_clr_quiesce(toep);
833 switch (sopt->sopt_name) {
834 case TCP_TLSOM_GET_TLS_TOM:
836 * TLS TX is permitted on any TOE socket, but
837 * TLS RX requires a TLS ULP mode.
839 optval = TLS_TOM_NONE;
840 if (can_tls_offload(td_adapter(toep->td))) {
841 switch (toep->ulp_mode) {
843 case ULP_MODE_TCPDDP:
844 optval = TLS_TOM_TXONLY;
847 optval = TLS_TOM_BOTH;
851 CTR3(KTR_CXGBE, "%s: tid %d GET_TLS_TOM = %d",
852 __func__, toep->tid, optval);
854 error = sooptcopyout(sopt, &optval, sizeof(optval));
867 tls_init_toep(struct toepcb *toep)
869 struct tls_ofld_info *tls_ofld = &toep->tls;
871 tls_ofld->key_location = TLS_SFO_WR_CONTEXTLOC_DDR;
872 tls_ofld->rx_key_addr = -1;
873 tls_ofld->tx_key_addr = -1;
874 if (toep->ulp_mode == ULP_MODE_TLS)
875 callout_init_mtx(&tls_ofld->handshake_timer,
876 &tls_handshake_lock, 0);
880 tls_establish(struct toepcb *toep)
884 * Enable PDU extraction.
886 * XXX: Supposedly this should be done by the firmware when
887 * the ULP_MODE FLOWC parameter is set in send_flowc_wr(), but
888 * in practice this seems to be required.
890 CTR2(KTR_CXGBE, "%s: tid %d setting TLS_ENABLE", __func__, toep->tid);
891 t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, V_TCB_ULP_RAW(M_TCB_ULP_RAW),
892 V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1)));
894 toep->flags |= TPF_FORCE_CREDITS;
896 tls_start_handshake_timer(toep);
900 tls_uninit_toep(struct toepcb *toep)
903 if (toep->ulp_mode == ULP_MODE_TLS)
904 tls_stop_handshake_timer(toep);
905 clear_tls_keyid(toep);
908 #define MAX_OFLD_TX_CREDITS (SGE_MAX_WR_LEN / 16)
909 #define MIN_OFLD_TLSTX_CREDITS(toep) \
910 (howmany(sizeof(struct fw_tlstx_data_wr) + \
911 sizeof(struct cpl_tx_tls_sfo) + key_size((toep)) + \
912 CIPHER_BLOCK_SIZE + 1, 16))
915 max_imm_tls_space(int tx_credits)
917 const int n = 2; /* Use only up to 2 desc for imm. data WR */
920 KASSERT(tx_credits >= 0 &&
921 tx_credits <= MAX_OFLD_TX_CREDITS,
922 ("%s: %d credits", __func__, tx_credits));
924 if (tx_credits >= (n * EQ_ESIZE) / 16)
925 space = (n * EQ_ESIZE);
927 space = tx_credits * 16;
932 count_mbuf_segs(struct mbuf *m, int skip, int len, int *max_nsegs_1mbufp)
934 int max_nsegs_1mbuf, n, nsegs;
936 while (skip >= m->m_len) {
944 n = sglist_count(mtod(m, char *) + skip, m->m_len - skip);
945 if (n > max_nsegs_1mbuf)
948 len -= m->m_len - skip;
952 *max_nsegs_1mbufp = max_nsegs_1mbuf;
957 write_tlstx_wr(struct fw_tlstx_data_wr *txwr, struct toepcb *toep,
958 unsigned int immdlen, unsigned int plen, unsigned int expn,
959 unsigned int pdus, uint8_t credits, int shove, int imm_ivs)
961 struct tls_ofld_info *tls_ofld = &toep->tls;
962 unsigned int len = plen + expn;
964 txwr->op_to_immdlen = htobe32(V_WR_OP(FW_TLSTX_DATA_WR) |
965 V_FW_TLSTX_DATA_WR_COMPL(1) |
966 V_FW_TLSTX_DATA_WR_IMMDLEN(immdlen));
967 txwr->flowid_len16 = htobe32(V_FW_TLSTX_DATA_WR_FLOWID(toep->tid) |
968 V_FW_TLSTX_DATA_WR_LEN16(credits));
969 txwr->plen = htobe32(len);
970 txwr->lsodisable_to_flags = htobe32(V_TX_ULP_MODE(ULP_MODE_TLS) |
971 V_TX_URG(0) | /* F_T6_TX_FORCE | */ V_TX_SHOVE(shove));
972 txwr->ctxloc_to_exp = htobe32(V_FW_TLSTX_DATA_WR_NUMIVS(pdus) |
973 V_FW_TLSTX_DATA_WR_EXP(expn) |
974 V_FW_TLSTX_DATA_WR_CTXLOC(tls_ofld->key_location) |
975 V_FW_TLSTX_DATA_WR_IVDSGL(!imm_ivs) |
976 V_FW_TLSTX_DATA_WR_KEYSIZE(tls_ofld->k_ctx.tx_key_info_size >> 4));
977 txwr->mfs = htobe16(tls_ofld->k_ctx.frag_size);
978 txwr->adjustedplen_pkd = htobe16(
979 V_FW_TLSTX_DATA_WR_ADJUSTEDPLEN(tls_ofld->adjusted_plen));
980 txwr->expinplenmax_pkd = htobe16(
981 V_FW_TLSTX_DATA_WR_EXPINPLENMAX(tls_ofld->expn_per_ulp));
982 txwr->pdusinplenmax_pkd = htobe16(
983 V_FW_TLSTX_DATA_WR_PDUSINPLENMAX(tls_ofld->pdus_per_ulp));
987 write_tlstx_cpl(struct cpl_tx_tls_sfo *cpl, struct toepcb *toep,
988 struct tls_hdr *tls_hdr, unsigned int plen, unsigned int pdus)
990 struct tls_ofld_info *tls_ofld = &toep->tls;
991 int data_type, seglen;
993 if (plen < tls_ofld->k_ctx.frag_size)
996 seglen = tls_ofld->k_ctx.frag_size;
997 data_type = tls_content_type(tls_hdr->type);
998 cpl->op_to_seg_len = htobe32(V_CPL_TX_TLS_SFO_OPCODE(CPL_TX_TLS_SFO) |
999 V_CPL_TX_TLS_SFO_DATA_TYPE(data_type) |
1000 V_CPL_TX_TLS_SFO_CPL_LEN(2) | V_CPL_TX_TLS_SFO_SEG_LEN(seglen));
1001 cpl->pld_len = htobe32(plen);
1002 if (data_type == CPL_TX_TLS_SFO_TYPE_HEARTBEAT)
1003 cpl->type_protover = htobe32(
1004 V_CPL_TX_TLS_SFO_TYPE(tls_hdr->type));
1005 cpl->seqno_numivs = htobe32(tls_ofld->scmd0.seqno_numivs |
1006 V_SCMD_NUM_IVS(pdus));
1007 cpl->ivgen_hdrlen = htobe32(tls_ofld->scmd0.ivgen_hdrlen);
1008 cpl->scmd1 = htobe64(tls_ofld->tx_seq_no);
1009 tls_ofld->tx_seq_no += pdus;
1013 * Similar to write_tx_sgl() except that it accepts an optional
1014 * trailer buffer for IVs.
1017 write_tlstx_sgl(void *dst, struct mbuf *start, int skip, int plen,
1018 void *iv_buffer, int iv_len, int nsegs, int n)
1021 struct ulptx_sgl *usgl = dst;
1024 struct sglist_seg segs[n];
1026 KASSERT(nsegs > 0, ("%s: nsegs 0", __func__));
1028 sglist_init(&sg, n, segs);
1029 usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
1030 V_ULPTX_NSGE(nsegs));
1032 for (m = start; skip >= m->m_len; m = m->m_next)
1036 for (m = start; plen > 0; m = m->m_next) {
1037 rc = sglist_append(&sg, mtod(m, char *) + skip,
1039 if (__predict_false(rc != 0))
1040 panic("%s: sglist_append %d", __func__, rc);
1041 plen -= m->m_len - skip;
1044 for (j = 0; j < sg.sg_nseg; i++, j++) {
1046 usgl->len0 = htobe32(segs[j].ss_len);
1047 usgl->addr0 = htobe64(segs[j].ss_paddr);
1049 usgl->sge[i / 2].len[i & 1] =
1050 htobe32(segs[j].ss_len);
1051 usgl->sge[i / 2].addr[i & 1] =
1052 htobe64(segs[j].ss_paddr);
1060 if (iv_buffer != NULL) {
1061 rc = sglist_append(&sg, iv_buffer, iv_len);
1062 if (__predict_false(rc != 0))
1063 panic("%s: sglist_append %d", __func__, rc);
1065 for (j = 0; j < sg.sg_nseg; i++, j++) {
1067 usgl->len0 = htobe32(segs[j].ss_len);
1068 usgl->addr0 = htobe64(segs[j].ss_paddr);
1070 usgl->sge[i / 2].len[i & 1] =
1071 htobe32(segs[j].ss_len);
1072 usgl->sge[i / 2].addr[i & 1] =
1073 htobe64(segs[j].ss_paddr);
1081 usgl->sge[i / 2].len[1] = htobe32(0);
1082 KASSERT(nsegs == 0, ("%s: nsegs %d, start %p, iv_buffer %p",
1083 __func__, nsegs, start, iv_buffer));
1087 * Similar to t4_push_frames() but handles TLS sockets when TLS offload
1088 * is enabled. Rather than transmitting bulk data, the socket buffer
1089 * contains TLS records. The work request requires a full TLS record,
1090 * so batch mbufs up until a full TLS record is seen. This requires
1091 * reading the TLS header out of the start of each record to determine
1095 t4_push_tls_records(struct adapter *sc, struct toepcb *toep, int drop)
1097 struct tls_hdr thdr;
1098 struct mbuf *sndptr;
1099 struct fw_tlstx_data_wr *txwr;
1100 struct cpl_tx_tls_sfo *cpl;
1102 u_int plen, nsegs, credits, space, max_nsegs_1mbuf, wr_len;
1103 u_int expn_size, iv_len, pdus, sndptroff;
1104 struct tls_ofld_info *tls_ofld = &toep->tls;
1105 struct inpcb *inp = toep->inp;
1106 struct tcpcb *tp = intotcpcb(inp);
1107 struct socket *so = inp->inp_socket;
1108 struct sockbuf *sb = &so->so_snd;
1109 int tls_size, tx_credits, shove, /* compl,*/ sowwakeup;
1110 struct ofld_tx_sdesc *txsd;
1111 bool imm_ivs, imm_payload;
1112 void *iv_buffer, *iv_dst, *buf;
1114 INP_WLOCK_ASSERT(inp);
1115 KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
1116 ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
1118 KASSERT(toep->ulp_mode == ULP_MODE_NONE ||
1119 toep->ulp_mode == ULP_MODE_TCPDDP || toep->ulp_mode == ULP_MODE_TLS,
1120 ("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep));
1121 KASSERT(tls_tx_key(toep),
1122 ("%s: TX key not set for toep %p", __func__, toep));
1124 #ifdef VERBOSE_TRACES
1125 CTR4(KTR_CXGBE, "%s: tid %d toep flags %#x tp flags %#x drop %d",
1126 __func__, toep->tid, toep->flags, tp->t_flags);
1128 if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN))
1132 if (__predict_false(inp->inp_flags2 & INP_RATE_LIMIT_CHANGED) &&
1133 (update_tx_rate_limit(sc, toep, so->so_max_pacing_rate) == 0)) {
1134 inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED;
1139 * This function doesn't resume by itself. Someone else must clear the
1140 * flag and call this function.
1142 if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) {
1144 ("%s: drop (%d) != 0 but tx is suspended", __func__, drop));
1148 txsd = &toep->txsd[toep->txsd_pidx];
1150 tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS);
1151 space = max_imm_tls_space(tx_credits);
1152 wr_len = sizeof(struct fw_tlstx_data_wr) +
1153 sizeof(struct cpl_tx_tls_sfo) + key_size(toep);
1154 if (wr_len + CIPHER_BLOCK_SIZE + 1 > space) {
1155 #ifdef VERBOSE_TRACES
1157 "%s: tid %d tx_credits %d min_wr %d space %d",
1158 __func__, toep->tid, tx_credits, wr_len +
1159 CIPHER_BLOCK_SIZE + 1, space);
1167 sbdrop_locked(sb, drop);
1168 MPASS(tls_ofld->sb_off >= drop);
1169 tls_ofld->sb_off -= drop;
1174 * Send a FIN if requested, but only if there's no
1175 * more data to send.
1177 if (sbavail(sb) == tls_ofld->sb_off &&
1178 toep->flags & TPF_SEND_FIN) {
1180 sowwakeup_locked(so);
1183 SOCKBUF_UNLOCK_ASSERT(sb);
1184 t4_close_conn(sc, toep);
1188 if (sbavail(sb) < tls_ofld->sb_off + TLS_HEADER_LENGTH) {
1190 * A full TLS header is not yet queued, stop
1191 * for now until more data is added to the
1192 * socket buffer. However, if the connection
1193 * has been closed, we will never get the rest
1194 * of the header so just discard the partial
1195 * header and close the connection.
1197 #ifdef VERBOSE_TRACES
1198 CTR5(KTR_CXGBE, "%s: tid %d sbavail %d sb_off %d%s",
1199 __func__, toep->tid, sbavail(sb), tls_ofld->sb_off,
1200 toep->flags & TPF_SEND_FIN ? "" : " SEND_FIN");
1203 sowwakeup_locked(so);
1206 SOCKBUF_UNLOCK_ASSERT(sb);
1207 if (toep->flags & TPF_SEND_FIN)
1208 t4_close_conn(sc, toep);
1212 /* Read the header of the next TLS record. */
1213 sndptr = sbsndmbuf(sb, tls_ofld->sb_off, &sndptroff);
1214 MPASS(!IS_AIOTX_MBUF(sndptr));
1215 m_copydata(sndptr, sndptroff, sizeof(thdr), (caddr_t)&thdr);
1216 tls_size = htons(thdr.length);
1217 plen = TLS_HEADER_LENGTH + tls_size;
1218 pdus = howmany(tls_size, tls_ofld->k_ctx.frag_size);
1219 iv_len = pdus * CIPHER_BLOCK_SIZE;
1221 if (sbavail(sb) < tls_ofld->sb_off + plen) {
1223 * The full TLS record is not yet queued, stop
1224 * for now until more data is added to the
1225 * socket buffer. However, if the connection
1226 * has been closed, we will never get the rest
1227 * of the record so just discard the partial
1228 * record and close the connection.
1230 #ifdef VERBOSE_TRACES
1232 "%s: tid %d sbavail %d sb_off %d plen %d%s",
1233 __func__, toep->tid, sbavail(sb), tls_ofld->sb_off,
1234 plen, toep->flags & TPF_SEND_FIN ? "" :
1238 sowwakeup_locked(so);
1241 SOCKBUF_UNLOCK_ASSERT(sb);
1242 if (toep->flags & TPF_SEND_FIN)
1243 t4_close_conn(sc, toep);
1247 /* Shove if there is no additional data pending. */
1248 shove = (sbavail(sb) == tls_ofld->sb_off + plen) &&
1249 !(tp->t_flags & TF_MORETOCOME);
1251 if (sb->sb_flags & SB_AUTOSIZE &&
1252 V_tcp_do_autosndbuf &&
1253 sb->sb_hiwat < V_tcp_autosndbuf_max &&
1254 sbused(sb) >= sb->sb_hiwat * 7 / 8) {
1255 int newsize = min(sb->sb_hiwat + V_tcp_autosndbuf_inc,
1256 V_tcp_autosndbuf_max);
1258 if (!sbreserve_locked(sb, newsize, so, NULL))
1259 sb->sb_flags &= ~SB_AUTOSIZE;
1261 sowwakeup = 1; /* room available */
1264 sowwakeup_locked(so);
1267 SOCKBUF_UNLOCK_ASSERT(sb);
1269 if (__predict_false(toep->flags & TPF_FIN_SENT))
1270 panic("%s: excess tx.", __func__);
1272 /* Determine whether to use immediate vs SGL. */
1273 imm_payload = false;
1275 if (wr_len + iv_len <= space) {
1278 if (wr_len + tls_size <= space) {
1284 /* Allocate space for IVs if needed. */
1286 iv_buffer = malloc(iv_len, M_CXGBE, M_NOWAIT);
1287 if (iv_buffer == NULL) {
1289 * XXX: How to restart this?
1292 sowwakeup_locked(so);
1295 SOCKBUF_UNLOCK_ASSERT(sb);
1297 "%s: tid %d failed to alloc IV space len %d",
1298 __func__, toep->tid, iv_len);
1304 /* Determine size of SGL. */
1306 max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */
1308 nsegs = count_mbuf_segs(sndptr, sndptroff +
1309 TLS_HEADER_LENGTH, tls_size, &max_nsegs_1mbuf);
1311 int n = sglist_count(iv_buffer, iv_len);
1313 if (n > max_nsegs_1mbuf)
1314 max_nsegs_1mbuf = n;
1317 /* Account for SGL in work request length. */
1318 wr_len += sizeof(struct ulptx_sgl) +
1319 ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
1322 wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq);
1324 /* XXX: how will we recover from this? */
1325 toep->flags |= TPF_TX_SUSPENDED;
1329 #ifdef VERBOSE_TRACES
1330 CTR5(KTR_CXGBE, "%s: tid %d TLS record %d len %#x pdus %d",
1331 __func__, toep->tid, thdr.type, tls_size, pdus);
1334 cpl = (struct cpl_tx_tls_sfo *)(txwr + 1);
1335 memset(txwr, 0, roundup2(wr_len, 16));
1336 credits = howmany(wr_len, 16);
1337 expn_size = tls_expansion_size(toep, tls_size, 0, NULL);
1338 write_tlstx_wr(txwr, toep, imm_payload ? tls_size : 0,
1339 tls_size, expn_size, pdus, credits, shove, imm_ivs ? 1 : 0);
1340 write_tlstx_cpl(cpl, toep, &thdr, tls_size, pdus);
1341 tls_copy_tx_key(toep, cpl + 1);
1343 /* Generate random IVs */
1344 buf = (char *)(cpl + 1) + key_size(toep);
1346 MPASS(iv_buffer == NULL);
1348 buf = (char *)iv_dst + iv_len;
1351 arc4rand(iv_dst, iv_len, 0);
1354 m_copydata(sndptr, sndptroff + TLS_HEADER_LENGTH,
1357 write_tlstx_sgl(buf, sndptr,
1358 sndptroff + TLS_HEADER_LENGTH, tls_size, iv_buffer,
1359 iv_len, nsegs, max_nsegs_1mbuf);
1362 KASSERT(toep->tx_credits >= credits,
1363 ("%s: not enough credits", __func__));
1365 toep->tx_credits -= credits;
1367 tp->snd_nxt += plen;
1368 tp->snd_max += plen;
1371 sbsndptr_adv(sb, sb->sb_sndptr, plen);
1372 tls_ofld->sb_off += plen;
1375 toep->flags |= TPF_TX_DATA_SENT;
1376 if (toep->tx_credits < MIN_OFLD_TLSTX_CREDITS(toep))
1377 toep->flags |= TPF_TX_SUSPENDED;
1379 KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__));
1381 txsd->tx_credits = credits;
1382 txsd->iv_buffer = iv_buffer;
1384 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) {
1385 toep->txsd_pidx = 0;
1386 txsd = &toep->txsd[0];
1390 atomic_add_long(&toep->vi->pi->tx_tls_records, 1);
1391 atomic_add_long(&toep->vi->pi->tx_tls_octets, plen);
1393 t4_l2t_send(sc, wr, toep->l2te);
1398 * For TLS data we place received mbufs received via CPL_TLS_DATA into
1399 * an mbufq in the TLS offload state. When CPL_RX_TLS_CMP is
1400 * received, the completed PDUs are placed into the socket receive
1403 * The TLS code reuses the ulp_pdu_reclaimq to hold the pending mbufs.
1406 do_tls_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
1408 struct adapter *sc = iq->adapter;
1409 const struct cpl_tls_data *cpl = mtod(m, const void *);
1410 unsigned int tid = GET_TID(cpl);
1411 struct toepcb *toep = lookup_tid(sc, tid);
1412 struct inpcb *inp = toep->inp;
1416 /* XXX: Should this match do_rx_data instead? */
1417 KASSERT(!(toep->flags & TPF_SYNQE),
1418 ("%s: toep %p claims to be a synq entry", __func__, toep));
1420 KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__));
1422 /* strip off CPL header */
1423 m_adj(m, sizeof(*cpl));
1424 len = m->m_pkthdr.len;
1426 atomic_add_long(&toep->vi->pi->rx_tls_octets, len);
1428 KASSERT(len == G_CPL_TLS_DATA_LENGTH(be32toh(cpl->length_pkd)),
1429 ("%s: payload length mismatch", __func__));
1432 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) {
1433 CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
1434 __func__, tid, len, inp->inp_flags);
1440 /* Save TCP sequence number. */
1441 m->m_pkthdr.tls_tcp_seq = be32toh(cpl->seq);
1443 if (mbufq_enqueue(&toep->ulp_pdu_reclaimq, m)) {
1445 panic("Failed to queue TLS data packet");
1447 printf("%s: Failed to queue TLS data packet\n", __func__);
1454 tp = intotcpcb(inp);
1455 tp->t_rcvtime = ticks;
1457 #ifdef VERBOSE_TRACES
1458 CTR4(KTR_CXGBE, "%s: tid %u len %d seq %u", __func__, tid, len,
1467 do_rx_tls_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
1469 struct adapter *sc = iq->adapter;
1470 const struct cpl_rx_tls_cmp *cpl = mtod(m, const void *);
1471 struct tlsrx_hdr_pkt *tls_hdr_pkt;
1472 unsigned int tid = GET_TID(cpl);
1473 struct toepcb *toep = lookup_tid(sc, tid);
1474 struct inpcb *inp = toep->inp;
1478 struct mbuf *tls_data;
1479 int len, pdu_length, pdu_overhead, sb_length;
1481 KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__));
1482 KASSERT(!(toep->flags & TPF_SYNQE),
1483 ("%s: toep %p claims to be a synq entry", __func__, toep));
1485 /* strip off CPL header */
1486 m_adj(m, sizeof(*cpl));
1487 len = m->m_pkthdr.len;
1489 atomic_add_long(&toep->vi->pi->rx_tls_records, 1);
1491 KASSERT(len == G_CPL_RX_TLS_CMP_LENGTH(be32toh(cpl->pdulength_length)),
1492 ("%s: payload length mismatch", __func__));
1495 if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) {
1496 CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
1497 __func__, tid, len, inp->inp_flags);
1503 pdu_length = G_CPL_RX_TLS_CMP_PDULENGTH(be32toh(cpl->pdulength_length));
1505 tp = intotcpcb(inp);
1507 #ifdef VERBOSE_TRACES
1508 CTR6(KTR_CXGBE, "%s: tid %u PDU len %d len %d seq %u, rcv_nxt %u",
1509 __func__, tid, pdu_length, len, be32toh(cpl->seq), tp->rcv_nxt);
1512 tp->rcv_nxt += pdu_length;
1513 if (tp->rcv_wnd < pdu_length) {
1514 toep->tls.rcv_over += pdu_length - tp->rcv_wnd;
1517 tp->rcv_wnd -= pdu_length;
1519 /* XXX: Not sure what to do about urgent data. */
1522 * The payload of this CPL is the TLS header followed by
1523 * additional fields.
1525 KASSERT(m->m_len >= sizeof(*tls_hdr_pkt),
1526 ("%s: payload too small", __func__));
1527 tls_hdr_pkt = mtod(m, void *);
1530 * Only the TLS header is sent to OpenSSL, so report errors by
1531 * altering the record type.
1533 if ((tls_hdr_pkt->res_to_mac_error & M_TLSRX_HDR_PKT_ERROR) != 0)
1534 tls_hdr_pkt->type = CONTENT_TYPE_ERROR;
1536 /* Trim this CPL's mbuf to only include the TLS header. */
1537 KASSERT(m->m_len == len && m->m_next == NULL,
1538 ("%s: CPL spans multiple mbufs", __func__));
1539 m->m_len = TLS_HEADER_LENGTH;
1540 m->m_pkthdr.len = TLS_HEADER_LENGTH;
1542 tls_data = mbufq_dequeue(&toep->ulp_pdu_reclaimq);
1543 if (tls_data != NULL) {
1544 KASSERT(be32toh(cpl->seq) == tls_data->m_pkthdr.tls_tcp_seq,
1545 ("%s: sequence mismatch", __func__));
1548 * Update the TLS header length to be the length of
1551 tls_hdr_pkt->length = htobe16(tls_data->m_pkthdr.len);
1553 m->m_next = tls_data;
1554 m->m_pkthdr.len += tls_data->m_len;
1557 so = inp_inpcbtosocket(inp);
1561 if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) {
1562 struct epoch_tracker et;
1564 CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)",
1565 __func__, tid, pdu_length);
1570 CURVNET_SET(toep->vnet);
1571 INP_INFO_RLOCK_ET(&V_tcbinfo, et);
1573 tp = tcp_drop(tp, ECONNRESET);
1576 INP_INFO_RUNLOCK_ET(&V_tcbinfo, et);
1583 * Not all of the bytes on the wire are included in the socket
1584 * buffer (e.g. the MAC of the TLS record). However, those
1585 * bytes are included in the TCP sequence space. To handle
1586 * this, compute the delta for this TLS record in
1587 * 'pdu_overhead' and treat those bytes as having already been
1588 * "read" by the application for the purposes of expanding the
1589 * window. The meat of the TLS record passed to the
1590 * application ('sb_length') will still not be counted as
1591 * "read" until userland actually reads the bytes.
1593 * XXX: Some of the calculations below are probably still not
1596 sb_length = m->m_pkthdr.len;
1597 pdu_overhead = pdu_length - sb_length;
1598 toep->rx_credits += pdu_overhead;
1599 tp->rcv_wnd += pdu_overhead;
1600 tp->rcv_adv += pdu_overhead;
1602 /* receive buffer autosize */
1603 MPASS(toep->vnet == so->so_vnet);
1604 CURVNET_SET(toep->vnet);
1605 if (sb->sb_flags & SB_AUTOSIZE &&
1606 V_tcp_do_autorcvbuf &&
1607 sb->sb_hiwat < V_tcp_autorcvbuf_max &&
1608 sb_length > (sbspace(sb) / 8 * 7)) {
1609 unsigned int hiwat = sb->sb_hiwat;
1610 unsigned int newsize = min(hiwat + V_tcp_autorcvbuf_inc,
1611 V_tcp_autorcvbuf_max);
1613 if (!sbreserve_locked(sb, newsize, so, NULL))
1614 sb->sb_flags &= ~SB_AUTOSIZE;
1616 toep->rx_credits += newsize - hiwat;
1619 KASSERT(toep->sb_cc >= sbused(sb),
1620 ("%s: sb %p has more data (%d) than last time (%d).",
1621 __func__, sb, sbused(sb), toep->sb_cc));
1622 toep->rx_credits += toep->sb_cc - sbused(sb);
1623 sbappendstream_locked(sb, m, 0);
1624 toep->sb_cc = sbused(sb);
1625 #ifdef VERBOSE_TRACES
1626 CTR5(KTR_CXGBE, "%s: tid %u PDU overhead %d rx_credits %u rcv_wnd %u",
1627 __func__, tid, pdu_overhead, toep->rx_credits, tp->rcv_wnd);
1629 if (toep->rx_credits > 0 && toep->sb_cc + tp->rcv_wnd < sb->sb_lowat) {
1632 credits = send_rx_credits(sc, toep, toep->rx_credits);
1633 toep->rx_credits -= credits;
1634 tp->rcv_wnd += credits;
1635 tp->rcv_adv += credits;
1638 sorwakeup_locked(so);
1639 SOCKBUF_UNLOCK_ASSERT(sb);
1647 t4_tls_mod_load(void)
1650 mtx_init(&tls_handshake_lock, "t4tls handshake", NULL, MTX_DEF);
1651 t4_register_cpl_handler(CPL_TLS_DATA, do_tls_data);
1652 t4_register_cpl_handler(CPL_RX_TLS_CMP, do_rx_tls_cmp);
1656 t4_tls_mod_unload(void)
1659 t4_register_cpl_handler(CPL_TLS_DATA, NULL);
1660 t4_register_cpl_handler(CPL_RX_TLS_CMP, NULL);
1661 mtx_destroy(&tls_handshake_lock);
1663 #endif /* TCP_OFFLOAD */