2 * SPDX-License-Identifier: BSD-2-Clause
4 * Copyright (c) 2019 Netflix Inc.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/counter.h>
35 #include <sys/endian.h>
36 #include <sys/kernel.h>
39 #include <sys/malloc.h>
40 #include <sys/module.h>
41 #include <sys/mutex.h>
42 #include <sys/sysctl.h>
44 #include <opencrypto/cryptodev.h>
51 struct ocf_operation {
52 struct ocf_session *os;
56 static MALLOC_DEFINE(M_KTLS_OCF, "ktls_ocf", "OCF KTLS");
58 SYSCTL_DECL(_kern_ipc_tls);
59 SYSCTL_DECL(_kern_ipc_tls_stats);
61 static SYSCTL_NODE(_kern_ipc_tls_stats, OID_AUTO, ocf,
62 CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
63 "Kernel TLS offload via OCF stats");
65 static counter_u64_t ocf_tls12_gcm_crypts;
66 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_gcm_crypts,
67 CTLFLAG_RD, &ocf_tls12_gcm_crypts,
68 "Total number of OCF TLS 1.2 GCM encryption operations");
70 static counter_u64_t ocf_tls13_gcm_crypts;
71 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_crypts,
72 CTLFLAG_RD, &ocf_tls13_gcm_crypts,
73 "Total number of OCF TLS 1.3 GCM encryption operations");
75 static counter_u64_t ocf_inplace;
76 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, inplace,
77 CTLFLAG_RD, &ocf_inplace,
78 "Total number of OCF in-place operations");
80 static counter_u64_t ocf_separate_output;
81 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, separate_output,
82 CTLFLAG_RD, &ocf_separate_output,
83 "Total number of OCF operations with a separate output buffer");
85 static counter_u64_t ocf_retries;
86 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, retries, CTLFLAG_RD,
88 "Number of OCF encryption operation retries");
91 ktls_ocf_callback(struct cryptop *crp)
93 struct ocf_operation *oo;
96 mtx_lock(&oo->os->lock);
98 mtx_unlock(&oo->os->lock);
104 ktls_ocf_dispatch(struct ocf_session *os, struct cryptop *crp)
106 struct ocf_operation oo;
112 crp->crp_opaque = &oo;
113 crp->crp_callback = ktls_ocf_callback;
115 error = crypto_dispatch(crp);
121 mtx_sleep(&oo, &os->lock, 0, "ocfktls", 0);
122 mtx_unlock(&os->lock);
124 if (crp->crp_etype != EAGAIN) {
125 error = crp->crp_etype;
130 crp->crp_flags &= ~CRYPTO_F_DONE;
132 counter_u64_add(ocf_retries, 1);
138 ktls_ocf_tls12_gcm_encrypt(struct ktls_session *tls,
139 const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov,
140 struct iovec *outiov, int iovcnt, uint64_t seqno,
141 uint8_t record_type __unused)
143 struct uio uio, out_uio, *tag_uio;
144 struct tls_aead_data ad;
146 struct ocf_session *os;
147 struct iovec iov[iovcnt + 1];
149 uint16_t tls_comp_len;
155 uio.uio_iovcnt = iovcnt;
157 uio.uio_segflg = UIO_SYSSPACE;
158 uio.uio_td = curthread;
160 out_uio.uio_iov = outiov;
161 out_uio.uio_iovcnt = iovcnt;
162 out_uio.uio_offset = 0;
163 out_uio.uio_segflg = UIO_SYSSPACE;
164 out_uio.uio_td = curthread;
166 crypto_initreq(&crp, os->sid);
169 memcpy(crp.crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN);
170 memcpy(crp.crp_iv + TLS_AEAD_GCM_LEN, hdr + 1, sizeof(uint64_t));
173 tls_comp_len = ntohs(hdr->tls_length) -
174 (AES_GMAC_HASH_LEN + sizeof(uint64_t));
175 ad.seq = htobe64(seqno);
176 ad.type = hdr->tls_type;
177 ad.tls_vmajor = hdr->tls_vmajor;
178 ad.tls_vminor = hdr->tls_vminor;
179 ad.tls_length = htons(tls_comp_len);
181 crp.crp_aad_length = sizeof(ad);
183 /* Compute payload length and determine if encryption is in place. */
185 crp.crp_payload_start = 0;
186 for (i = 0; i < iovcnt; i++) {
187 if (iniov[i].iov_base != outiov[i].iov_base)
189 crp.crp_payload_length += iniov[i].iov_len;
191 uio.uio_resid = crp.crp_payload_length;
192 out_uio.uio_resid = crp.crp_payload_length;
199 /* Duplicate iovec and append vector for tag. */
200 memcpy(iov, tag_uio->uio_iov, iovcnt * sizeof(struct iovec));
201 iov[iovcnt].iov_base = trailer;
202 iov[iovcnt].iov_len = AES_GMAC_HASH_LEN;
203 tag_uio->uio_iov = iov;
204 tag_uio->uio_iovcnt++;
205 crp.crp_digest_start = tag_uio->uio_resid;
206 tag_uio->uio_resid += AES_GMAC_HASH_LEN;
208 crp.crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST;
209 crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
210 crypto_use_uio(&crp, &uio);
212 crypto_use_output_uio(&crp, &out_uio);
214 counter_u64_add(ocf_tls12_gcm_crypts, 1);
216 counter_u64_add(ocf_inplace, 1);
218 counter_u64_add(ocf_separate_output, 1);
219 error = ktls_ocf_dispatch(os, &crp);
221 crypto_destroyreq(&crp);
226 ktls_ocf_tls12_gcm_decrypt(struct ktls_session *tls,
227 const struct tls_record_layer *hdr, struct mbuf *m, uint64_t seqno,
230 struct tls_aead_data ad;
232 struct ocf_session *os;
233 struct ocf_operation oo;
235 uint16_t tls_comp_len;
242 crypto_initreq(&crp, os->sid);
245 memcpy(crp.crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN);
246 memcpy(crp.crp_iv + TLS_AEAD_GCM_LEN, hdr + 1, sizeof(uint64_t));
249 tls_comp_len = ntohs(hdr->tls_length) -
250 (AES_GMAC_HASH_LEN + sizeof(uint64_t));
251 ad.seq = htobe64(seqno);
252 ad.type = hdr->tls_type;
253 ad.tls_vmajor = hdr->tls_vmajor;
254 ad.tls_vminor = hdr->tls_vminor;
255 ad.tls_length = htons(tls_comp_len);
257 crp.crp_aad_length = sizeof(ad);
259 crp.crp_payload_start = tls->params.tls_hlen;
260 crp.crp_payload_length = tls_comp_len;
261 crp.crp_digest_start = crp.crp_payload_start + crp.crp_payload_length;
263 crp.crp_op = CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST;
264 crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
265 crypto_use_mbuf(&crp, m);
267 counter_u64_add(ocf_tls12_gcm_crypts, 1);
268 error = ktls_ocf_dispatch(os, &crp);
270 crypto_destroyreq(&crp);
271 *trailer_len = AES_GMAC_HASH_LEN;
276 ktls_ocf_tls13_gcm_encrypt(struct ktls_session *tls,
277 const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov,
278 struct iovec *outiov, int iovcnt, uint64_t seqno, uint8_t record_type)
280 struct uio uio, out_uio;
281 struct tls_aead_data_13 ad;
284 struct ocf_session *os;
285 struct iovec iov[iovcnt + 1], out_iov[iovcnt + 1];
291 crypto_initreq(&crp, os->sid);
293 /* Setup the nonce. */
294 memcpy(nonce, tls->params.iv, tls->params.iv_len);
295 *(uint64_t *)(nonce + 4) ^= htobe64(seqno);
298 ad.type = hdr->tls_type;
299 ad.tls_vmajor = hdr->tls_vmajor;
300 ad.tls_vminor = hdr->tls_vminor;
301 ad.tls_length = hdr->tls_length;
303 crp.crp_aad_length = sizeof(ad);
305 /* Compute payload length and determine if encryption is in place. */
307 crp.crp_payload_start = 0;
308 for (i = 0; i < iovcnt; i++) {
309 if (iniov[i].iov_base != outiov[i].iov_base)
311 crp.crp_payload_length += iniov[i].iov_len;
314 /* Store the record type as the first byte of the trailer. */
315 trailer[0] = record_type;
316 crp.crp_payload_length++;
317 crp.crp_digest_start = crp.crp_payload_length;
320 * Duplicate the input iov to append the trailer. Always
321 * include the full trailer as input to get the record_type
322 * even if only the first byte is used.
324 memcpy(iov, iniov, iovcnt * sizeof(*iov));
325 iov[iovcnt].iov_base = trailer;
326 iov[iovcnt].iov_len = AES_GMAC_HASH_LEN + 1;
328 uio.uio_iovcnt = iovcnt + 1;
330 uio.uio_resid = crp.crp_payload_length + AES_GMAC_HASH_LEN;
331 uio.uio_segflg = UIO_SYSSPACE;
332 uio.uio_td = curthread;
333 crypto_use_uio(&crp, &uio);
336 /* Duplicate the output iov to append the trailer. */
337 memcpy(out_iov, outiov, iovcnt * sizeof(*out_iov));
338 out_iov[iovcnt] = iov[iovcnt];
340 out_uio.uio_iov = out_iov;
341 out_uio.uio_iovcnt = iovcnt + 1;
342 out_uio.uio_offset = 0;
343 out_uio.uio_resid = crp.crp_payload_length +
345 out_uio.uio_segflg = UIO_SYSSPACE;
346 out_uio.uio_td = curthread;
347 crypto_use_output_uio(&crp, &out_uio);
350 crp.crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST;
351 crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
353 memcpy(crp.crp_iv, nonce, sizeof(nonce));
355 counter_u64_add(ocf_tls13_gcm_crypts, 1);
357 counter_u64_add(ocf_inplace, 1);
359 counter_u64_add(ocf_separate_output, 1);
360 error = ktls_ocf_dispatch(os, &crp);
362 crypto_destroyreq(&crp);
367 ktls_ocf_free(struct ktls_session *tls)
369 struct ocf_session *os;
372 crypto_freesession(os->sid);
373 mtx_destroy(&os->lock);
374 zfree(os, M_KTLS_OCF);
378 ktls_ocf_try(struct socket *so, struct ktls_session *tls, int direction)
380 struct crypto_session_params csp;
381 struct ocf_session *os;
384 memset(&csp, 0, sizeof(csp));
385 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD;
387 switch (tls->params.cipher_algorithm) {
388 case CRYPTO_AES_NIST_GCM_16:
389 switch (tls->params.cipher_key_len) {
396 csp.csp_mode = CSP_MODE_AEAD;
397 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16;
398 csp.csp_cipher_key = tls->params.cipher_key;
399 csp.csp_cipher_klen = tls->params.cipher_key_len;
400 csp.csp_ivlen = AES_GCM_IV_LEN;
403 return (EPROTONOSUPPORT);
406 /* Only TLS 1.2 and 1.3 are supported. */
407 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE ||
408 tls->params.tls_vminor < TLS_MINOR_VER_TWO ||
409 tls->params.tls_vminor > TLS_MINOR_VER_THREE)
410 return (EPROTONOSUPPORT);
412 /* TLS 1.3 is not yet supported for receive. */
413 if (direction == KTLS_RX &&
414 tls->params.tls_vminor == TLS_MINOR_VER_THREE)
415 return (EPROTONOSUPPORT);
417 os = malloc(sizeof(*os), M_KTLS_OCF, M_NOWAIT | M_ZERO);
421 error = crypto_newsession(&os->sid, &csp,
422 CRYPTO_FLAG_HARDWARE | CRYPTO_FLAG_SOFTWARE);
424 free(os, M_KTLS_OCF);
428 mtx_init(&os->lock, "ktls_ocf", NULL, MTX_DEF);
430 if (direction == KTLS_TX) {
431 if (tls->params.tls_vminor == TLS_MINOR_VER_THREE)
432 tls->sw_encrypt = ktls_ocf_tls13_gcm_encrypt;
434 tls->sw_encrypt = ktls_ocf_tls12_gcm_encrypt;
436 tls->sw_decrypt = ktls_ocf_tls12_gcm_decrypt;
438 tls->free = ktls_ocf_free;
442 struct ktls_crypto_backend ocf_backend = {
445 .api_version = KTLS_API_VERSION,
450 ktls_ocf_modevent(module_t mod, int what, void *arg)
456 ocf_tls12_gcm_crypts = counter_u64_alloc(M_WAITOK);
457 ocf_tls13_gcm_crypts = counter_u64_alloc(M_WAITOK);
458 ocf_inplace = counter_u64_alloc(M_WAITOK);
459 ocf_separate_output = counter_u64_alloc(M_WAITOK);
460 ocf_retries = counter_u64_alloc(M_WAITOK);
461 return (ktls_crypto_backend_register(&ocf_backend));
463 error = ktls_crypto_backend_deregister(&ocf_backend);
466 counter_u64_free(ocf_tls12_gcm_crypts);
467 counter_u64_free(ocf_tls13_gcm_crypts);
468 counter_u64_free(ocf_inplace);
469 counter_u64_free(ocf_separate_output);
470 counter_u64_free(ocf_retries);
477 static moduledata_t ktls_ocf_moduledata = {
483 DECLARE_MODULE(ktls_ocf, ktls_ocf_moduledata, SI_SUB_PROTO_END, SI_ORDER_ANY);