2 * SPDX-License-Identifier: BSD-2-Clause
4 * Copyright (c) 2019 Netflix Inc.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/counter.h>
35 #include <sys/endian.h>
36 #include <sys/kernel.h>
39 #include <sys/malloc.h>
40 #include <sys/module.h>
41 #include <sys/mutex.h>
42 #include <sys/sysctl.h>
44 #include <opencrypto/cryptodev.h>
52 struct ocf_operation {
53 struct ocf_session *os;
58 static MALLOC_DEFINE(M_KTLS_OCF, "ktls_ocf", "OCF KTLS");
60 SYSCTL_DECL(_kern_ipc_tls);
61 SYSCTL_DECL(_kern_ipc_tls_stats);
63 static SYSCTL_NODE(_kern_ipc_tls_stats, OID_AUTO, ocf, CTLFLAG_RD, 0,
64 "Kernel TLS offload via OCF stats");
66 static counter_u64_t ocf_tls12_gcm_crypts;
67 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_gcm_crypts,
68 CTLFLAG_RD, &ocf_tls12_gcm_crypts,
69 "Total number of OCF TLS 1.2 GCM encryption operations");
71 static counter_u64_t ocf_tls13_gcm_crypts;
72 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_crypts,
73 CTLFLAG_RD, &ocf_tls13_gcm_crypts,
74 "Total number of OCF TLS 1.3 GCM encryption operations");
76 static counter_u64_t ocf_retries;
77 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, retries, CTLFLAG_RD,
79 "Number of OCF encryption operation retries");
82 ktls_ocf_callback(struct cryptop *crp)
84 struct ocf_operation *oo;
87 mtx_lock(&oo->os->lock);
89 mtx_unlock(&oo->os->lock);
95 ktls_ocf_tls12_gcm_encrypt(struct ktls_session *tls,
96 const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov,
97 struct iovec *outiov, int iovcnt, uint64_t seqno,
98 uint8_t record_type __unused)
101 struct tls_aead_data ad;
102 struct tls_nonce_data nd;
103 struct cryptodesc *crde, *crda;
105 struct ocf_session *os;
106 struct ocf_operation *oo;
109 uint16_t tls_comp_len;
113 oo = malloc(sizeof(*oo) + (iovcnt + 2) * sizeof(*iov), M_KTLS_OCF,
118 crp = crypto_getreq(2);
120 free(oo, M_KTLS_OCF);
125 memcpy(nd.fixed, tls->params.iv, TLS_AEAD_GCM_LEN);
126 memcpy(&nd.seq, hdr + 1, sizeof(nd.seq));
129 tls_comp_len = ntohs(hdr->tls_length) -
130 (AES_GMAC_HASH_LEN + sizeof(nd.seq));
131 ad.seq = htobe64(seqno);
132 ad.type = hdr->tls_type;
133 ad.tls_vmajor = hdr->tls_vmajor;
134 ad.tls_vminor = hdr->tls_vminor;
135 ad.tls_length = htons(tls_comp_len);
136 iov[0].iov_base = &ad;
137 iov[0].iov_len = sizeof(ad);
138 uio.uio_resid = sizeof(ad);
141 * OCF always does encryption in place, so copy the data if
144 for (i = 0; i < iovcnt; i++) {
145 iov[i + 1] = outiov[i];
146 if (iniov[i].iov_base != outiov[i].iov_base)
147 memcpy(outiov[i].iov_base, iniov[i].iov_base,
149 uio.uio_resid += outiov[i].iov_len;
152 iov[iovcnt + 1].iov_base = trailer;
153 iov[iovcnt + 1].iov_len = AES_GMAC_HASH_LEN;
154 uio.uio_resid += AES_GMAC_HASH_LEN;
157 uio.uio_iovcnt = iovcnt + 2;
159 uio.uio_segflg = UIO_SYSSPACE;
160 uio.uio_td = curthread;
162 crp->crp_session = os->sid;
163 crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM;
165 crp->crp_ilen = uio.uio_resid;
166 crp->crp_opaque = oo;
167 crp->crp_callback = ktls_ocf_callback;
169 crde = crp->crp_desc;
170 crda = crde->crd_next;
172 crda->crd_alg = os->crda_alg;
174 crda->crd_len = sizeof(ad);
175 crda->crd_inject = crp->crp_ilen - AES_GMAC_HASH_LEN;
177 crde->crd_alg = CRYPTO_AES_NIST_GCM_16;
178 crde->crd_skip = sizeof(ad);
179 crde->crd_len = crp->crp_ilen - (sizeof(ad) + AES_GMAC_HASH_LEN);
180 crde->crd_flags = CRD_F_ENCRYPT | CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
181 memcpy(crde->crd_iv, &nd, sizeof(nd));
183 counter_u64_add(ocf_tls12_gcm_crypts, 1);
185 error = crypto_dispatch(crp);
191 mtx_sleep(oo, &os->lock, 0, "ocfktls", 0);
192 mtx_unlock(&os->lock);
194 if (crp->crp_etype != EAGAIN) {
195 error = crp->crp_etype;
200 crp->crp_flags &= ~CRYPTO_F_DONE;
202 counter_u64_add(ocf_retries, 1);
206 free(oo, M_KTLS_OCF);
211 ktls_ocf_tls13_gcm_encrypt(struct ktls_session *tls,
212 const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov,
213 struct iovec *outiov, int iovcnt, uint64_t seqno, uint8_t record_type)
216 struct tls_aead_data_13 ad;
218 struct cryptodesc *crde, *crda;
220 struct ocf_session *os;
221 struct ocf_operation *oo;
227 oo = malloc(sizeof(*oo) + (iovcnt + 2) * sizeof(*iov), M_KTLS_OCF,
232 crp = crypto_getreq(2);
234 free(oo, M_KTLS_OCF);
238 /* Setup the nonce. */
239 memcpy(nonce, tls->params.iv, tls->params.iv_len);
240 *(uint64_t *)(nonce + 4) ^= htobe64(seqno);
243 ad.type = hdr->tls_type;
244 ad.tls_vmajor = hdr->tls_vmajor;
245 ad.tls_vminor = hdr->tls_vminor;
246 ad.tls_length = hdr->tls_length;
247 iov[0].iov_base = &ad;
248 iov[0].iov_len = sizeof(ad);
249 uio.uio_resid = sizeof(ad);
252 * OCF always does encryption in place, so copy the data if
255 for (i = 0; i < iovcnt; i++) {
256 iov[i + 1] = outiov[i];
257 if (iniov[i].iov_base != outiov[i].iov_base)
258 memcpy(outiov[i].iov_base, iniov[i].iov_base,
260 uio.uio_resid += outiov[i].iov_len;
263 trailer[0] = record_type;
264 iov[iovcnt + 1].iov_base = trailer;
265 iov[iovcnt + 1].iov_len = AES_GMAC_HASH_LEN + 1;
266 uio.uio_resid += AES_GMAC_HASH_LEN + 1;
269 uio.uio_iovcnt = iovcnt + 2;
271 uio.uio_segflg = UIO_SYSSPACE;
272 uio.uio_td = curthread;
274 crp->crp_session = os->sid;
275 crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM;
277 crp->crp_ilen = uio.uio_resid;
278 crp->crp_opaque = oo;
279 crp->crp_callback = ktls_ocf_callback;
281 crde = crp->crp_desc;
282 crda = crde->crd_next;
284 crda->crd_alg = os->crda_alg;
286 crda->crd_len = sizeof(ad);
287 crda->crd_inject = crp->crp_ilen - AES_GMAC_HASH_LEN;
289 crde->crd_alg = CRYPTO_AES_NIST_GCM_16;
290 crde->crd_skip = sizeof(ad);
291 crde->crd_len = crp->crp_ilen - (sizeof(ad) + AES_GMAC_HASH_LEN);
292 crde->crd_flags = CRD_F_ENCRYPT | CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
293 memcpy(crde->crd_iv, nonce, sizeof(nonce));
295 counter_u64_add(ocf_tls13_gcm_crypts, 1);
297 error = crypto_dispatch(crp);
303 mtx_sleep(oo, &os->lock, 0, "ocfktls", 0);
304 mtx_unlock(&os->lock);
306 if (crp->crp_etype != EAGAIN) {
307 error = crp->crp_etype;
312 crp->crp_flags &= ~CRYPTO_F_DONE;
314 counter_u64_add(ocf_retries, 1);
318 free(oo, M_KTLS_OCF);
323 ktls_ocf_free(struct ktls_session *tls)
325 struct ocf_session *os;
328 mtx_destroy(&os->lock);
329 explicit_bzero(os, sizeof(*os));
330 free(os, M_KTLS_OCF);
334 ktls_ocf_try(struct socket *so, struct ktls_session *tls)
336 struct cryptoini cria, crie;
337 struct ocf_session *os;
340 memset(&cria, 0, sizeof(cria));
341 memset(&crie, 0, sizeof(crie));
343 switch (tls->params.cipher_algorithm) {
344 case CRYPTO_AES_NIST_GCM_16:
345 switch (tls->params.cipher_key_len) {
347 cria.cri_alg = CRYPTO_AES_128_NIST_GMAC;
350 cria.cri_alg = CRYPTO_AES_256_NIST_GMAC;
355 cria.cri_key = tls->params.cipher_key;
356 cria.cri_klen = tls->params.cipher_key_len * 8;
359 return (EPROTONOSUPPORT);
362 /* Only TLS 1.2 and 1.3 are supported. */
363 if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE ||
364 tls->params.tls_vminor < TLS_MINOR_VER_TWO ||
365 tls->params.tls_vminor > TLS_MINOR_VER_THREE)
366 return (EPROTONOSUPPORT);
368 os = malloc(sizeof(*os), M_KTLS_OCF, M_NOWAIT | M_ZERO);
372 crie.cri_alg = tls->params.cipher_algorithm;
373 crie.cri_key = tls->params.cipher_key;
374 crie.cri_klen = tls->params.cipher_key_len * 8;
376 crie.cri_next = &cria;
377 error = crypto_newsession(&os->sid, &crie,
378 CRYPTO_FLAG_HARDWARE | CRYPTO_FLAG_SOFTWARE);
380 free(os, M_KTLS_OCF);
384 os->crda_alg = cria.cri_alg;
385 mtx_init(&os->lock, "ktls_ocf", NULL, MTX_DEF);
387 if (tls->params.tls_vminor == TLS_MINOR_VER_THREE)
388 tls->sw_encrypt = ktls_ocf_tls13_gcm_encrypt;
390 tls->sw_encrypt = ktls_ocf_tls12_gcm_encrypt;
391 tls->free = ktls_ocf_free;
395 struct ktls_crypto_backend ocf_backend = {
398 .api_version = KTLS_API_VERSION,
403 ktls_ocf_modevent(module_t mod, int what, void *arg)
409 ocf_tls12_gcm_crypts = counter_u64_alloc(M_WAITOK);
410 ocf_tls13_gcm_crypts = counter_u64_alloc(M_WAITOK);
411 ocf_retries = counter_u64_alloc(M_WAITOK);
412 return (ktls_crypto_backend_register(&ocf_backend));
414 error = ktls_crypto_backend_deregister(&ocf_backend);
417 counter_u64_free(ocf_tls12_gcm_crypts);
418 counter_u64_free(ocf_tls13_gcm_crypts);
419 counter_u64_free(ocf_retries);
426 static moduledata_t ktls_ocf_moduledata = {
432 DECLARE_MODULE(ktls_ocf, ktls_ocf_moduledata, SI_SUB_PROTO_END, SI_ORDER_ANY);