1 /* $OpenBSD: cryptodev.c,v 1.52 2002/06/19 07:22:46 deraadt Exp $ */
4 * Copyright (c) 2001 Theo de Raadt
5 * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting
6 * Copyright (c) 2014 The FreeBSD Foundation
9 * Portions of this software were developed by John-Mark Gurney
10 * under sponsorship of the FreeBSD Foundation and
11 * Rubicon Communications, LLC (Netgate).
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in the
21 * documentation and/or other materials provided with the distribution.
22 * 3. The name of the author may not be used to endorse or promote products
23 * derived from this software without specific prior written permission.
25 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
26 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
27 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
28 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
29 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
30 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
31 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
32 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
33 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
34 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 * Effort sponsored in part by the Defense Advanced Research Projects
37 * Agency (DARPA) and Air Force Research Laboratory, Air Force
38 * Materiel Command, USAF, under agreement number F30602-01-2-0537.
41 #include <sys/cdefs.h>
42 __FBSDID("$FreeBSD$");
44 #include <sys/param.h>
45 #include <sys/systm.h>
46 #include <sys/malloc.h>
49 #include <sys/mutex.h>
51 #include <sys/sysctl.h>
52 #include <sys/errno.h>
53 #include <sys/random.h>
55 #include <sys/kernel.h>
56 #include <sys/module.h>
57 #include <sys/fcntl.h>
60 #include <sys/syscallsubr.h>
62 #include <opencrypto/cryptodev.h>
63 #include <opencrypto/xform.h>
65 SDT_PROVIDER_DECLARE(opencrypto);
67 SDT_PROBE_DEFINE1(opencrypto, dev, ioctl, error, "int"/*line number*/);
69 #ifdef COMPAT_FREEBSD12
71 * Previously, most ioctls were performed against a cloned descriptor
72 * of /dev/crypto obtained via CRIOGET. Now all ioctls are performed
73 * against /dev/crypto directly.
75 #define CRIOGET _IOWR('c', 100, uint32_t)
78 /* the following are done against the cloned descriptor */
80 #ifdef COMPAT_FREEBSD32
81 #include <sys/mount.h>
82 #include <compat/freebsd32/freebsd32.h>
94 struct session2_op32 {
116 struct crypt_aead32 {
130 #define CIOCGSESSION32 _IOWR('c', 101, struct session_op32)
131 #define CIOCCRYPT32 _IOWR('c', 103, struct crypt_op32)
132 #define CIOCGSESSION232 _IOWR('c', 106, struct session2_op32)
133 #define CIOCCRYPTAEAD32 _IOWR('c', 109, struct crypt_aead32)
136 session_op_from_32(const struct session_op32 *from, struct session2_op *to)
139 memset(to, 0, sizeof(*to));
140 CP(*from, *to, cipher);
142 CP(*from, *to, keylen);
143 PTRIN_CP(*from, *to, key);
144 CP(*from, *to, mackeylen);
145 PTRIN_CP(*from, *to, mackey);
147 to->crid = CRYPTOCAP_F_HARDWARE;
151 session2_op_from_32(const struct session2_op32 *from, struct session2_op *to)
154 session_op_from_32((const struct session_op32 *)from, to);
155 CP(*from, *to, crid);
159 session_op_to_32(const struct session2_op *from, struct session_op32 *to)
162 CP(*from, *to, cipher);
164 CP(*from, *to, keylen);
165 PTROUT_CP(*from, *to, key);
166 CP(*from, *to, mackeylen);
167 PTROUT_CP(*from, *to, mackey);
172 session2_op_to_32(const struct session2_op *from, struct session2_op32 *to)
175 session_op_to_32(from, (struct session_op32 *)to);
176 CP(*from, *to, crid);
180 crypt_op_from_32(const struct crypt_op32 *from, struct crypt_op *to)
185 CP(*from, *to, flags);
187 PTRIN_CP(*from, *to, src);
188 PTRIN_CP(*from, *to, dst);
189 PTRIN_CP(*from, *to, mac);
190 PTRIN_CP(*from, *to, iv);
194 crypt_op_to_32(const struct crypt_op *from, struct crypt_op32 *to)
199 CP(*from, *to, flags);
201 PTROUT_CP(*from, *to, src);
202 PTROUT_CP(*from, *to, dst);
203 PTROUT_CP(*from, *to, mac);
204 PTROUT_CP(*from, *to, iv);
208 crypt_aead_from_32(const struct crypt_aead32 *from, struct crypt_aead *to)
213 CP(*from, *to, flags);
215 CP(*from, *to, aadlen);
216 CP(*from, *to, ivlen);
217 PTRIN_CP(*from, *to, src);
218 PTRIN_CP(*from, *to, dst);
219 PTRIN_CP(*from, *to, aad);
220 PTRIN_CP(*from, *to, tag);
221 PTRIN_CP(*from, *to, iv);
225 crypt_aead_to_32(const struct crypt_aead *from, struct crypt_aead32 *to)
230 CP(*from, *to, flags);
232 CP(*from, *to, aadlen);
233 CP(*from, *to, ivlen);
234 PTROUT_CP(*from, *to, src);
235 PTROUT_CP(*from, *to, dst);
236 PTROUT_CP(*from, *to, aad);
237 PTROUT_CP(*from, *to, tag);
238 PTROUT_CP(*from, *to, iv);
243 session2_op_from_op(const struct session_op *from, struct session2_op *to)
246 memset(to, 0, sizeof(*to));
247 memcpy(to, from, sizeof(*from));
248 to->crid = CRYPTOCAP_F_HARDWARE;
252 session2_op_to_op(const struct session2_op *from, struct session_op *to)
255 memcpy(to, from, sizeof(*to));
259 TAILQ_ENTRY(csession) next;
260 crypto_session_t cses;
263 struct mtx lock; /* for op submission */
265 const struct enc_xform *txform;
274 struct cryptop_data {
275 struct csession *cse;
284 TAILQ_HEAD(csessionlist, csession) csessions;
289 static bool use_outputbuffers;
290 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_use_output, CTLFLAG_RW,
291 &use_outputbuffers, 0,
292 "Use separate output buffers for /dev/crypto requests.");
294 static bool use_separate_aad;
295 SYSCTL_BOOL(_kern_crypto, OID_AUTO, cryptodev_separate_aad, CTLFLAG_RW,
296 &use_separate_aad, 0,
297 "Use separate AAD buffer for /dev/crypto requests.");
300 * Check a crypto identifier to see if it requested
301 * a software device/driver. This can be done either
302 * by device name/class or through search constraints.
305 checkforsoftware(int *cridp)
311 if (!crypto_devallowsoft) {
312 if (crid & CRYPTOCAP_F_SOFTWARE) {
313 if (crid & CRYPTOCAP_F_HARDWARE) {
314 *cridp = CRYPTOCAP_F_HARDWARE;
319 if ((crid & CRYPTOCAP_F_HARDWARE) == 0 &&
320 (crypto_getcaps(crid) & CRYPTOCAP_F_HARDWARE) == 0)
327 cse_create(struct fcrypt *fcr, struct session2_op *sop)
329 struct crypto_session_params csp;
330 struct csession *cse;
331 const struct enc_xform *txform;
332 const struct auth_hash *thash;
335 crypto_session_t cses;
338 switch (sop->cipher) {
343 txform = &enc_xform_rijndael128;
346 txform = &enc_xform_aes_xts;
348 case CRYPTO_NULL_CBC:
349 txform = &enc_xform_null;
351 case CRYPTO_CAMELLIA_CBC:
352 txform = &enc_xform_camellia;
355 txform = &enc_xform_aes_icm;
357 case CRYPTO_AES_NIST_GCM_16:
358 txform = &enc_xform_aes_nist_gcm;
360 case CRYPTO_CHACHA20:
361 txform = &enc_xform_chacha20;
363 case CRYPTO_AES_CCM_16:
364 txform = &enc_xform_ccm;
366 case CRYPTO_CHACHA20_POLY1305:
367 txform = &enc_xform_chacha20_poly1305;
370 CRYPTDEB("invalid cipher");
371 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
379 case CRYPTO_POLY1305:
380 thash = &auth_hash_poly1305;
382 case CRYPTO_SHA1_HMAC:
383 thash = &auth_hash_hmac_sha1;
385 case CRYPTO_SHA2_224_HMAC:
386 thash = &auth_hash_hmac_sha2_224;
388 case CRYPTO_SHA2_256_HMAC:
389 thash = &auth_hash_hmac_sha2_256;
391 case CRYPTO_SHA2_384_HMAC:
392 thash = &auth_hash_hmac_sha2_384;
394 case CRYPTO_SHA2_512_HMAC:
395 thash = &auth_hash_hmac_sha2_512;
397 case CRYPTO_RIPEMD160_HMAC:
398 thash = &auth_hash_hmac_ripemd_160;
400 #ifdef COMPAT_FREEBSD12
401 case CRYPTO_AES_128_NIST_GMAC:
402 case CRYPTO_AES_192_NIST_GMAC:
403 case CRYPTO_AES_256_NIST_GMAC:
404 /* Should always be paired with GCM. */
405 if (sop->cipher != CRYPTO_AES_NIST_GCM_16) {
406 CRYPTDEB("GMAC without GCM");
407 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
412 case CRYPTO_AES_NIST_GMAC:
413 switch (sop->mackeylen * 8) {
415 thash = &auth_hash_nist_gmac_aes_128;
418 thash = &auth_hash_nist_gmac_aes_192;
421 thash = &auth_hash_nist_gmac_aes_256;
424 CRYPTDEB("invalid GMAC key length");
425 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
429 case CRYPTO_AES_CCM_CBC_MAC:
430 switch (sop->mackeylen) {
432 thash = &auth_hash_ccm_cbc_mac_128;
435 thash = &auth_hash_ccm_cbc_mac_192;
438 thash = &auth_hash_ccm_cbc_mac_256;
441 CRYPTDEB("Invalid CBC MAC key size %d", sop->keylen);
442 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
447 thash = &auth_hash_sha1;
449 case CRYPTO_SHA2_224:
450 thash = &auth_hash_sha2_224;
452 case CRYPTO_SHA2_256:
453 thash = &auth_hash_sha2_256;
455 case CRYPTO_SHA2_384:
456 thash = &auth_hash_sha2_384;
458 case CRYPTO_SHA2_512:
459 thash = &auth_hash_sha2_512;
462 case CRYPTO_NULL_HMAC:
463 thash = &auth_hash_null;
467 thash = &auth_hash_blake2b;
470 thash = &auth_hash_blake2s;
474 CRYPTDEB("invalid mac");
475 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
479 if (txform == NULL && thash == NULL) {
480 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
484 memset(&csp, 0, sizeof(csp));
485 if (use_outputbuffers)
486 csp.csp_flags |= CSP_F_SEPARATE_OUTPUT;
488 if (sop->cipher == CRYPTO_AES_NIST_GCM_16) {
490 #ifdef COMPAT_FREEBSD12
491 case CRYPTO_AES_128_NIST_GMAC:
492 case CRYPTO_AES_192_NIST_GMAC:
493 case CRYPTO_AES_256_NIST_GMAC:
494 if (sop->keylen != sop->mackeylen) {
495 SDT_PROBE1(opencrypto, dev, ioctl, error,
504 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
507 csp.csp_mode = CSP_MODE_AEAD;
508 } else if (sop->cipher == CRYPTO_AES_CCM_16) {
510 #ifdef COMPAT_FREEBSD12
511 case CRYPTO_AES_CCM_CBC_MAC:
512 if (sop->keylen != sop->mackeylen) {
513 SDT_PROBE1(opencrypto, dev, ioctl, error,
523 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
526 csp.csp_mode = CSP_MODE_AEAD;
527 } else if (sop->cipher == CRYPTO_CHACHA20_POLY1305) {
529 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
532 csp.csp_mode = CSP_MODE_AEAD;
533 } else if (txform != NULL && thash != NULL)
534 csp.csp_mode = CSP_MODE_ETA;
535 else if (txform != NULL)
536 csp.csp_mode = CSP_MODE_CIPHER;
538 csp.csp_mode = CSP_MODE_DIGEST;
540 switch (csp.csp_mode) {
543 if (use_separate_aad)
544 csp.csp_flags |= CSP_F_SEPARATE_AAD;
548 if (txform != NULL) {
549 csp.csp_cipher_alg = txform->type;
550 csp.csp_cipher_klen = sop->keylen;
551 if (sop->keylen > txform->maxkey ||
552 sop->keylen < txform->minkey) {
553 CRYPTDEB("invalid cipher parameters");
555 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
559 key = malloc(csp.csp_cipher_klen, M_XDATA, M_WAITOK);
560 error = copyin(sop->key, key, csp.csp_cipher_klen);
562 CRYPTDEB("invalid key");
563 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
566 csp.csp_cipher_key = key;
567 csp.csp_ivlen = txform->ivsize;
571 csp.csp_auth_alg = thash->type;
572 csp.csp_auth_klen = sop->mackeylen;
573 if (sop->mackeylen > thash->keysize || sop->mackeylen < 0) {
574 CRYPTDEB("invalid mac key length");
576 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
580 if (csp.csp_auth_klen != 0) {
581 mackey = malloc(csp.csp_auth_klen, M_XDATA, M_WAITOK);
582 error = copyin(sop->mackey, mackey, csp.csp_auth_klen);
584 CRYPTDEB("invalid mac key");
585 SDT_PROBE1(opencrypto, dev, ioctl, error,
589 csp.csp_auth_key = mackey;
592 if (csp.csp_auth_alg == CRYPTO_AES_NIST_GMAC)
593 csp.csp_ivlen = AES_GCM_IV_LEN;
594 if (csp.csp_auth_alg == CRYPTO_AES_CCM_CBC_MAC)
595 csp.csp_ivlen = AES_CCM_IV_LEN;
599 error = checkforsoftware(&crid);
601 CRYPTDEB("checkforsoftware");
602 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
605 error = crypto_newsession(&cses, &csp, crid);
607 CRYPTDEB("crypto_newsession");
608 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
612 cse = malloc(sizeof(struct csession), M_XDATA, M_WAITOK | M_ZERO);
613 mtx_init(&cse->lock, "cryptodev", "crypto session lock", MTX_DEF);
614 refcount_init(&cse->refs, 1);
616 cse->mackey = mackey;
617 cse->mode = csp.csp_mode;
619 cse->txform = txform;
621 cse->hashsize = thash->hashsize;
622 else if (csp.csp_cipher_alg == CRYPTO_AES_NIST_GCM_16)
623 cse->hashsize = AES_GMAC_HASH_LEN;
624 else if (csp.csp_cipher_alg == CRYPTO_AES_CCM_16)
625 cse->hashsize = AES_CBC_MAC_HASH_LEN;
626 else if (csp.csp_cipher_alg == CRYPTO_CHACHA20_POLY1305)
627 cse->hashsize = POLY1305_HASH_LEN;
628 cse->ivsize = csp.csp_ivlen;
630 mtx_lock(&fcr->lock);
631 TAILQ_INSERT_TAIL(&fcr->csessions, cse, next);
632 cse->ses = fcr->sesn++;
633 mtx_unlock(&fcr->lock);
637 /* return hardware/driver id */
638 sop->crid = crypto_ses2hid(cse->cses);
642 free(mackey, M_XDATA);
647 static struct csession *
648 cse_find(struct fcrypt *fcr, u_int ses)
650 struct csession *cse;
652 mtx_lock(&fcr->lock);
653 TAILQ_FOREACH(cse, &fcr->csessions, next) {
654 if (cse->ses == ses) {
655 refcount_acquire(&cse->refs);
656 mtx_unlock(&fcr->lock);
660 mtx_unlock(&fcr->lock);
665 cse_free(struct csession *cse)
668 if (!refcount_release(&cse->refs))
670 crypto_freesession(cse->cses);
671 mtx_destroy(&cse->lock);
673 free(cse->key, M_XDATA);
675 free(cse->mackey, M_XDATA);
680 cse_delete(struct fcrypt *fcr, u_int ses)
682 struct csession *cse;
684 mtx_lock(&fcr->lock);
685 TAILQ_FOREACH(cse, &fcr->csessions, next) {
686 if (cse->ses == ses) {
687 TAILQ_REMOVE(&fcr->csessions, cse, next);
688 mtx_unlock(&fcr->lock);
693 mtx_unlock(&fcr->lock);
697 static struct cryptop_data *
698 cod_alloc(struct csession *cse, size_t aad_len, size_t len)
700 struct cryptop_data *cod;
702 cod = malloc(sizeof(struct cryptop_data), M_XDATA, M_WAITOK | M_ZERO);
705 if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_AAD) {
707 cod->aad = malloc(aad_len, M_XDATA, M_WAITOK);
708 cod->buf = malloc(len, M_XDATA, M_WAITOK);
710 cod->buf = malloc(aad_len + len, M_XDATA, M_WAITOK);
711 if (crypto_get_params(cse->cses)->csp_flags & CSP_F_SEPARATE_OUTPUT)
712 cod->obuf = malloc(len, M_XDATA, M_WAITOK);
717 cod_free(struct cryptop_data *cod)
720 free(cod->aad, M_XDATA);
721 free(cod->obuf, M_XDATA);
722 free(cod->buf, M_XDATA);
727 cryptodev_cb(struct cryptop *crp)
729 struct cryptop_data *cod = crp->crp_opaque;
732 * Lock to ensure the wakeup() is not missed by the loops
733 * waiting on cod->done in cryptodev_op() and
736 mtx_lock(&cod->cse->lock);
738 mtx_unlock(&cod->cse->lock);
744 cryptodev_op(struct csession *cse, const struct crypt_op *cop)
746 struct cryptop_data *cod = NULL;
747 struct cryptop *crp = NULL;
751 if (cop->len > 256*1024-4) {
752 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
757 if (cop->len == 0 || (cop->len % cse->txform->blocksize) != 0) {
758 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
763 if (cop->mac && cse->hashsize == 0) {
764 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
769 * The COP_F_CIPHER_FIRST flag predates explicit session
770 * modes, but the only way it was used was for EtA so allow it
771 * as long as it is consistent with EtA.
773 if (cop->flags & COP_F_CIPHER_FIRST) {
774 if (cop->op != COP_ENCRYPT) {
775 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
780 cod = cod_alloc(cse, 0, cop->len + cse->hashsize);
783 crp = crypto_getreq(cse->cses, M_WAITOK);
785 error = copyin(cop->src, cod->buf, cop->len);
787 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
790 crp->crp_payload_start = 0;
791 crp->crp_payload_length = cop->len;
793 crp->crp_digest_start = cop->len;
796 case CSP_MODE_COMPRESS:
799 crp->crp_op = CRYPTO_OP_COMPRESS;
802 crp->crp_op = CRYPTO_OP_DECOMPRESS;
805 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
810 case CSP_MODE_CIPHER:
813 crp->crp_op = CRYPTO_OP_ENCRYPT;
816 crp->crp_op = CRYPTO_OP_DECRYPT;
819 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
824 case CSP_MODE_DIGEST:
829 crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST;
830 if (cod->obuf != NULL)
831 crp->crp_digest_start = 0;
834 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
842 crp->crp_op = CRYPTO_OP_ENCRYPT |
843 CRYPTO_OP_COMPUTE_DIGEST;
846 crp->crp_op = CRYPTO_OP_DECRYPT |
847 CRYPTO_OP_VERIFY_DIGEST;
850 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
856 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
861 crp->crp_flags = CRYPTO_F_CBIMM | (cop->flags & COP_F_BATCH);
862 crypto_use_buf(crp, cod->buf, cop->len + cse->hashsize);
864 crypto_use_output_buf(crp, cod->obuf, cop->len + cse->hashsize);
865 crp->crp_callback = cryptodev_cb;
866 crp->crp_opaque = cod;
869 if (cse->ivsize == 0) {
870 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
874 error = copyin(cop->iv, crp->crp_iv, cse->ivsize);
876 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
879 crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
880 } else if (cse->ivsize != 0) {
881 if (crp->crp_payload_length < cse->ivsize) {
882 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
886 crp->crp_iv_start = 0;
887 crp->crp_payload_start += cse->ivsize;
888 crp->crp_payload_length -= cse->ivsize;
892 if (cop->mac != NULL && crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
893 error = copyin(cop->mac, cod->buf + crp->crp_digest_start,
896 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
902 * Let the dispatch run unlocked, then, interlock against the
903 * callback before checking if the operation completed and going
904 * to sleep. This insures drivers don't inherit our lock which
905 * results in a lock order reversal between crypto_dispatch forced
906 * entry and the crypto_done callback into us.
908 error = crypto_dispatch(crp);
910 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
914 mtx_lock(&cse->lock);
916 mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0);
917 mtx_unlock(&cse->lock);
919 if (crp->crp_etype == EAGAIN) {
921 crp->crp_flags &= ~CRYPTO_F_DONE;
926 if (crp->crp_etype != 0) {
927 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
928 error = crp->crp_etype;
932 if (cop->dst != NULL) {
933 error = copyout(cod->obuf != NULL ? cod->obuf :
934 cod->buf + crp->crp_payload_start, dst,
935 crp->crp_payload_length);
937 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
942 if (cop->mac != NULL && (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) {
943 error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) +
944 crp->crp_digest_start, cop->mac, cse->hashsize);
946 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
959 cryptodev_aead(struct csession *cse, struct crypt_aead *caead)
961 struct cryptop_data *cod = NULL;
962 struct cryptop *crp = NULL;
966 if (caead->len > 256*1024-4 || caead->aadlen > 256*1024-4) {
967 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
971 if (cse->txform == NULL || cse->hashsize == 0 || caead->tag == NULL ||
972 (caead->len % cse->txform->blocksize) != 0) {
973 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
978 * The COP_F_CIPHER_FIRST flag predates explicit session
979 * modes, but the only way it was used was for EtA so allow it
980 * as long as it is consistent with EtA.
982 if (caead->flags & COP_F_CIPHER_FIRST) {
983 if (caead->op != COP_ENCRYPT) {
984 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
989 cod = cod_alloc(cse, caead->aadlen, caead->len + cse->hashsize);
992 crp = crypto_getreq(cse->cses, M_WAITOK);
994 if (cod->aad != NULL)
995 error = copyin(caead->aad, cod->aad, caead->aadlen);
997 error = copyin(caead->aad, cod->buf, caead->aadlen);
999 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1002 crp->crp_aad = cod->aad;
1003 crp->crp_aad_start = 0;
1004 crp->crp_aad_length = caead->aadlen;
1006 if (cod->aad != NULL)
1007 crp->crp_payload_start = 0;
1009 crp->crp_payload_start = caead->aadlen;
1010 error = copyin(caead->src, cod->buf + crp->crp_payload_start,
1013 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1016 crp->crp_payload_length = caead->len;
1017 if (caead->op == COP_ENCRYPT && cod->obuf != NULL)
1018 crp->crp_digest_start = crp->crp_payload_output_start +
1021 crp->crp_digest_start = crp->crp_payload_start + caead->len;
1023 switch (cse->mode) {
1026 switch (caead->op) {
1028 crp->crp_op = CRYPTO_OP_ENCRYPT |
1029 CRYPTO_OP_COMPUTE_DIGEST;
1032 crp->crp_op = CRYPTO_OP_DECRYPT |
1033 CRYPTO_OP_VERIFY_DIGEST;
1036 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1042 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1047 crp->crp_flags = CRYPTO_F_CBIMM | (caead->flags & COP_F_BATCH);
1048 crypto_use_buf(crp, cod->buf, crp->crp_payload_start + caead->len +
1050 if (cod->obuf != NULL)
1051 crypto_use_output_buf(crp, cod->obuf, caead->len +
1053 crp->crp_callback = cryptodev_cb;
1054 crp->crp_opaque = cod;
1058 * Permit a 16-byte IV for AES-XTS, but only use the
1059 * first 8 bytes as a block number.
1061 if (cse->mode == CSP_MODE_ETA &&
1062 caead->ivlen == AES_BLOCK_LEN &&
1063 cse->ivsize == AES_XTS_IV_LEN)
1064 caead->ivlen = AES_XTS_IV_LEN;
1066 if (cse->ivsize == 0) {
1067 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1071 if (caead->ivlen != cse->ivsize) {
1073 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1077 error = copyin(caead->iv, crp->crp_iv, cse->ivsize);
1079 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1082 crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
1085 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1089 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
1090 error = copyin(caead->tag, cod->buf + crp->crp_digest_start,
1093 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1099 * Let the dispatch run unlocked, then, interlock against the
1100 * callback before checking if the operation completed and going
1101 * to sleep. This insures drivers don't inherit our lock which
1102 * results in a lock order reversal between crypto_dispatch forced
1103 * entry and the crypto_done callback into us.
1105 error = crypto_dispatch(crp);
1107 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1111 mtx_lock(&cse->lock);
1113 mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0);
1114 mtx_unlock(&cse->lock);
1116 if (crp->crp_etype == EAGAIN) {
1118 crp->crp_flags &= ~CRYPTO_F_DONE;
1123 if (crp->crp_etype != 0) {
1124 error = crp->crp_etype;
1125 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1129 if (caead->dst != NULL) {
1130 error = copyout(cod->obuf != NULL ? cod->obuf :
1131 cod->buf + crp->crp_payload_start, dst,
1132 crp->crp_payload_length);
1134 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1139 if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) == 0) {
1140 error = copyout((cod->obuf != NULL ? cod->obuf : cod->buf) +
1141 crp->crp_digest_start, caead->tag, cse->hashsize);
1143 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1149 crypto_freereq(crp);
1156 cryptodev_find(struct crypt_find_op *find)
1159 size_t fnlen = sizeof find->name;
1161 if (find->crid != -1) {
1162 dev = crypto_find_device_byhid(find->crid);
1165 strncpy(find->name, device_get_nameunit(dev), fnlen);
1166 find->name[fnlen - 1] = '\x0';
1168 find->name[fnlen - 1] = '\x0';
1169 find->crid = crypto_find_driver(find->name);
1170 if (find->crid == -1)
1177 fcrypt_dtor(void *data)
1179 struct fcrypt *fcr = data;
1180 struct csession *cse;
1182 while ((cse = TAILQ_FIRST(&fcr->csessions))) {
1183 TAILQ_REMOVE(&fcr->csessions, cse, next);
1184 KASSERT(refcount_load(&cse->refs) == 1,
1185 ("%s: crypto session %p with %d refs", __func__, cse,
1186 refcount_load(&cse->refs)));
1189 mtx_destroy(&fcr->lock);
1194 crypto_open(struct cdev *dev, int oflags, int devtype, struct thread *td)
1199 fcr = malloc(sizeof(struct fcrypt), M_XDATA, M_WAITOK | M_ZERO);
1200 TAILQ_INIT(&fcr->csessions);
1201 mtx_init(&fcr->lock, "fcrypt", NULL, MTX_DEF);
1202 error = devfs_set_cdevpriv(fcr, fcrypt_dtor);
1209 crypto_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag,
1213 struct csession *cse;
1214 struct session2_op *sop;
1215 struct crypt_op *cop;
1216 struct crypt_aead *caead;
1220 struct session2_op sopc;
1221 #ifdef COMPAT_FREEBSD32
1222 struct crypt_op copc;
1223 struct crypt_aead aeadc;
1226 #ifdef COMPAT_FREEBSD32
1233 case CIOCGSESSION32:
1237 data = (void *)&thunk.sopc;
1238 session_op_from_32((struct session_op32 *)data32, &thunk.sopc);
1240 case CIOCGSESSION232:
1243 cmd = CIOCGSESSION2;
1244 data = (void *)&thunk.sopc;
1245 session2_op_from_32((struct session2_op32 *)data32,
1252 data = (void *)&thunk.copc;
1253 crypt_op_from_32((struct crypt_op32 *)data32, &thunk.copc);
1255 case CIOCCRYPTAEAD32:
1258 cmd = CIOCCRYPTAEAD;
1259 data = (void *)&thunk.aeadc;
1260 crypt_aead_from_32((struct crypt_aead32 *)data32, &thunk.aeadc);
1265 devfs_get_cdevpriv((void **)&fcr);
1268 #ifdef COMPAT_FREEBSD12
1271 * NB: This may fail in cases that the old
1272 * implementation did not if the current process has
1273 * restricted filesystem access (e.g. running in a
1274 * jail that does not expose /dev/crypto or in
1277 error = kern_openat(td, AT_FDCWD, "/dev/crypto", UIO_SYSSPACE,
1280 *(uint32_t *)data = td->td_retval[0];
1285 if (cmd == CIOCGSESSION) {
1286 session2_op_from_op((void *)data, &thunk.sopc);
1289 sop = (struct session2_op *)data;
1291 error = cse_create(fcr, sop);
1292 if (cmd == CIOCGSESSION && error == 0)
1293 session2_op_to_op(sop, (void *)data);
1296 ses = *(uint32_t *)data;
1297 if (!cse_delete(fcr, ses)) {
1298 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1303 cop = (struct crypt_op *)data;
1304 cse = cse_find(fcr, cop->ses);
1306 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1309 error = cryptodev_op(cse, cop);
1313 error = cryptodev_find((struct crypt_find_op *)data);
1316 caead = (struct crypt_aead *)data;
1317 cse = cse_find(fcr, caead->ses);
1319 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1322 error = cryptodev_aead(cse, caead);
1327 SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
1331 #ifdef COMPAT_FREEBSD32
1333 case CIOCGSESSION32:
1335 session_op_to_32((void *)data, data32);
1337 case CIOCGSESSION232:
1339 session2_op_to_32((void *)data, data32);
1343 crypt_op_to_32((void *)data, data32);
1345 case CIOCCRYPTAEAD32:
1347 crypt_aead_to_32((void *)data, data32);
1354 static struct cdevsw crypto_cdevsw = {
1355 .d_version = D_VERSION,
1356 .d_open = crypto_open,
1357 .d_ioctl = crypto_ioctl,
1360 static struct cdev *crypto_dev;
1363 * Initialization code, both for static and dynamic loading.
1366 cryptodev_modevent(module_t mod, int type, void *unused)
1371 printf("crypto: <crypto device>\n");
1372 crypto_dev = make_dev(&crypto_cdevsw, 0,
1373 UID_ROOT, GID_WHEEL, 0666,
1377 /*XXX disallow if active sessions */
1378 destroy_dev(crypto_dev);
1384 static moduledata_t cryptodev_mod = {
1389 MODULE_VERSION(cryptodev, 1);
1390 DECLARE_MODULE(cryptodev, cryptodev_mod, SI_SUB_PSEUDO, SI_ORDER_ANY);
1391 MODULE_DEPEND(cryptodev, crypto, 1, 1, 1);
1392 MODULE_DEPEND(cryptodev, zlib, 1, 1, 1);