2 * Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd@FreeBSD.org>
3 * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org>
4 * Copyright (c) 2014 The FreeBSD Foundation
7 * Portions of this software were developed by John-Mark Gurney
8 * under sponsorship of the FreeBSD Foundation and
9 * Rubicon Communications, LLC (Netgate).
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
33 #include <sys/cdefs.h>
34 __FBSDID("$FreeBSD$");
36 #include <sys/param.h>
37 #include <sys/systm.h>
38 #include <sys/kernel.h>
40 #include <sys/libkern.h>
42 #include <sys/module.h>
43 #include <sys/malloc.h>
44 #include <sys/rwlock.h>
49 #include <crypto/aesni/aesni.h>
50 #include <cryptodev_if.h>
51 #include <opencrypto/gmac.h>
53 static struct mtx_padalign *ctx_mtx;
54 static struct fpu_kern_ctx **ctx_fpu;
60 TAILQ_HEAD(aesni_sessions_head, aesni_session) sessions;
64 #define AQUIRE_CTX(i, ctx) \
66 (i) = PCPU_GET(cpuid); \
67 mtx_lock(&ctx_mtx[(i)]); \
68 (ctx) = ctx_fpu[(i)]; \
70 #define RELEASE_CTX(i, ctx) \
72 mtx_unlock(&ctx_mtx[(i)]); \
77 static int aesni_newsession(device_t, uint32_t *sidp, struct cryptoini *cri);
78 static int aesni_freesession(device_t, uint64_t tid);
79 static void aesni_freesession_locked(struct aesni_softc *sc,
80 struct aesni_session *ses);
81 static int aesni_cipher_setup(struct aesni_session *ses,
82 struct cryptoini *encini);
83 static int aesni_cipher_process(struct aesni_session *ses,
84 struct cryptodesc *enccrd, struct cryptodesc *authcrd, struct cryptop *crp);
86 MALLOC_DEFINE(M_AESNI, "aesni_data", "AESNI Data");
89 aesni_identify(driver_t *drv, device_t parent)
92 /* NB: order 10 is so we get attached after h/w devices */
93 if (device_find_child(parent, "aesni", -1) == NULL &&
94 BUS_ADD_CHILD(parent, 10, "aesni", -1) == 0)
95 panic("aesni: could not attach");
99 aesni_probe(device_t dev)
102 if ((cpu_feature2 & CPUID2_AESNI) == 0) {
103 device_printf(dev, "No AESNI support.\n");
107 if ((cpu_feature2 & CPUID2_SSE41) == 0) {
108 device_printf(dev, "No SSE4.1 support.\n");
112 device_set_desc_copy(dev, "AES-CBC,AES-XTS,AES-GCM,AES-ICM");
121 /* XXX - no way to return driverid */
123 if (ctx_fpu[i] != NULL) {
124 mtx_destroy(&ctx_mtx[i]);
125 fpu_kern_free_ctx(ctx_fpu[i]);
129 free(ctx_mtx, M_AESNI);
131 free(ctx_fpu, M_AESNI);
136 aesni_attach(device_t dev)
138 struct aesni_softc *sc;
141 sc = device_get_softc(dev);
143 TAILQ_INIT(&sc->sessions);
146 sc->cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE |
149 device_printf(dev, "Could not get crypto driver id.\n");
153 ctx_mtx = malloc(sizeof *ctx_mtx * (mp_maxid + 1), M_AESNI,
155 ctx_fpu = malloc(sizeof *ctx_fpu * (mp_maxid + 1), M_AESNI,
159 ctx_fpu[i] = fpu_kern_alloc_ctx(0);
160 mtx_init(&ctx_mtx[i], "anifpumtx", NULL, MTX_DEF|MTX_NEW);
163 rw_init(&sc->lock, "aesni_lock");
164 crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0);
165 crypto_register(sc->cid, CRYPTO_AES_ICM, 0, 0);
166 crypto_register(sc->cid, CRYPTO_AES_NIST_GCM_16, 0, 0);
167 crypto_register(sc->cid, CRYPTO_AES_128_NIST_GMAC, 0, 0);
168 crypto_register(sc->cid, CRYPTO_AES_192_NIST_GMAC, 0, 0);
169 crypto_register(sc->cid, CRYPTO_AES_256_NIST_GMAC, 0, 0);
170 crypto_register(sc->cid, CRYPTO_AES_XTS, 0, 0);
175 aesni_detach(device_t dev)
177 struct aesni_softc *sc;
178 struct aesni_session *ses;
180 sc = device_get_softc(dev);
183 TAILQ_FOREACH(ses, &sc->sessions, next) {
185 rw_wunlock(&sc->lock);
187 "Cannot detach, sessions still active.\n");
192 while ((ses = TAILQ_FIRST(&sc->sessions)) != NULL) {
193 TAILQ_REMOVE(&sc->sessions, ses, next);
196 rw_wunlock(&sc->lock);
197 crypto_unregister_all(sc->cid);
199 rw_destroy(&sc->lock);
207 aesni_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri)
209 struct aesni_softc *sc;
210 struct aesni_session *ses;
211 struct cryptoini *encini;
214 if (sidp == NULL || cri == NULL) {
215 CRYPTDEB("no sidp or cri");
219 sc = device_get_softc(dev);
225 for (; cri != NULL; cri = cri->cri_next) {
226 switch (cri->cri_alg) {
230 case CRYPTO_AES_NIST_GCM_16:
231 if (encini != NULL) {
232 CRYPTDEB("encini already set");
237 case CRYPTO_AES_128_NIST_GMAC:
238 case CRYPTO_AES_192_NIST_GMAC:
239 case CRYPTO_AES_256_NIST_GMAC:
241 * nothing to do here, maybe in the future cache some
246 CRYPTDEB("unhandled algorithm");
250 if (encini == NULL) {
251 CRYPTDEB("no cipher");
257 rw_wunlock(&sc->lock);
261 * Free sessions goes first, so if first session is used, we need to
264 ses = TAILQ_FIRST(&sc->sessions);
265 if (ses == NULL || ses->used) {
266 ses = malloc(sizeof(*ses), M_AESNI, M_NOWAIT | M_ZERO);
268 rw_wunlock(&sc->lock);
273 TAILQ_REMOVE(&sc->sessions, ses, next);
276 TAILQ_INSERT_TAIL(&sc->sessions, ses, next);
277 rw_wunlock(&sc->lock);
278 ses->algo = encini->cri_alg;
280 error = aesni_cipher_setup(ses, encini);
282 CRYPTDEB("setup failed");
284 aesni_freesession_locked(sc, ses);
285 rw_wunlock(&sc->lock);
294 aesni_freesession_locked(struct aesni_softc *sc, struct aesni_session *ses)
298 rw_assert(&sc->lock, RA_WLOCKED);
301 TAILQ_REMOVE(&sc->sessions, ses, next);
302 *ses = (struct aesni_session){};
304 TAILQ_INSERT_HEAD(&sc->sessions, ses, next);
308 aesni_freesession(device_t dev, uint64_t tid)
310 struct aesni_softc *sc;
311 struct aesni_session *ses;
314 sc = device_get_softc(dev);
315 sid = ((uint32_t)tid) & 0xffffffff;
317 TAILQ_FOREACH_REVERSE(ses, &sc->sessions, aesni_sessions_head, next) {
322 rw_wunlock(&sc->lock);
325 aesni_freesession_locked(sc, ses);
326 rw_wunlock(&sc->lock);
331 aesni_process(device_t dev, struct cryptop *crp, int hint __unused)
333 struct aesni_softc *sc = device_get_softc(dev);
334 struct aesni_session *ses = NULL;
335 struct cryptodesc *crd, *enccrd, *authcrd;
347 if (crp->crp_callback == NULL || crp->crp_desc == NULL) {
352 for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) {
353 switch (crd->crd_alg) {
357 if (enccrd != NULL) {
364 case CRYPTO_AES_NIST_GCM_16:
365 if (enccrd != NULL) {
373 case CRYPTO_AES_128_NIST_GMAC:
374 case CRYPTO_AES_192_NIST_GMAC:
375 case CRYPTO_AES_256_NIST_GMAC:
376 if (authcrd != NULL) {
390 if (enccrd == NULL || (needauth && authcrd == NULL)) {
395 /* CBC & XTS can only handle full blocks for now */
396 if ((enccrd->crd_alg == CRYPTO_AES_CBC || enccrd->crd_alg ==
397 CRYPTO_AES_XTS) && (enccrd->crd_len % AES_BLOCK_LEN) != 0) {
403 TAILQ_FOREACH_REVERSE(ses, &sc->sessions, aesni_sessions_head, next) {
404 if (ses->id == (crp->crp_sid & 0xffffffff))
407 rw_runlock(&sc->lock);
413 error = aesni_cipher_process(ses, enccrd, authcrd, crp);
418 crp->crp_etype = error;
424 aesni_cipher_alloc(struct cryptodesc *enccrd, struct cryptop *crp,
432 if (crp->crp_flags & CRYPTO_F_IMBUF) {
433 m = (struct mbuf *)crp->crp_buf;
434 if (m->m_next != NULL)
436 addr = mtod(m, uint8_t *);
437 } else if (crp->crp_flags & CRYPTO_F_IOV) {
438 uio = (struct uio *)crp->crp_buf;
439 if (uio->uio_iovcnt != 1)
442 addr = (uint8_t *)iov->iov_base;
444 addr = (uint8_t *)crp->crp_buf;
446 addr += enccrd->crd_skip;
450 addr = malloc(enccrd->crd_len, M_AESNI, M_NOWAIT);
453 crypto_copydata(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
454 enccrd->crd_len, addr);
460 static device_method_t aesni_methods[] = {
461 DEVMETHOD(device_identify, aesni_identify),
462 DEVMETHOD(device_probe, aesni_probe),
463 DEVMETHOD(device_attach, aesni_attach),
464 DEVMETHOD(device_detach, aesni_detach),
466 DEVMETHOD(cryptodev_newsession, aesni_newsession),
467 DEVMETHOD(cryptodev_freesession, aesni_freesession),
468 DEVMETHOD(cryptodev_process, aesni_process),
473 static driver_t aesni_driver = {
476 sizeof(struct aesni_softc),
478 static devclass_t aesni_devclass;
480 DRIVER_MODULE(aesni, nexus, aesni_driver, aesni_devclass, 0, 0);
481 MODULE_VERSION(aesni, 1);
482 MODULE_DEPEND(aesni, crypto, 1, 1, 1);
485 aesni_cipher_setup(struct aesni_session *ses, struct cryptoini *encini)
487 struct fpu_kern_ctx *ctx;
491 kt = is_fpu_kern_thread(0);
493 AQUIRE_CTX(ctxidx, ctx);
494 error = fpu_kern_enter(curthread, ctx,
495 FPU_KERN_NORMAL | FPU_KERN_KTHR);
500 error = aesni_cipher_setup_common(ses, encini->cri_key,
504 fpu_kern_leave(curthread, ctx);
506 RELEASE_CTX(ctxidx, ctx);
512 * authcrd contains the associated date.
515 aesni_cipher_process(struct aesni_session *ses, struct cryptodesc *enccrd,
516 struct cryptodesc *authcrd, struct cryptop *crp)
518 struct fpu_kern_ctx *ctx;
519 uint8_t iv[AES_BLOCK_LEN];
520 uint8_t tag[GMAC_DIGEST_LEN];
521 uint8_t *buf, *authbuf;
522 int error, allocated, authallocated;
526 encflag = (enccrd->crd_flags & CRD_F_ENCRYPT) == CRD_F_ENCRYPT;
528 if ((enccrd->crd_alg == CRYPTO_AES_ICM ||
529 enccrd->crd_alg == CRYPTO_AES_NIST_GCM_16) &&
530 (enccrd->crd_flags & CRD_F_IV_EXPLICIT) == 0)
533 buf = aesni_cipher_alloc(enccrd, crp, &allocated);
540 if (authcrd != NULL) {
541 authbuf = aesni_cipher_alloc(authcrd, crp, &authallocated);
542 if (authbuf == NULL) {
548 kt = is_fpu_kern_thread(0);
550 AQUIRE_CTX(ctxidx, ctx);
551 error = fpu_kern_enter(curthread, ctx,
552 FPU_KERN_NORMAL|FPU_KERN_KTHR);
557 if ((enccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) {
558 error = aesni_cipher_setup_common(ses, enccrd->crd_key,
564 /* XXX - validate that enccrd and authcrd have/use same key? */
565 switch (enccrd->crd_alg) {
568 ivlen = AES_BLOCK_LEN;
573 case CRYPTO_AES_NIST_GCM_16:
574 ivlen = 12; /* should support arbitarily larger */
580 if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
581 bcopy(enccrd->crd_iv, iv, ivlen);
583 arc4rand(iv, ivlen, 0);
585 if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0)
586 crypto_copyback(crp->crp_flags, crp->crp_buf,
587 enccrd->crd_inject, ivlen, iv);
589 if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
590 bcopy(enccrd->crd_iv, iv, ivlen);
592 crypto_copydata(crp->crp_flags, crp->crp_buf,
593 enccrd->crd_inject, ivlen, iv);
596 if (authcrd != NULL && !encflag)
597 crypto_copydata(crp->crp_flags, crp->crp_buf,
598 authcrd->crd_inject, GMAC_DIGEST_LEN, tag);
600 bzero(tag, sizeof tag);
606 aesni_encrypt_cbc(ses->rounds, ses->enc_schedule,
607 enccrd->crd_len, buf, buf, iv);
609 aesni_decrypt_cbc(ses->rounds, ses->dec_schedule,
610 enccrd->crd_len, buf, iv);
613 /* encryption & decryption are the same */
614 aesni_encrypt_icm(ses->rounds, ses->enc_schedule,
615 enccrd->crd_len, buf, buf, iv);
619 aesni_encrypt_xts(ses->rounds, ses->enc_schedule,
620 ses->xts_schedule, enccrd->crd_len, buf, buf,
623 aesni_decrypt_xts(ses->rounds, ses->dec_schedule,
624 ses->xts_schedule, enccrd->crd_len, buf, buf,
627 case CRYPTO_AES_NIST_GCM_16:
629 AES_GCM_encrypt(buf, buf, authbuf, iv, tag,
630 enccrd->crd_len, authcrd->crd_len, ivlen,
631 ses->enc_schedule, ses->rounds);
633 if (!AES_GCM_decrypt(buf, buf, authbuf, iv, tag,
634 enccrd->crd_len, authcrd->crd_len, ivlen,
635 ses->enc_schedule, ses->rounds))
642 crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
643 enccrd->crd_len, buf);
645 if (!error && authcrd != NULL) {
646 crypto_copyback(crp->crp_flags, crp->crp_buf,
647 authcrd->crd_inject, GMAC_DIGEST_LEN, tag);
652 fpu_kern_leave(curthread, ctx);
654 RELEASE_CTX(ctxidx, ctx);
659 bzero(buf, enccrd->crd_len);
663 free(authbuf, M_AESNI);