2 * Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd@FreeBSD.org>
3 * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org>
4 * Copyright (c) 2014,2016 The FreeBSD Foundation
5 * Copyright (c) 2020 Ampere Computing
8 * Portions of this software were developed by John-Mark Gurney
9 * under sponsorship of the FreeBSD Foundation and
10 * Rubicon Communications, LLC (Netgate).
12 * This software was developed by Andrew Turner under
13 * sponsorship from the FreeBSD Foundation.
15 * Redistribution and use in source and binary forms, with or without
16 * modification, are permitted provided that the following conditions
18 * 1. Redistributions of source code must retain the above copyright
19 * notice, this list of conditions and the following disclaimer.
20 * 2. Redistributions in binary form must reproduce the above copyright
21 * notice, this list of conditions and the following disclaimer in the
22 * documentation and/or other materials provided with the distribution.
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
38 * This is based on the aesni code.
41 #include <sys/cdefs.h>
42 __FBSDID("$FreeBSD$");
44 #include <sys/param.h>
45 #include <sys/systm.h>
46 #include <sys/kernel.h>
48 #include <sys/endian.h>
50 #include <sys/malloc.h>
52 #include <sys/module.h>
53 #include <sys/mutex.h>
54 #include <sys/queue.h>
55 #include <sys/rwlock.h>
59 #include <machine/vfp.h>
61 #include <opencrypto/cryptodev.h>
62 #include <opencrypto/gmac.h>
63 #include <cryptodev_if.h>
64 #include <crypto/armv8/armv8_crypto.h>
65 #include <crypto/rijndael/rijndael.h>
67 struct armv8_crypto_softc {
74 static struct mtx *ctx_mtx;
75 static struct fpu_kern_ctx **ctx_vfp;
77 #define AQUIRE_CTX(i, ctx) \
79 (i) = PCPU_GET(cpuid); \
80 mtx_lock(&ctx_mtx[(i)]); \
81 (ctx) = ctx_vfp[(i)]; \
83 #define RELEASE_CTX(i, ctx) \
85 mtx_unlock(&ctx_mtx[(i)]); \
90 static int armv8_crypto_cipher_process(struct armv8_crypto_session *,
93 MALLOC_DEFINE(M_ARMV8_CRYPTO, "armv8_crypto", "ARMv8 Crypto Data");
96 armv8_crypto_identify(driver_t *drv, device_t parent)
99 /* NB: order 10 is so we get attached after h/w devices */
100 if (device_find_child(parent, "armv8crypto", -1) == NULL &&
101 BUS_ADD_CHILD(parent, 10, "armv8crypto", -1) == 0)
102 panic("ARMv8 crypto: could not attach");
106 armv8_crypto_probe(device_t dev)
111 reg = READ_SPECIALREG(id_aa64isar0_el1);
113 switch (ID_AA64ISAR0_AES_VAL(reg)) {
114 case ID_AA64ISAR0_AES_BASE:
116 device_set_desc(dev, "AES-CBC,AES-XTS");
118 case ID_AA64ISAR0_AES_PMULL:
120 device_set_desc(dev, "AES-CBC,AES-XTS,AES-GCM");
124 case ID_AA64ISAR0_AES_NONE:
125 device_printf(dev, "CPU lacks AES instructions\n");
129 /* TODO: Check more fields as we support more features */
135 armv8_crypto_attach(device_t dev)
137 struct armv8_crypto_softc *sc;
141 sc = device_get_softc(dev);
144 reg = READ_SPECIALREG(id_aa64isar0_el1);
146 if (ID_AA64ISAR0_AES_VAL(reg) == ID_AA64ISAR0_AES_PMULL)
149 sc->cid = crypto_get_driverid(dev, sizeof(struct armv8_crypto_session),
150 CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC | CRYPTOCAP_F_ACCEL_SOFTWARE);
152 device_printf(dev, "Could not get crypto driver id.\n");
156 rw_init(&sc->lock, "armv8crypto");
158 ctx_mtx = malloc(sizeof(*ctx_mtx) * (mp_maxid + 1), M_ARMV8_CRYPTO,
160 ctx_vfp = malloc(sizeof(*ctx_vfp) * (mp_maxid + 1), M_ARMV8_CRYPTO,
164 ctx_vfp[i] = fpu_kern_alloc_ctx(0);
165 mtx_init(&ctx_mtx[i], "armv8cryptoctx", NULL, MTX_DEF|MTX_NEW);
172 armv8_crypto_detach(device_t dev)
174 struct armv8_crypto_softc *sc;
177 sc = device_get_softc(dev);
181 rw_wunlock(&sc->lock);
182 crypto_unregister_all(sc->cid);
184 rw_destroy(&sc->lock);
187 if (ctx_vfp[i] != NULL) {
188 mtx_destroy(&ctx_mtx[i]);
189 fpu_kern_free_ctx(ctx_vfp[i]);
193 free(ctx_mtx, M_ARMV8_CRYPTO);
195 free(ctx_vfp, M_ARMV8_CRYPTO);
201 #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)
204 armv8_crypto_probesession(device_t dev,
205 const struct crypto_session_params *csp)
207 struct armv8_crypto_softc *sc;
209 sc = device_get_softc(dev);
211 if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0)
214 switch (csp->csp_mode) {
216 switch (csp->csp_cipher_alg) {
217 case CRYPTO_AES_NIST_GCM_16:
220 if (csp->csp_auth_mlen != 0 &&
221 csp->csp_auth_mlen != GMAC_DIGEST_LEN)
223 switch (csp->csp_cipher_klen * 8) {
236 case CSP_MODE_CIPHER:
237 switch (csp->csp_cipher_alg) {
239 if (csp->csp_ivlen != AES_BLOCK_LEN)
241 switch (csp->csp_cipher_klen * 8) {
251 if (csp->csp_ivlen != AES_XTS_IV_LEN)
253 switch (csp->csp_cipher_klen * 8) {
268 return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
272 armv8_crypto_cipher_setup(struct armv8_crypto_session *ses,
273 const struct crypto_session_params *csp, const uint8_t *key, int keylen)
276 struct fpu_kern_ctx *ctx;
279 if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
282 switch (keylen * 8) {
291 kt = is_fpu_kern_thread(0);
294 fpu_kern_enter(curthread, ctx,
295 FPU_KERN_NORMAL | FPU_KERN_KTHR);
298 aes_v8_set_encrypt_key(key,
299 keylen * 8, &ses->enc_schedule);
301 if ((csp->csp_cipher_alg == CRYPTO_AES_XTS) ||
302 (csp->csp_cipher_alg == CRYPTO_AES_CBC))
303 aes_v8_set_decrypt_key(key,
304 keylen * 8, &ses->dec_schedule);
306 if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
307 aes_v8_set_encrypt_key(key + keylen, keylen * 8, &ses->xts_schedule);
309 if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) {
310 memset(H.c, 0, sizeof(H.c));
311 aes_v8_encrypt(H.c, H.c, &ses->enc_schedule);
312 H.u[0] = bswap64(H.u[0]);
313 H.u[1] = bswap64(H.u[1]);
314 gcm_init_v8(ses->Htable, H.u);
318 fpu_kern_leave(curthread, ctx);
326 armv8_crypto_newsession(device_t dev, crypto_session_t cses,
327 const struct crypto_session_params *csp)
329 struct armv8_crypto_softc *sc;
330 struct armv8_crypto_session *ses;
333 sc = device_get_softc(dev);
336 rw_wunlock(&sc->lock);
340 ses = crypto_get_driver_session(cses);
341 error = armv8_crypto_cipher_setup(ses, csp, csp->csp_cipher_key,
342 csp->csp_cipher_klen);
343 rw_wunlock(&sc->lock);
348 armv8_crypto_process(device_t dev, struct cryptop *crp, int hint __unused)
350 struct armv8_crypto_session *ses;
352 ses = crypto_get_driver_session(crp->crp_session);
353 crp->crp_etype = armv8_crypto_cipher_process(ses, crp);
359 armv8_crypto_cipher_alloc(struct cryptop *crp, int start, int length, int *allocated)
363 addr = crypto_contiguous_subsegment(crp, start, length);
368 addr = malloc(crp->crp_payload_length, M_ARMV8_CRYPTO, M_NOWAIT);
371 crypto_copydata(crp, start, length, addr);
378 armv8_crypto_cipher_process(struct armv8_crypto_session *ses,
381 struct crypto_buffer_cursor fromc, toc;
382 const struct crypto_session_params *csp;
383 struct fpu_kern_ctx *ctx;
385 uint8_t iv[AES_BLOCK_LEN], tag[GMAC_DIGEST_LEN];
386 int authallocated, i;
391 csp = crypto_get_params(crp->crp_session);
392 encflag = CRYPTO_OP_IS_ENCRYPT(crp->crp_op);
398 if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) {
399 if (crp->crp_aad != NULL)
400 authbuf = crp->crp_aad;
402 authbuf = armv8_crypto_cipher_alloc(crp, crp->crp_aad_start,
403 crp->crp_aad_length, &authallocated);
404 if (authbuf == NULL) {
409 crypto_cursor_init(&fromc, &crp->crp_buf);
410 crypto_cursor_advance(&fromc, crp->crp_payload_start);
411 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
412 crypto_cursor_init(&toc, &crp->crp_obuf);
413 crypto_cursor_advance(&toc, crp->crp_payload_output_start);
415 crypto_cursor_copy(&fromc, &toc);
418 kt = is_fpu_kern_thread(0);
421 fpu_kern_enter(curthread, ctx,
422 FPU_KERN_NORMAL | FPU_KERN_KTHR);
425 if (crp->crp_cipher_key != NULL) {
426 armv8_crypto_cipher_setup(ses, csp, crp->crp_cipher_key,
427 csp->csp_cipher_klen);
430 crypto_read_iv(crp, iv);
432 switch (csp->csp_cipher_alg) {
434 if ((crp->crp_payload_length % AES_BLOCK_LEN) != 0) {
439 armv8_aes_encrypt_cbc(&ses->enc_schedule,
440 crp->crp_payload_length, &fromc, &toc, iv);
442 armv8_aes_decrypt_cbc(&ses->dec_schedule,
443 crp->crp_payload_length, &fromc, &toc, iv);
447 armv8_aes_encrypt_xts(&ses->enc_schedule,
448 &ses->xts_schedule.aes_key, crp->crp_payload_length,
451 armv8_aes_decrypt_xts(&ses->dec_schedule,
452 &ses->xts_schedule.aes_key, crp->crp_payload_length,
455 case CRYPTO_AES_NIST_GCM_16:
457 memset(tag, 0, sizeof(tag));
458 armv8_aes_encrypt_gcm(&ses->enc_schedule,
459 crp->crp_payload_length, &fromc, &toc,
460 crp->crp_aad_length, authbuf, tag, iv, ses->Htable);
461 crypto_copyback(crp, crp->crp_digest_start, sizeof(tag),
464 crypto_copydata(crp, crp->crp_digest_start, sizeof(tag),
466 error = armv8_aes_decrypt_gcm(&ses->enc_schedule,
467 crp->crp_payload_length, &fromc, &toc,
468 crp->crp_aad_length, authbuf, tag, iv, ses->Htable);
478 fpu_kern_leave(curthread, ctx);
483 zfree(authbuf, M_ARMV8_CRYPTO);
484 explicit_bzero(iv, sizeof(iv));
485 explicit_bzero(tag, sizeof(tag));
490 static device_method_t armv8_crypto_methods[] = {
491 DEVMETHOD(device_identify, armv8_crypto_identify),
492 DEVMETHOD(device_probe, armv8_crypto_probe),
493 DEVMETHOD(device_attach, armv8_crypto_attach),
494 DEVMETHOD(device_detach, armv8_crypto_detach),
496 DEVMETHOD(cryptodev_probesession, armv8_crypto_probesession),
497 DEVMETHOD(cryptodev_newsession, armv8_crypto_newsession),
498 DEVMETHOD(cryptodev_process, armv8_crypto_process),
503 static DEFINE_CLASS_0(armv8crypto, armv8_crypto_driver, armv8_crypto_methods,
504 sizeof(struct armv8_crypto_softc));
505 static devclass_t armv8_crypto_devclass;
507 DRIVER_MODULE(armv8crypto, nexus, armv8_crypto_driver, armv8_crypto_devclass,