sys/crypto/armv8/armv8_crypto.c

   1 /*-
   2  * Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd@FreeBSD.org>
   3  * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org>
   4  * Copyright (c) 2014,2016 The FreeBSD Foundation
   5  * Copyright (c) 2020 Ampere Computing
   6  * All rights reserved.
   7  *
   8  * Portions of this software were developed by John-Mark Gurney
   9  * under sponsorship of the FreeBSD Foundation and
  10  * Rubicon Communications, LLC (Netgate).
  11  *
  12  * This software was developed by Andrew Turner under
  13  * sponsorship from the FreeBSD Foundation.
  14  *
  15  * Redistribution and use in source and binary forms, with or without
  16  * modification, are permitted provided that the following conditions
  17  * are met:
  18  * 1. Redistributions of source code must retain the above copyright
  19  *    notice, this list of conditions and the following disclaimer.
  20  * 2. Redistributions in binary form must reproduce the above copyright
  21  *    notice, this list of conditions and the following disclaimer in the
  22  *    documentation and/or other materials provided with the distribution.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  25  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  27  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  28  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  29  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  30  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  31  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  32  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  33  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  34  * SUCH DAMAGE.
  35  */
  36
  37 /*
  38  * This is based on the aesni code.
  39  */
  40
  41 #include <sys/cdefs.h>
  42 __FBSDID("$FreeBSD$");
  43
  44 #include <sys/param.h>
  45 #include <sys/systm.h>
  46 #include <sys/kernel.h>
  47 #include <sys/bus.h>
  48 #include <sys/endian.h>
  49 #include <sys/lock.h>
  50 #include <sys/malloc.h>
  51 #include <sys/mbuf.h>
  52 #include <sys/module.h>
  53 #include <sys/mutex.h>
  54 #include <sys/queue.h>
  55 #include <sys/rwlock.h>
  56 #include <sys/smp.h>
  57 #include <sys/uio.h>
  58
  59 #include <machine/vfp.h>
  60
  61 #include <opencrypto/cryptodev.h>
  62 #include <opencrypto/gmac.h>
  63 #include <cryptodev_if.h>
  64 #include <crypto/armv8/armv8_crypto.h>
  65 #include <crypto/rijndael/rijndael.h>
  66
  67 struct armv8_crypto_softc {
  68         int             dieing;
  69         int32_t         cid;
  70         struct rwlock   lock;
  71         bool            has_pmul;
  72 };
  73
  74 static struct mtx *ctx_mtx;
  75 static struct fpu_kern_ctx **ctx_vfp;
  76
  77 #define AQUIRE_CTX(i, ctx)                                      \
  78         do {                                                    \
  79                 (i) = PCPU_GET(cpuid);                          \
  80                 mtx_lock(&ctx_mtx[(i)]);                        \
  81                 (ctx) = ctx_vfp[(i)];                           \
  82         } while (0)
  83 #define RELEASE_CTX(i, ctx)                                     \
  84         do {                                                    \
  85                 mtx_unlock(&ctx_mtx[(i)]);                      \
  86                 (i) = -1;                                       \
  87                 (ctx) = NULL;                                   \
  88         } while (0)
  89
  90 static int armv8_crypto_cipher_process(struct armv8_crypto_session *,
  91     struct cryptop *);
  92
  93 MALLOC_DEFINE(M_ARMV8_CRYPTO, "armv8_crypto", "ARMv8 Crypto Data");
  94
  95 static void
  96 armv8_crypto_identify(driver_t *drv, device_t parent)
  97 {
  98
  99         /* NB: order 10 is so we get attached after h/w devices */
 100         if (device_find_child(parent, "armv8crypto", -1) == NULL &&
 101             BUS_ADD_CHILD(parent, 10, "armv8crypto", -1) == 0)
 102                 panic("ARMv8 crypto: could not attach");
 103 }
 104
 105 static int
 106 armv8_crypto_probe(device_t dev)
 107 {
 108         uint64_t reg;
 109         int ret = ENXIO;
 110
 111         reg = READ_SPECIALREG(id_aa64isar0_el1);
 112
 113         switch (ID_AA64ISAR0_AES_VAL(reg)) {
 114         case ID_AA64ISAR0_AES_BASE:
 115                 ret = 0;
 116                 device_set_desc(dev, "AES-CBC,AES-XTS");
 117                 break;
 118         case ID_AA64ISAR0_AES_PMULL:
 119                 ret = 0;
 120                 device_set_desc(dev, "AES-CBC,AES-XTS,AES-GCM");
 121                 break;
 122         default:
 123                 break;
 124         case ID_AA64ISAR0_AES_NONE:
 125                 device_printf(dev, "CPU lacks AES instructions\n");
 126                 break;
 127         }
 128
 129         /* TODO: Check more fields as we support more features */
 130
 131         return (ret);
 132 }
 133
 134 static int
 135 armv8_crypto_attach(device_t dev)
 136 {
 137         struct armv8_crypto_softc *sc;
 138         uint64_t reg;
 139         int i;
 140
 141         sc = device_get_softc(dev);
 142         sc->dieing = 0;
 143
 144         reg = READ_SPECIALREG(id_aa64isar0_el1);
 145
 146         if (ID_AA64ISAR0_AES_VAL(reg) == ID_AA64ISAR0_AES_PMULL)
 147                 sc->has_pmul = true;
 148
 149         sc->cid = crypto_get_driverid(dev, sizeof(struct armv8_crypto_session),
 150             CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC | CRYPTOCAP_F_ACCEL_SOFTWARE);
 151         if (sc->cid < 0) {
 152                 device_printf(dev, "Could not get crypto driver id.\n");
 153                 return (ENOMEM);
 154         }
 155
 156         rw_init(&sc->lock, "armv8crypto");
 157
 158         ctx_mtx = malloc(sizeof(*ctx_mtx) * (mp_maxid + 1), M_ARMV8_CRYPTO,
 159             M_WAITOK|M_ZERO);
 160         ctx_vfp = malloc(sizeof(*ctx_vfp) * (mp_maxid + 1), M_ARMV8_CRYPTO,
 161             M_WAITOK|M_ZERO);
 162
 163         CPU_FOREACH(i) {
 164                 ctx_vfp[i] = fpu_kern_alloc_ctx(0);
 165                 mtx_init(&ctx_mtx[i], "armv8cryptoctx", NULL, MTX_DEF|MTX_NEW);
 166         }
 167
 168         return (0);
 169 }
 170
 171 static int
 172 armv8_crypto_detach(device_t dev)
 173 {
 174         struct armv8_crypto_softc *sc;
 175         int i;
 176
 177         sc = device_get_softc(dev);
 178
 179         rw_wlock(&sc->lock);
 180         sc->dieing = 1;
 181         rw_wunlock(&sc->lock);
 182         crypto_unregister_all(sc->cid);
 183
 184         rw_destroy(&sc->lock);
 185
 186         CPU_FOREACH(i) {
 187                 if (ctx_vfp[i] != NULL) {
 188                         mtx_destroy(&ctx_mtx[i]);
 189                         fpu_kern_free_ctx(ctx_vfp[i]);
 190                 }
 191                 ctx_vfp[i] = NULL;
 192         }
 193         free(ctx_mtx, M_ARMV8_CRYPTO);
 194         ctx_mtx = NULL;
 195         free(ctx_vfp, M_ARMV8_CRYPTO);
 196         ctx_vfp = NULL;
 197
 198         return (0);
 199 }
 200
 201 #define SUPPORTED_SES (CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)
 202
 203 static int
 204 armv8_crypto_probesession(device_t dev,
 205     const struct crypto_session_params *csp)
 206 {
 207         struct armv8_crypto_softc *sc;
 208
 209         sc = device_get_softc(dev);
 210
 211         if ((csp->csp_flags & ~(SUPPORTED_SES)) != 0)
 212                 return (EINVAL);
 213
 214         switch (csp->csp_mode) {
 215         case CSP_MODE_AEAD:
 216                 switch (csp->csp_cipher_alg) {
 217                 case CRYPTO_AES_NIST_GCM_16:
 218                         if (!sc->has_pmul)
 219                                 return (EINVAL);
 220                         if (csp->csp_ivlen != AES_GCM_IV_LEN)
 221                                 return (EINVAL);
 222                         if (csp->csp_auth_mlen != 0 &&
 223                             csp->csp_auth_mlen != GMAC_DIGEST_LEN)
 224                                 return (EINVAL);
 225                         switch (csp->csp_cipher_klen * 8) {
 226                         case 128:
 227                         case 192:
 228                         case 256:
 229                                 break;
 230                         default:
 231                                 return (EINVAL);
 232                         }
 233                         break;
 234                 default:
 235                         return (EINVAL);
 236                 }
 237                 break;
 238         case CSP_MODE_CIPHER:
 239                 switch (csp->csp_cipher_alg) {
 240                 case CRYPTO_AES_CBC:
 241                         if (csp->csp_ivlen != AES_BLOCK_LEN)
 242                                 return (EINVAL);
 243                         switch (csp->csp_cipher_klen * 8) {
 244                         case 128:
 245                         case 192:
 246                         case 256:
 247                                 break;
 248                         default:
 249                                 return (EINVAL);
 250                         }
 251                         break;
 252                 case CRYPTO_AES_XTS:
 253                         if (csp->csp_ivlen != AES_XTS_IV_LEN)
 254                                 return (EINVAL);
 255                         switch (csp->csp_cipher_klen * 8) {
 256                         case 256:
 257                         case 512:
 258                                 break;
 259                         default:
 260                                 return (EINVAL);
 261                         }
 262                         break;
 263                 default:
 264                         return (EINVAL);
 265                 }
 266                 break;
 267         default:
 268                 return (EINVAL);
 269         }
 270         return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
 271 }
 272
 273 static int
 274 armv8_crypto_cipher_setup(struct armv8_crypto_session *ses,
 275     const struct crypto_session_params *csp, const uint8_t *key, int keylen)
 276 {
 277         __uint128_val_t H;
 278         struct fpu_kern_ctx *ctx;
 279         int kt, i;
 280
 281         if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
 282                 keylen /= 2;
 283
 284         switch (keylen * 8) {
 285         case 128:
 286         case 192:
 287         case 256:
 288                 break;
 289         default:
 290                 return (EINVAL);
 291         }
 292
 293         kt = is_fpu_kern_thread(0);
 294         if (!kt) {
 295                 AQUIRE_CTX(i, ctx);
 296                 fpu_kern_enter(curthread, ctx,
 297                     FPU_KERN_NORMAL | FPU_KERN_KTHR);
 298         }
 299
 300         aes_v8_set_encrypt_key(key,
 301             keylen * 8, &ses->enc_schedule);
 302
 303         if ((csp->csp_cipher_alg == CRYPTO_AES_XTS) ||
 304             (csp->csp_cipher_alg == CRYPTO_AES_CBC))
 305                 aes_v8_set_decrypt_key(key,
 306                     keylen * 8, &ses->dec_schedule);
 307
 308         if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
 309                 aes_v8_set_encrypt_key(key + keylen, keylen * 8, &ses->xts_schedule);
 310
 311         if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) {
 312                 memset(H.c, 0, sizeof(H.c));
 313                 aes_v8_encrypt(H.c, H.c, &ses->enc_schedule);
 314                 H.u[0] = bswap64(H.u[0]);
 315                 H.u[1] = bswap64(H.u[1]);
 316                 gcm_init_v8(ses->Htable, H.u);
 317         }
 318
 319         if (!kt) {
 320                 fpu_kern_leave(curthread, ctx);
 321                 RELEASE_CTX(i, ctx);
 322         }
 323
 324         return (0);
 325 }
 326
 327 static int
 328 armv8_crypto_newsession(device_t dev, crypto_session_t cses,
 329     const struct crypto_session_params *csp)
 330 {
 331         struct armv8_crypto_softc *sc;
 332         struct armv8_crypto_session *ses;
 333         int error;
 334
 335         sc = device_get_softc(dev);
 336         rw_wlock(&sc->lock);
 337         if (sc->dieing) {
 338                 rw_wunlock(&sc->lock);
 339                 return (EINVAL);
 340         }
 341
 342         ses = crypto_get_driver_session(cses);
 343         error = armv8_crypto_cipher_setup(ses, csp, csp->csp_cipher_key,
 344             csp->csp_cipher_klen);
 345         rw_wunlock(&sc->lock);
 346         return (error);
 347 }
 348
 349 static int
 350 armv8_crypto_process(device_t dev, struct cryptop *crp, int hint __unused)
 351 {
 352         struct armv8_crypto_session *ses;
 353
 354         ses = crypto_get_driver_session(crp->crp_session);
 355         crp->crp_etype = armv8_crypto_cipher_process(ses, crp);
 356         crypto_done(crp);
 357         return (0);
 358 }
 359
 360 static uint8_t *
 361 armv8_crypto_cipher_alloc(struct cryptop *crp, int start, int length, int *allocated)
 362 {
 363         uint8_t *addr;
 364
 365         addr = crypto_contiguous_subsegment(crp, start, length);
 366         if (addr != NULL) {
 367                 *allocated = 0;
 368                 return (addr);
 369         }
 370         addr = malloc(crp->crp_payload_length, M_ARMV8_CRYPTO, M_NOWAIT);
 371         if (addr != NULL) {
 372                 *allocated = 1;
 373                 crypto_copydata(crp, start, length, addr);
 374         } else
 375                 *allocated = 0;
 376         return (addr);
 377 }
 378
 379 static int
 380 armv8_crypto_cipher_process(struct armv8_crypto_session *ses,
 381     struct cryptop *crp)
 382 {
 383         const struct crypto_session_params *csp;
 384         struct fpu_kern_ctx *ctx;
 385         uint8_t *buf, *authbuf, *outbuf;
 386         uint8_t iv[AES_BLOCK_LEN], tag[GMAC_DIGEST_LEN];
 387         int allocated, authallocated, outallocated, i;
 388         int encflag;
 389         int kt;
 390         int error;
 391         bool outcopy;
 392
 393         csp = crypto_get_params(crp->crp_session);
 394         encflag = CRYPTO_OP_IS_ENCRYPT(crp->crp_op);
 395
 396         allocated = 0;
 397         outallocated = 0;
 398         authallocated = 0;
 399         authbuf = NULL;
 400         kt = 1;
 401
 402         buf = armv8_crypto_cipher_alloc(crp, crp->crp_payload_start,
 403             crp->crp_payload_length, &allocated);
 404         if (buf == NULL)
 405                 return (ENOMEM);
 406
 407         if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) {
 408                 if (crp->crp_aad != NULL)
 409                         authbuf = crp->crp_aad;
 410                 else
 411                         authbuf = armv8_crypto_cipher_alloc(crp, crp->crp_aad_start,
 412                             crp->crp_aad_length, &authallocated);
 413                 if (authbuf == NULL) {
 414                         error = ENOMEM;
 415                         goto out;
 416                 }
 417         }
 418
 419         if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
 420                 outbuf = crypto_buffer_contiguous_subsegment(&crp->crp_obuf,
 421                     crp->crp_payload_output_start, crp->crp_payload_length);
 422                 if (outbuf == NULL) {
 423                         outcopy = true;
 424                         if (allocated)
 425                                 outbuf = buf;
 426                         else {
 427                                 outbuf = malloc(crp->crp_payload_length,
 428                                     M_ARMV8_CRYPTO, M_NOWAIT);
 429                                 if (outbuf == NULL) {
 430                                         error = ENOMEM;
 431                                         goto out;
 432                                 }
 433                                 outallocated = true;
 434                         }
 435                 } else
 436                         outcopy = false;
 437         } else {
 438                 outbuf = buf;
 439                 outcopy = allocated;
 440         }
 441
 442         kt = is_fpu_kern_thread(0);
 443         if (!kt) {
 444                 AQUIRE_CTX(i, ctx);
 445                 fpu_kern_enter(curthread, ctx,
 446                     FPU_KERN_NORMAL | FPU_KERN_KTHR);
 447         }
 448
 449         if (crp->crp_cipher_key != NULL) {
 450                 armv8_crypto_cipher_setup(ses, csp, crp->crp_cipher_key,
 451                     csp->csp_cipher_klen);
 452         }
 453
 454         crypto_read_iv(crp, iv);
 455
 456         /* Do work */
 457         switch (csp->csp_cipher_alg) {
 458         case CRYPTO_AES_CBC:
 459                 if ((crp->crp_payload_length % AES_BLOCK_LEN) != 0) {
 460                         error = EINVAL;
 461                         goto out;
 462                 }
 463                 if (encflag)
 464                         armv8_aes_encrypt_cbc(&ses->enc_schedule,
 465                             crp->crp_payload_length, buf, buf, iv);
 466                 else
 467                         armv8_aes_decrypt_cbc(&ses->dec_schedule,
 468                             crp->crp_payload_length, buf, iv);
 469                 break;
 470         case CRYPTO_AES_XTS:
 471                 if (encflag)
 472                         armv8_aes_encrypt_xts(&ses->enc_schedule,
 473                             &ses->xts_schedule.aes_key, crp->crp_payload_length, buf,
 474                             buf, iv);
 475                 else
 476                         armv8_aes_decrypt_xts(&ses->dec_schedule,
 477                             &ses->xts_schedule.aes_key, crp->crp_payload_length, buf,
 478                             buf, iv);
 479                 break;
 480         case CRYPTO_AES_NIST_GCM_16:
 481                 if (encflag) {
 482                         memset(tag, 0, sizeof(tag));
 483                         armv8_aes_encrypt_gcm(&ses->enc_schedule,
 484                             crp->crp_payload_length,
 485                             buf, outbuf,
 486                             crp->crp_aad_length, authbuf,
 487                             tag, iv, ses->Htable);
 488                         crypto_copyback(crp, crp->crp_digest_start, sizeof(tag),
 489                             tag);
 490                 } else {
 491                         crypto_copydata(crp, crp->crp_digest_start, sizeof(tag),
 492                             tag);
 493                         if (armv8_aes_decrypt_gcm(&ses->enc_schedule,
 494                             crp->crp_payload_length,
 495                             buf, outbuf,
 496                             crp->crp_aad_length, authbuf,
 497                             tag, iv, ses->Htable) != 0) {
 498                                 error = EBADMSG;
 499                                 goto out;
 500                         }
 501                 }
 502                 break;
 503         }
 504
 505         if (outcopy)
 506                 crypto_copyback(crp, CRYPTO_HAS_OUTPUT_BUFFER(crp) ?
 507                     crp->crp_payload_output_start : crp->crp_payload_start,
 508                     crp->crp_payload_length, outbuf);
 509
 510         error = 0;
 511 out:
 512         if (!kt) {
 513                 fpu_kern_leave(curthread, ctx);
 514                 RELEASE_CTX(i, ctx);
 515         }
 516
 517         if (allocated)
 518                 zfree(buf, M_ARMV8_CRYPTO);
 519         if (authallocated)
 520                 zfree(authbuf, M_ARMV8_CRYPTO);
 521         if (outallocated)
 522                 zfree(outbuf, M_ARMV8_CRYPTO);
 523         explicit_bzero(iv, sizeof(iv));
 524         explicit_bzero(tag, sizeof(tag));
 525
 526         return (error);
 527 }
 528
 529 static device_method_t armv8_crypto_methods[] = {
 530         DEVMETHOD(device_identify,      armv8_crypto_identify),
 531         DEVMETHOD(device_probe,         armv8_crypto_probe),
 532         DEVMETHOD(device_attach,        armv8_crypto_attach),
 533         DEVMETHOD(device_detach,        armv8_crypto_detach),
 534
 535         DEVMETHOD(cryptodev_probesession, armv8_crypto_probesession),
 536         DEVMETHOD(cryptodev_newsession, armv8_crypto_newsession),
 537         DEVMETHOD(cryptodev_process,    armv8_crypto_process),
 538
 539         DEVMETHOD_END,
 540 };
 541
 542 static DEFINE_CLASS_0(armv8crypto, armv8_crypto_driver, armv8_crypto_methods,
 543     sizeof(struct armv8_crypto_softc));
 544 static devclass_t armv8_crypto_devclass;
 545
 546 DRIVER_MODULE(armv8crypto, nexus, armv8_crypto_driver, armv8_crypto_devclass,
 547     0, 0);