sys/crypto/aesni/aesni_wrap.c

   1 /*-
   2  * Copyright (C) 2008 Damien Miller <djm@mindrot.org>
   3  * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org>
   4  * Copyright (c) 2010-2011 Pawel Jakub Dawidek <pawel@dawidek.net>
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  26  * SUCH DAMAGE.
  27  */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include <sys/param.h>
  33 #include <sys/libkern.h>
  34 #include <sys/malloc.h>
  35 #include <sys/proc.h>
  36 #include <sys/systm.h>
  37 #include <crypto/aesni/aesni.h>
  38
  39 MALLOC_DECLARE(M_AESNI);
  40
  41 void
  42 aesni_encrypt_cbc(int rounds, const void *key_schedule, size_t len,
  43     const uint8_t *from, uint8_t *to, const uint8_t iv[AES_BLOCK_LEN])
  44 {
  45         const uint8_t *ivp;
  46         size_t i;
  47
  48         len /= AES_BLOCK_LEN;
  49         ivp = iv;
  50         for (i = 0; i < len; i++) {
  51                 aesni_enc(rounds - 1, key_schedule, from, to, ivp);
  52                 ivp = to;
  53                 from += AES_BLOCK_LEN;
  54                 to += AES_BLOCK_LEN;
  55         }
  56 }
  57
  58 void
  59 aesni_encrypt_ecb(int rounds, const void *key_schedule, size_t len,
  60     const uint8_t from[AES_BLOCK_LEN], uint8_t to[AES_BLOCK_LEN])
  61 {
  62         size_t i;
  63
  64         len /= AES_BLOCK_LEN;
  65         for (i = 0; i < len; i++) {
  66                 aesni_enc(rounds - 1, key_schedule, from, to, NULL);
  67                 from += AES_BLOCK_LEN;
  68                 to += AES_BLOCK_LEN;
  69         }
  70 }
  71
  72 void
  73 aesni_decrypt_ecb(int rounds, const void *key_schedule, size_t len,
  74     const uint8_t from[AES_BLOCK_LEN], uint8_t to[AES_BLOCK_LEN])
  75 {
  76         size_t i;
  77
  78         len /= AES_BLOCK_LEN;
  79         for (i = 0; i < len; i++) {
  80                 aesni_dec(rounds - 1, key_schedule, from, to, NULL);
  81                 from += AES_BLOCK_LEN;
  82                 to += AES_BLOCK_LEN;
  83         }
  84 }
  85
  86 #define AES_XTS_BLOCKSIZE       16
  87 #define AES_XTS_IVSIZE          8
  88 #define AES_XTS_ALPHA           0x87    /* GF(2^128) generator polynomial */
  89
  90 static void
  91 aesni_crypt_xts_block(int rounds, const void *key_schedule, uint64_t *tweak,
  92     const uint64_t *from, uint64_t *to, uint64_t *block, int do_encrypt)
  93 {
  94         int carry;
  95
  96         block[0] = from[0] ^ tweak[0];
  97         block[1] = from[1] ^ tweak[1];
  98
  99         if (do_encrypt)
 100                 aesni_enc(rounds - 1, key_schedule, (uint8_t *)block, (uint8_t *)to, NULL);
 101         else
 102                 aesni_dec(rounds - 1, key_schedule, (uint8_t *)block, (uint8_t *)to, NULL);
 103
 104         to[0] ^= tweak[0];
 105         to[1] ^= tweak[1];
 106
 107         /* Exponentiate tweak. */
 108         carry = ((tweak[0] & 0x8000000000000000ULL) > 0);
 109         tweak[0] <<= 1;
 110         if (tweak[1] & 0x8000000000000000ULL) {
 111                 uint8_t *twk = (uint8_t *)tweak;
 112
 113                 twk[0] ^= AES_XTS_ALPHA;
 114         }
 115         tweak[1] <<= 1;
 116         if (carry)
 117                 tweak[1] |= 1;
 118 }
 119
 120 static void
 121 aesni_crypt_xts(int rounds, const void *data_schedule,
 122     const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to,
 123     const uint8_t iv[AES_BLOCK_LEN], int do_encrypt)
 124 {
 125         uint64_t block[AES_XTS_BLOCKSIZE / 8];
 126         uint8_t tweak[AES_XTS_BLOCKSIZE];
 127         size_t i;
 128
 129         /*
 130          * Prepare tweak as E_k2(IV). IV is specified as LE representation
 131          * of a 64-bit block number which we allow to be passed in directly.
 132          */
 133 #if BYTE_ORDER == LITTLE_ENDIAN
 134         bcopy(iv, tweak, AES_XTS_IVSIZE);
 135         /* Last 64 bits of IV are always zero. */
 136         bzero(tweak + AES_XTS_IVSIZE, AES_XTS_IVSIZE);
 137 #else
 138 #error Only LITTLE_ENDIAN architectures are supported.
 139 #endif
 140         aesni_enc(rounds - 1, tweak_schedule, tweak, tweak, NULL);
 141
 142         len /= AES_XTS_BLOCKSIZE;
 143         for (i = 0; i < len; i++) {
 144                 aesni_crypt_xts_block(rounds, data_schedule, (uint64_t *)tweak,
 145                     (const uint64_t *)from, (uint64_t *)to, block, do_encrypt);
 146                 from += AES_XTS_BLOCKSIZE;
 147                 to += AES_XTS_BLOCKSIZE;
 148         }
 149
 150         bzero(tweak, sizeof(tweak));
 151         bzero(block, sizeof(block));
 152 }
 153
 154 static void
 155 aesni_encrypt_xts(int rounds, const void *data_schedule,
 156     const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to,
 157     const uint8_t iv[AES_BLOCK_LEN])
 158 {
 159
 160         aesni_crypt_xts(rounds, data_schedule, tweak_schedule, len, from, to,
 161             iv, 1);
 162 }
 163
 164 static void
 165 aesni_decrypt_xts(int rounds, const void *data_schedule,
 166     const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to,
 167     const uint8_t iv[AES_BLOCK_LEN])
 168 {
 169
 170         aesni_crypt_xts(rounds, data_schedule, tweak_schedule, len, from, to,
 171             iv, 0);
 172 }
 173
 174 static int
 175 aesni_cipher_setup_common(struct aesni_session *ses, const uint8_t *key,
 176     int keylen)
 177 {
 178
 179         switch (ses->algo) {
 180         case CRYPTO_AES_CBC:
 181                 switch (keylen) {
 182                 case 128:
 183                         ses->rounds = AES128_ROUNDS;
 184                         break;
 185                 case 192:
 186                         ses->rounds = AES192_ROUNDS;
 187                         break;
 188                 case 256:
 189                         ses->rounds = AES256_ROUNDS;
 190                         break;
 191                 default:
 192                         return (EINVAL);
 193                 }
 194                 break;
 195         case CRYPTO_AES_XTS:
 196                 switch (keylen) {
 197                 case 256:
 198                         ses->rounds = AES128_ROUNDS;
 199                         break;
 200                 case 512:
 201                         ses->rounds = AES256_ROUNDS;
 202                         break;
 203                 default:
 204                         return (EINVAL);
 205                 }
 206                 break;
 207         default:
 208                 return (EINVAL);
 209         }
 210
 211         aesni_set_enckey(key, ses->enc_schedule, ses->rounds);
 212         aesni_set_deckey(ses->enc_schedule, ses->dec_schedule, ses->rounds);
 213         if (ses->algo == CRYPTO_AES_CBC)
 214                 arc4rand(ses->iv, sizeof(ses->iv), 0);
 215         else /* if (ses->algo == CRYPTO_AES_XTS) */ {
 216                 aesni_set_enckey(key + keylen / 16, ses->xts_schedule,
 217                     ses->rounds);
 218         }
 219
 220         return (0);
 221 }
 222
 223 int
 224 aesni_cipher_setup(struct aesni_session *ses, struct cryptoini *encini)
 225 {
 226         struct thread *td;
 227         int error, saved_ctx;
 228
 229         td = curthread;
 230         if (!is_fpu_kern_thread(0)) {
 231                 error = fpu_kern_enter(td, ses->fpu_ctx, FPU_KERN_NORMAL);
 232                 saved_ctx = 1;
 233         } else {
 234                 error = 0;
 235                 saved_ctx = 0;
 236         }
 237         if (error == 0) {
 238                 error = aesni_cipher_setup_common(ses, encini->cri_key,
 239                     encini->cri_klen);
 240                 if (saved_ctx)
 241                         fpu_kern_leave(td, ses->fpu_ctx);
 242         }
 243         return (error);
 244 }
 245
 246 int
 247 aesni_cipher_process(struct aesni_session *ses, struct cryptodesc *enccrd,
 248     struct cryptop *crp)
 249 {
 250         struct thread *td;
 251         uint8_t *buf;
 252         int error, allocated, saved_ctx;
 253
 254         buf = aesni_cipher_alloc(enccrd, crp, &allocated);
 255         if (buf == NULL)
 256                 return (ENOMEM);
 257
 258         td = curthread;
 259         if (!is_fpu_kern_thread(0)) {
 260                 error = fpu_kern_enter(td, ses->fpu_ctx, FPU_KERN_NORMAL);
 261                 if (error != 0)
 262                         goto out;
 263                 saved_ctx = 1;
 264         } else {
 265                 saved_ctx = 0;
 266                 error = 0;
 267         }
 268
 269         if ((enccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) {
 270                 error = aesni_cipher_setup_common(ses, enccrd->crd_key,
 271                     enccrd->crd_klen);
 272                 if (error != 0)
 273                         goto out;
 274         }
 275
 276         if ((enccrd->crd_flags & CRD_F_ENCRYPT) != 0) {
 277                 if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
 278                         bcopy(enccrd->crd_iv, ses->iv, AES_BLOCK_LEN);
 279                 if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0)
 280                         crypto_copyback(crp->crp_flags, crp->crp_buf,
 281                             enccrd->crd_inject, AES_BLOCK_LEN, ses->iv);
 282                 if (ses->algo == CRYPTO_AES_CBC) {
 283                         aesni_encrypt_cbc(ses->rounds, ses->enc_schedule,
 284                             enccrd->crd_len, buf, buf, ses->iv);
 285                 } else /* if (ses->algo == CRYPTO_AES_XTS) */ {
 286                         aesni_encrypt_xts(ses->rounds, ses->enc_schedule,
 287                             ses->xts_schedule, enccrd->crd_len, buf, buf,
 288                             ses->iv);
 289                 }
 290         } else {
 291                 if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
 292                         bcopy(enccrd->crd_iv, ses->iv, AES_BLOCK_LEN);
 293                 else
 294                         crypto_copydata(crp->crp_flags, crp->crp_buf,
 295                             enccrd->crd_inject, AES_BLOCK_LEN, ses->iv);
 296                 if (ses->algo == CRYPTO_AES_CBC) {
 297                         aesni_decrypt_cbc(ses->rounds, ses->dec_schedule,
 298                             enccrd->crd_len, buf, ses->iv);
 299                 } else /* if (ses->algo == CRYPTO_AES_XTS) */ {
 300                         aesni_decrypt_xts(ses->rounds, ses->dec_schedule,
 301                             ses->xts_schedule, enccrd->crd_len, buf, buf,
 302                             ses->iv);
 303                 }
 304         }
 305         if (saved_ctx)
 306                 fpu_kern_leave(td, ses->fpu_ctx);
 307         if (allocated)
 308                 crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
 309                     enccrd->crd_len, buf);
 310         if ((enccrd->crd_flags & CRD_F_ENCRYPT) != 0)
 311                 crypto_copydata(crp->crp_flags, crp->crp_buf,
 312                     enccrd->crd_skip + enccrd->crd_len - AES_BLOCK_LEN,
 313                     AES_BLOCK_LEN, ses->iv);
 314  out:
 315         if (allocated) {
 316                 bzero(buf, enccrd->crd_len);
 317                 free(buf, M_AESNI);
 318         }
 319         return (error);
 320 }