contrib/bearssl/src/symcipher/aes_x86ni_ctrcbc.c

   1 /*
   2  * Copyright (c) 2017 Thomas Pornin <pornin@bolet.org>
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining
   5  * a copy of this software and associated documentation files (the
   6  * "Software"), to deal in the Software without restriction, including
   7  * without limitation the rights to use, copy, modify, merge, publish,
   8  * distribute, sublicense, and/or sell copies of the Software, and to
   9  * permit persons to whom the Software is furnished to do so, subject to
  10  * the following conditions:
  11  *
  12  * The above copyright notice and this permission notice shall be
  13  * included in all copies or substantial portions of the Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  16  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  17  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  18  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  19  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  20  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  21  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  22  * SOFTWARE.
  23  */
  24
  25 #define BR_ENABLE_INTRINSICS   1
  26 #include "inner.h"
  27
  28 #if BR_AES_X86NI
  29
  30 /* see bearssl_block.h */
  31 const br_block_ctrcbc_class *
  32 br_aes_x86ni_ctrcbc_get_vtable(void)
  33 {
  34         return br_aes_x86ni_supported() ? &br_aes_x86ni_ctrcbc_vtable : NULL;
  35 }
  36
  37 /* see bearssl_block.h */
  38 void
  39 br_aes_x86ni_ctrcbc_init(br_aes_x86ni_ctrcbc_keys *ctx,
  40         const void *key, size_t len)
  41 {
  42         ctx->vtable = &br_aes_x86ni_ctrcbc_vtable;
  43         ctx->num_rounds = br_aes_x86ni_keysched_enc(ctx->skey.skni, key, len);
  44 }
  45
  46 BR_TARGETS_X86_UP
  47
  48 /* see bearssl_block.h */
  49 BR_TARGET("sse2,sse4.1,aes")
  50 void
  51 br_aes_x86ni_ctrcbc_ctr(const br_aes_x86ni_ctrcbc_keys *ctx,
  52         void *ctr, void *data, size_t len)
  53 {
  54         unsigned char *buf;
  55         unsigned num_rounds;
  56         __m128i sk[15];
  57         __m128i ivx0, ivx1, ivx2, ivx3;
  58         __m128i erev, zero, one, four, notthree;
  59         unsigned u;
  60
  61         buf = data;
  62         num_rounds = ctx->num_rounds;
  63         for (u = 0; u <= num_rounds; u ++) {
  64                 sk[u] = _mm_loadu_si128((void *)(ctx->skey.skni + (u << 4)));
  65         }
  66
  67         /*
  68          * Some SSE2 constants.
  69          */
  70         erev = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7,
  71                 8, 9, 10, 11, 12, 13, 14, 15);
  72         zero = _mm_setzero_si128();
  73         one = _mm_set_epi64x(0, 1);
  74         four = _mm_set_epi64x(0, 4);
  75         notthree = _mm_sub_epi64(zero, four);
  76
  77         /*
  78          * Decode the counter in big-endian and pre-increment the other
  79          * three counters.
  80          */
  81         ivx0 = _mm_shuffle_epi8(_mm_loadu_si128((void *)ctr), erev);
  82         ivx1 = _mm_add_epi64(ivx0, one);
  83         ivx1 = _mm_sub_epi64(ivx1,
  84                 _mm_slli_si128(_mm_cmpeq_epi64(ivx1, zero), 8));
  85         ivx2 = _mm_add_epi64(ivx1, one);
  86         ivx2 = _mm_sub_epi64(ivx2,
  87                 _mm_slli_si128(_mm_cmpeq_epi64(ivx2, zero), 8));
  88         ivx3 = _mm_add_epi64(ivx2, one);
  89         ivx3 = _mm_sub_epi64(ivx3,
  90                 _mm_slli_si128(_mm_cmpeq_epi64(ivx3, zero), 8));
  91         while (len > 0) {
  92                 __m128i x0, x1, x2, x3;
  93
  94                 /*
  95                  * Load counter values; we need to byteswap them because
  96                  * the specification says that they use big-endian.
  97                  */
  98                 x0 = _mm_shuffle_epi8(ivx0, erev);
  99                 x1 = _mm_shuffle_epi8(ivx1, erev);
 100                 x2 = _mm_shuffle_epi8(ivx2, erev);
 101                 x3 = _mm_shuffle_epi8(ivx3, erev);
 102
 103                 x0 = _mm_xor_si128(x0, sk[0]);
 104                 x1 = _mm_xor_si128(x1, sk[0]);
 105                 x2 = _mm_xor_si128(x2, sk[0]);
 106                 x3 = _mm_xor_si128(x3, sk[0]);
 107                 x0 = _mm_aesenc_si128(x0, sk[1]);
 108                 x1 = _mm_aesenc_si128(x1, sk[1]);
 109                 x2 = _mm_aesenc_si128(x2, sk[1]);
 110                 x3 = _mm_aesenc_si128(x3, sk[1]);
 111                 x0 = _mm_aesenc_si128(x0, sk[2]);
 112                 x1 = _mm_aesenc_si128(x1, sk[2]);
 113                 x2 = _mm_aesenc_si128(x2, sk[2]);
 114                 x3 = _mm_aesenc_si128(x3, sk[2]);
 115                 x0 = _mm_aesenc_si128(x0, sk[3]);
 116                 x1 = _mm_aesenc_si128(x1, sk[3]);
 117                 x2 = _mm_aesenc_si128(x2, sk[3]);
 118                 x3 = _mm_aesenc_si128(x3, sk[3]);
 119                 x0 = _mm_aesenc_si128(x0, sk[4]);
 120                 x1 = _mm_aesenc_si128(x1, sk[4]);
 121                 x2 = _mm_aesenc_si128(x2, sk[4]);
 122                 x3 = _mm_aesenc_si128(x3, sk[4]);
 123                 x0 = _mm_aesenc_si128(x0, sk[5]);
 124                 x1 = _mm_aesenc_si128(x1, sk[5]);
 125                 x2 = _mm_aesenc_si128(x2, sk[5]);
 126                 x3 = _mm_aesenc_si128(x3, sk[5]);
 127                 x0 = _mm_aesenc_si128(x0, sk[6]);
 128                 x1 = _mm_aesenc_si128(x1, sk[6]);
 129                 x2 = _mm_aesenc_si128(x2, sk[6]);
 130                 x3 = _mm_aesenc_si128(x3, sk[6]);
 131                 x0 = _mm_aesenc_si128(x0, sk[7]);
 132                 x1 = _mm_aesenc_si128(x1, sk[7]);
 133                 x2 = _mm_aesenc_si128(x2, sk[7]);
 134                 x3 = _mm_aesenc_si128(x3, sk[7]);
 135                 x0 = _mm_aesenc_si128(x0, sk[8]);
 136                 x1 = _mm_aesenc_si128(x1, sk[8]);
 137                 x2 = _mm_aesenc_si128(x2, sk[8]);
 138                 x3 = _mm_aesenc_si128(x3, sk[8]);
 139                 x0 = _mm_aesenc_si128(x0, sk[9]);
 140                 x1 = _mm_aesenc_si128(x1, sk[9]);
 141                 x2 = _mm_aesenc_si128(x2, sk[9]);
 142                 x3 = _mm_aesenc_si128(x3, sk[9]);
 143                 if (num_rounds == 10) {
 144                         x0 = _mm_aesenclast_si128(x0, sk[10]);
 145                         x1 = _mm_aesenclast_si128(x1, sk[10]);
 146                         x2 = _mm_aesenclast_si128(x2, sk[10]);
 147                         x3 = _mm_aesenclast_si128(x3, sk[10]);
 148                 } else if (num_rounds == 12) {
 149                         x0 = _mm_aesenc_si128(x0, sk[10]);
 150                         x1 = _mm_aesenc_si128(x1, sk[10]);
 151                         x2 = _mm_aesenc_si128(x2, sk[10]);
 152                         x3 = _mm_aesenc_si128(x3, sk[10]);
 153                         x0 = _mm_aesenc_si128(x0, sk[11]);
 154                         x1 = _mm_aesenc_si128(x1, sk[11]);
 155                         x2 = _mm_aesenc_si128(x2, sk[11]);
 156                         x3 = _mm_aesenc_si128(x3, sk[11]);
 157                         x0 = _mm_aesenclast_si128(x0, sk[12]);
 158                         x1 = _mm_aesenclast_si128(x1, sk[12]);
 159                         x2 = _mm_aesenclast_si128(x2, sk[12]);
 160                         x3 = _mm_aesenclast_si128(x3, sk[12]);
 161                 } else {
 162                         x0 = _mm_aesenc_si128(x0, sk[10]);
 163                         x1 = _mm_aesenc_si128(x1, sk[10]);
 164                         x2 = _mm_aesenc_si128(x2, sk[10]);
 165                         x3 = _mm_aesenc_si128(x3, sk[10]);
 166                         x0 = _mm_aesenc_si128(x0, sk[11]);
 167                         x1 = _mm_aesenc_si128(x1, sk[11]);
 168                         x2 = _mm_aesenc_si128(x2, sk[11]);
 169                         x3 = _mm_aesenc_si128(x3, sk[11]);
 170                         x0 = _mm_aesenc_si128(x0, sk[12]);
 171                         x1 = _mm_aesenc_si128(x1, sk[12]);
 172                         x2 = _mm_aesenc_si128(x2, sk[12]);
 173                         x3 = _mm_aesenc_si128(x3, sk[12]);
 174                         x0 = _mm_aesenc_si128(x0, sk[13]);
 175                         x1 = _mm_aesenc_si128(x1, sk[13]);
 176                         x2 = _mm_aesenc_si128(x2, sk[13]);
 177                         x3 = _mm_aesenc_si128(x3, sk[13]);
 178                         x0 = _mm_aesenclast_si128(x0, sk[14]);
 179                         x1 = _mm_aesenclast_si128(x1, sk[14]);
 180                         x2 = _mm_aesenclast_si128(x2, sk[14]);
 181                         x3 = _mm_aesenclast_si128(x3, sk[14]);
 182                 }
 183                 if (len >= 64) {
 184                         x0 = _mm_xor_si128(x0,
 185                                 _mm_loadu_si128((void *)(buf +  0)));
 186                         x1 = _mm_xor_si128(x1,
 187                                 _mm_loadu_si128((void *)(buf + 16)));
 188                         x2 = _mm_xor_si128(x2,
 189                                 _mm_loadu_si128((void *)(buf + 32)));
 190                         x3 = _mm_xor_si128(x3,
 191                                 _mm_loadu_si128((void *)(buf + 48)));
 192                         _mm_storeu_si128((void *)(buf +  0), x0);
 193                         _mm_storeu_si128((void *)(buf + 16), x1);
 194                         _mm_storeu_si128((void *)(buf + 32), x2);
 195                         _mm_storeu_si128((void *)(buf + 48), x3);
 196                         buf += 64;
 197                         len -= 64;
 198                 } else {
 199                         unsigned char tmp[64];
 200
 201                         _mm_storeu_si128((void *)(tmp +  0), x0);
 202                         _mm_storeu_si128((void *)(tmp + 16), x1);
 203                         _mm_storeu_si128((void *)(tmp + 32), x2);
 204                         _mm_storeu_si128((void *)(tmp + 48), x3);
 205                         for (u = 0; u < len; u ++) {
 206                                 buf[u] ^= tmp[u];
 207                         }
 208                         switch (len) {
 209                         case 16:
 210                                 ivx0 = ivx1;
 211                                 break;
 212                         case 32:
 213                                 ivx0 = ivx2;
 214                                 break;
 215                         case 48:
 216                                 ivx0 = ivx3;
 217                                 break;
 218                         }
 219                         break;
 220                 }
 221
 222                 /*
 223                  * Add 4 to each counter value. For carry propagation
 224                  * into the upper 64-bit words, we would need to compare
 225                  * the results with 4, but SSE2+ has only _signed_
 226                  * comparisons. Instead, we mask out the low two bits,
 227                  * and check whether the remaining bits are zero.
 228                  */
 229                 ivx0 = _mm_add_epi64(ivx0, four);
 230                 ivx1 = _mm_add_epi64(ivx1, four);
 231                 ivx2 = _mm_add_epi64(ivx2, four);
 232                 ivx3 = _mm_add_epi64(ivx3, four);
 233                 ivx0 = _mm_sub_epi64(ivx0,
 234                         _mm_slli_si128(_mm_cmpeq_epi64(
 235                                 _mm_and_si128(ivx0, notthree), zero), 8));
 236                 ivx1 = _mm_sub_epi64(ivx1,
 237                         _mm_slli_si128(_mm_cmpeq_epi64(
 238                                 _mm_and_si128(ivx1, notthree), zero), 8));
 239                 ivx2 = _mm_sub_epi64(ivx2,
 240                         _mm_slli_si128(_mm_cmpeq_epi64(
 241                                 _mm_and_si128(ivx2, notthree), zero), 8));
 242                 ivx3 = _mm_sub_epi64(ivx3,
 243                         _mm_slli_si128(_mm_cmpeq_epi64(
 244                                 _mm_and_si128(ivx3, notthree), zero), 8));
 245         }
 246
 247         /*
 248          * Write back new counter value. The loop took care to put the
 249          * right counter value in ivx0.
 250          */
 251         _mm_storeu_si128((void *)ctr, _mm_shuffle_epi8(ivx0, erev));
 252 }
 253
 254 /* see bearssl_block.h */
 255 BR_TARGET("sse2,sse4.1,aes")
 256 void
 257 br_aes_x86ni_ctrcbc_mac(const br_aes_x86ni_ctrcbc_keys *ctx,
 258         void *cbcmac, const void *data, size_t len)
 259 {
 260         const unsigned char *buf;
 261         unsigned num_rounds;
 262         __m128i sk[15], ivx;
 263         unsigned u;
 264
 265         buf = data;
 266         ivx = _mm_loadu_si128(cbcmac);
 267         num_rounds = ctx->num_rounds;
 268         for (u = 0; u <= num_rounds; u ++) {
 269                 sk[u] = _mm_loadu_si128((void *)(ctx->skey.skni + (u << 4)));
 270         }
 271         while (len > 0) {
 272                 __m128i x;
 273
 274                 x = _mm_xor_si128(_mm_loadu_si128((void *)buf), ivx);
 275                 x = _mm_xor_si128(x, sk[0]);
 276                 x = _mm_aesenc_si128(x, sk[1]);
 277                 x = _mm_aesenc_si128(x, sk[2]);
 278                 x = _mm_aesenc_si128(x, sk[3]);
 279                 x = _mm_aesenc_si128(x, sk[4]);
 280                 x = _mm_aesenc_si128(x, sk[5]);
 281                 x = _mm_aesenc_si128(x, sk[6]);
 282                 x = _mm_aesenc_si128(x, sk[7]);
 283                 x = _mm_aesenc_si128(x, sk[8]);
 284                 x = _mm_aesenc_si128(x, sk[9]);
 285                 if (num_rounds == 10) {
 286                         x = _mm_aesenclast_si128(x, sk[10]);
 287                 } else if (num_rounds == 12) {
 288                         x = _mm_aesenc_si128(x, sk[10]);
 289                         x = _mm_aesenc_si128(x, sk[11]);
 290                         x = _mm_aesenclast_si128(x, sk[12]);
 291                 } else {
 292                         x = _mm_aesenc_si128(x, sk[10]);
 293                         x = _mm_aesenc_si128(x, sk[11]);
 294                         x = _mm_aesenc_si128(x, sk[12]);
 295                         x = _mm_aesenc_si128(x, sk[13]);
 296                         x = _mm_aesenclast_si128(x, sk[14]);
 297                 }
 298                 ivx = x;
 299                 buf += 16;
 300                 len -= 16;
 301         }
 302         _mm_storeu_si128(cbcmac, ivx);
 303 }
 304
 305 /* see bearssl_block.h */
 306 BR_TARGET("sse2,sse4.1,aes")
 307 void
 308 br_aes_x86ni_ctrcbc_encrypt(const br_aes_x86ni_ctrcbc_keys *ctx,
 309         void *ctr, void *cbcmac, void *data, size_t len)
 310 {
 311         unsigned char *buf;
 312         unsigned num_rounds;
 313         __m128i sk[15];
 314         __m128i ivx, cmx;
 315         __m128i erev, zero, one;
 316         unsigned u;
 317         int first_iter;
 318
 319         num_rounds = ctx->num_rounds;
 320         for (u = 0; u <= num_rounds; u ++) {
 321                 sk[u] = _mm_loadu_si128((void *)(ctx->skey.skni + (u << 4)));
 322         }
 323
 324         /*
 325          * Some SSE2 constants.
 326          */
 327         erev = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7,
 328                 8, 9, 10, 11, 12, 13, 14, 15);
 329         zero = _mm_setzero_si128();
 330         one = _mm_set_epi64x(0, 1);
 331
 332         /*
 333          * Decode the counter in big-endian.
 334          */
 335         ivx = _mm_shuffle_epi8(_mm_loadu_si128(ctr), erev);
 336         cmx = _mm_loadu_si128(cbcmac);
 337
 338         buf = data;
 339         first_iter = 1;
 340         while (len > 0) {
 341                 __m128i dx, x0, x1;
 342
 343                 /*
 344                  * Load initial values:
 345                  *   dx   encrypted block of data
 346                  *   x0   counter (for CTR encryption)
 347                  *   x1   input for CBC-MAC
 348                  */
 349                 dx = _mm_loadu_si128((void *)buf);
 350                 x0 = _mm_shuffle_epi8(ivx, erev);
 351                 x1 = cmx;
 352
 353                 x0 = _mm_xor_si128(x0, sk[0]);
 354                 x1 = _mm_xor_si128(x1, sk[0]);
 355                 x0 = _mm_aesenc_si128(x0, sk[1]);
 356                 x1 = _mm_aesenc_si128(x1, sk[1]);
 357                 x0 = _mm_aesenc_si128(x0, sk[2]);
 358                 x1 = _mm_aesenc_si128(x1, sk[2]);
 359                 x0 = _mm_aesenc_si128(x0, sk[3]);
 360                 x1 = _mm_aesenc_si128(x1, sk[3]);
 361                 x0 = _mm_aesenc_si128(x0, sk[4]);
 362                 x1 = _mm_aesenc_si128(x1, sk[4]);
 363                 x0 = _mm_aesenc_si128(x0, sk[5]);
 364                 x1 = _mm_aesenc_si128(x1, sk[5]);
 365                 x0 = _mm_aesenc_si128(x0, sk[6]);
 366                 x1 = _mm_aesenc_si128(x1, sk[6]);
 367                 x0 = _mm_aesenc_si128(x0, sk[7]);
 368                 x1 = _mm_aesenc_si128(x1, sk[7]);
 369                 x0 = _mm_aesenc_si128(x0, sk[8]);
 370                 x1 = _mm_aesenc_si128(x1, sk[8]);
 371                 x0 = _mm_aesenc_si128(x0, sk[9]);
 372                 x1 = _mm_aesenc_si128(x1, sk[9]);
 373                 if (num_rounds == 10) {
 374                         x0 = _mm_aesenclast_si128(x0, sk[10]);
 375                         x1 = _mm_aesenclast_si128(x1, sk[10]);
 376                 } else if (num_rounds == 12) {
 377                         x0 = _mm_aesenc_si128(x0, sk[10]);
 378                         x1 = _mm_aesenc_si128(x1, sk[10]);
 379                         x0 = _mm_aesenc_si128(x0, sk[11]);
 380                         x1 = _mm_aesenc_si128(x1, sk[11]);
 381                         x0 = _mm_aesenclast_si128(x0, sk[12]);
 382                         x1 = _mm_aesenclast_si128(x1, sk[12]);
 383                 } else {
 384                         x0 = _mm_aesenc_si128(x0, sk[10]);
 385                         x1 = _mm_aesenc_si128(x1, sk[10]);
 386                         x0 = _mm_aesenc_si128(x0, sk[11]);
 387                         x1 = _mm_aesenc_si128(x1, sk[11]);
 388                         x0 = _mm_aesenc_si128(x0, sk[12]);
 389                         x1 = _mm_aesenc_si128(x1, sk[12]);
 390                         x0 = _mm_aesenc_si128(x0, sk[13]);
 391                         x1 = _mm_aesenc_si128(x1, sk[13]);
 392                         x0 = _mm_aesenclast_si128(x0, sk[14]);
 393                         x1 = _mm_aesenclast_si128(x1, sk[14]);
 394                 }
 395
 396                 x0 = _mm_xor_si128(x0, dx);
 397                 if (first_iter) {
 398                         cmx = _mm_xor_si128(cmx, x0);
 399                         first_iter = 0;
 400                 } else {
 401                         cmx = _mm_xor_si128(x1, x0);
 402                 }
 403                 _mm_storeu_si128((void *)buf, x0);
 404
 405                 buf += 16;
 406                 len -= 16;
 407
 408                 /*
 409                  * Increment the counter value.
 410                  */
 411                 ivx = _mm_add_epi64(ivx, one);
 412                 ivx = _mm_sub_epi64(ivx,
 413                         _mm_slli_si128(_mm_cmpeq_epi64(ivx, zero), 8));
 414
 415                 /*
 416                  * If this was the last iteration, then compute the
 417                  * extra block encryption to complete CBC-MAC.
 418                  */
 419                 if (len == 0) {
 420                         cmx = _mm_xor_si128(cmx, sk[0]);
 421                         cmx = _mm_aesenc_si128(cmx, sk[1]);
 422                         cmx = _mm_aesenc_si128(cmx, sk[2]);
 423                         cmx = _mm_aesenc_si128(cmx, sk[3]);
 424                         cmx = _mm_aesenc_si128(cmx, sk[4]);
 425                         cmx = _mm_aesenc_si128(cmx, sk[5]);
 426                         cmx = _mm_aesenc_si128(cmx, sk[6]);
 427                         cmx = _mm_aesenc_si128(cmx, sk[7]);
 428                         cmx = _mm_aesenc_si128(cmx, sk[8]);
 429                         cmx = _mm_aesenc_si128(cmx, sk[9]);
 430                         if (num_rounds == 10) {
 431                                 cmx = _mm_aesenclast_si128(cmx, sk[10]);
 432                         } else if (num_rounds == 12) {
 433                                 cmx = _mm_aesenc_si128(cmx, sk[10]);
 434                                 cmx = _mm_aesenc_si128(cmx, sk[11]);
 435                                 cmx = _mm_aesenclast_si128(cmx, sk[12]);
 436                         } else {
 437                                 cmx = _mm_aesenc_si128(cmx, sk[10]);
 438                                 cmx = _mm_aesenc_si128(cmx, sk[11]);
 439                                 cmx = _mm_aesenc_si128(cmx, sk[12]);
 440                                 cmx = _mm_aesenc_si128(cmx, sk[13]);
 441                                 cmx = _mm_aesenclast_si128(cmx, sk[14]);
 442                         }
 443                         break;
 444                 }
 445         }
 446
 447         /*
 448          * Write back new counter value and CBC-MAC value.
 449          */
 450         _mm_storeu_si128(ctr, _mm_shuffle_epi8(ivx, erev));
 451         _mm_storeu_si128(cbcmac, cmx);
 452 }
 453
 454 /* see bearssl_block.h */
 455 BR_TARGET("sse2,sse4.1,aes")
 456 void
 457 br_aes_x86ni_ctrcbc_decrypt(const br_aes_x86ni_ctrcbc_keys *ctx,
 458         void *ctr, void *cbcmac, void *data, size_t len)
 459 {
 460         unsigned char *buf;
 461         unsigned num_rounds;
 462         __m128i sk[15];
 463         __m128i ivx, cmx;
 464         __m128i erev, zero, one;
 465         unsigned u;
 466
 467         num_rounds = ctx->num_rounds;
 468         for (u = 0; u <= num_rounds; u ++) {
 469                 sk[u] = _mm_loadu_si128((void *)(ctx->skey.skni + (u << 4)));
 470         }
 471
 472         /*
 473          * Some SSE2 constants.
 474          */
 475         erev = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7,
 476                 8, 9, 10, 11, 12, 13, 14, 15);
 477         zero = _mm_setzero_si128();
 478         one = _mm_set_epi64x(0, 1);
 479
 480         /*
 481          * Decode the counter in big-endian.
 482          */
 483         ivx = _mm_shuffle_epi8(_mm_loadu_si128(ctr), erev);
 484         cmx = _mm_loadu_si128(cbcmac);
 485
 486         buf = data;
 487         while (len > 0) {
 488                 __m128i dx, x0, x1;
 489
 490                 /*
 491                  * Load initial values:
 492                  *   dx   encrypted block of data
 493                  *   x0   counter (for CTR encryption)
 494                  *   x1   input for CBC-MAC
 495                  */
 496                 dx = _mm_loadu_si128((void *)buf);
 497                 x0 = _mm_shuffle_epi8(ivx, erev);
 498                 x1 = _mm_xor_si128(cmx, dx);
 499
 500                 x0 = _mm_xor_si128(x0, sk[0]);
 501                 x1 = _mm_xor_si128(x1, sk[0]);
 502                 x0 = _mm_aesenc_si128(x0, sk[1]);
 503                 x1 = _mm_aesenc_si128(x1, sk[1]);
 504                 x0 = _mm_aesenc_si128(x0, sk[2]);
 505                 x1 = _mm_aesenc_si128(x1, sk[2]);
 506                 x0 = _mm_aesenc_si128(x0, sk[3]);
 507                 x1 = _mm_aesenc_si128(x1, sk[3]);
 508                 x0 = _mm_aesenc_si128(x0, sk[4]);
 509                 x1 = _mm_aesenc_si128(x1, sk[4]);
 510                 x0 = _mm_aesenc_si128(x0, sk[5]);
 511                 x1 = _mm_aesenc_si128(x1, sk[5]);
 512                 x0 = _mm_aesenc_si128(x0, sk[6]);
 513                 x1 = _mm_aesenc_si128(x1, sk[6]);
 514                 x0 = _mm_aesenc_si128(x0, sk[7]);
 515                 x1 = _mm_aesenc_si128(x1, sk[7]);
 516                 x0 = _mm_aesenc_si128(x0, sk[8]);
 517                 x1 = _mm_aesenc_si128(x1, sk[8]);
 518                 x0 = _mm_aesenc_si128(x0, sk[9]);
 519                 x1 = _mm_aesenc_si128(x1, sk[9]);
 520                 if (num_rounds == 10) {
 521                         x0 = _mm_aesenclast_si128(x0, sk[10]);
 522                         x1 = _mm_aesenclast_si128(x1, sk[10]);
 523                 } else if (num_rounds == 12) {
 524                         x0 = _mm_aesenc_si128(x0, sk[10]);
 525                         x1 = _mm_aesenc_si128(x1, sk[10]);
 526                         x0 = _mm_aesenc_si128(x0, sk[11]);
 527                         x1 = _mm_aesenc_si128(x1, sk[11]);
 528                         x0 = _mm_aesenclast_si128(x0, sk[12]);
 529                         x1 = _mm_aesenclast_si128(x1, sk[12]);
 530                 } else {
 531                         x0 = _mm_aesenc_si128(x0, sk[10]);
 532                         x1 = _mm_aesenc_si128(x1, sk[10]);
 533                         x0 = _mm_aesenc_si128(x0, sk[11]);
 534                         x1 = _mm_aesenc_si128(x1, sk[11]);
 535                         x0 = _mm_aesenc_si128(x0, sk[12]);
 536                         x1 = _mm_aesenc_si128(x1, sk[12]);
 537                         x0 = _mm_aesenc_si128(x0, sk[13]);
 538                         x1 = _mm_aesenc_si128(x1, sk[13]);
 539                         x0 = _mm_aesenclast_si128(x0, sk[14]);
 540                         x1 = _mm_aesenclast_si128(x1, sk[14]);
 541                 }
 542                 x0 = _mm_xor_si128(x0, dx);
 543                 cmx = x1;
 544                 _mm_storeu_si128((void *)buf, x0);
 545
 546                 buf += 16;
 547                 len -= 16;
 548
 549                 /*
 550                  * Increment the counter value.
 551                  */
 552                 ivx = _mm_add_epi64(ivx, one);
 553                 ivx = _mm_sub_epi64(ivx,
 554                         _mm_slli_si128(_mm_cmpeq_epi64(ivx, zero), 8));
 555         }
 556
 557         /*
 558          * Write back new counter value and CBC-MAC value.
 559          */
 560         _mm_storeu_si128(ctr, _mm_shuffle_epi8(ivx, erev));
 561         _mm_storeu_si128(cbcmac, cmx);
 562 }
 563
 564 BR_TARGETS_X86_DOWN
 565
 566 /* see bearssl_block.h */
 567 const br_block_ctrcbc_class br_aes_x86ni_ctrcbc_vtable = {
 568         sizeof(br_aes_x86ni_ctrcbc_keys),
 569         16,
 570         4,
 571         (void (*)(const br_block_ctrcbc_class **, const void *, size_t))
 572                 &br_aes_x86ni_ctrcbc_init,
 573         (void (*)(const br_block_ctrcbc_class *const *,
 574                 void *, void *, void *, size_t))
 575                 &br_aes_x86ni_ctrcbc_encrypt,
 576         (void (*)(const br_block_ctrcbc_class *const *,
 577                 void *, void *, void *, size_t))
 578                 &br_aes_x86ni_ctrcbc_decrypt,
 579         (void (*)(const br_block_ctrcbc_class *const *,
 580                 void *, void *, size_t))
 581                 &br_aes_x86ni_ctrcbc_ctr,
 582         (void (*)(const br_block_ctrcbc_class *const *,
 583                 void *, const void *, size_t))
 584                 &br_aes_x86ni_ctrcbc_mac
 585 };
 586
 587 #else
 588
 589 /* see bearssl_block.h */
 590 const br_block_ctrcbc_class *
 591 br_aes_x86ni_ctrcbc_get_vtable(void)
 592 {
 593         return NULL;
 594 }
 595
 596 #endif