sys/crypto/aesni/aeskeys_amd64.S

   1 /*-
   2 * The white paper of AES-NI instructions can be downloaded from:
   3  *   http://softwarecommunity.intel.com/isn/downloads/intelavx/AES-Instructions-Set_WP.pdf
   4  *
   5  * Copyright (C) 2008-2010, Intel Corporation
   6  *    Author: Huang Ying <ying.huang@intel.com>
   7  *            Vinodh Gopal <vinodh.gopal@intel.com>
   8  *            Kahraman Akdemir
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following
  12  * conditions are met:
  13  *
  14  * - Redistributions of source code must retain the above copyright
  15  *   notice, this list of conditions and the following disclaimer.
  16  *
  17  * - Redistributions in binary form must reproduce the above copyright
  18  *   notice, this list of conditions and the following disclaimer in the
  19  *   documentation and/or other materials provided with the
  20  *   distribution.
  21  *
  22  * - Neither the name of Intel Corporation nor the names of its
  23  *   contributors may be used to endorse or promote products
  24  *   derived from this software without specific prior written
  25  *   permission.
  26  *
  27  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  28  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  29  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  30  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  31  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  32  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  33  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  34  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
  35  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  36  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  37  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  38  */
  39
  40 #include <machine/asmacros.h>
  41
  42         .text
  43
  44 ENTRY(_key_expansion_128)
  45 _key_expansion_256a:
  46         .cfi_startproc
  47         pshufd  $0b11111111,%xmm1,%xmm1
  48         shufps  $0b00010000,%xmm0,%xmm4
  49         pxor    %xmm4,%xmm0
  50         shufps  $0b10001100,%xmm0,%xmm4
  51         pxor    %xmm4,%xmm0
  52         pxor    %xmm1,%xmm0
  53         movaps  %xmm0,(%rsi)
  54         addq    $0x10,%rsi
  55         retq
  56         .cfi_endproc
  57 END(_key_expansion_128)
  58
  59 ENTRY(_key_expansion_192a)
  60         .cfi_startproc
  61         pshufd  $0b01010101,%xmm1,%xmm1
  62         shufps  $0b00010000,%xmm0,%xmm4
  63         pxor    %xmm4,%xmm0
  64         shufps  $0b10001100,%xmm0,%xmm4
  65         pxor    %xmm4,%xmm0
  66         pxor    %xmm1,%xmm0
  67         movaps  %xmm2,%xmm5
  68         movaps  %xmm2,%xmm6
  69         pslldq  $4,%xmm5
  70         pshufd  $0b11111111,%xmm0,%xmm3
  71         pxor    %xmm3,%xmm2
  72         pxor    %xmm5,%xmm2
  73         movaps  %xmm0,%xmm1
  74         shufps  $0b01000100,%xmm0,%xmm6
  75         movaps  %xmm6,(%rsi)
  76         shufps  $0b01001110,%xmm2,%xmm1
  77         movaps  %xmm1,0x10(%rsi)
  78         addq    $0x20,%rsi
  79         retq
  80         .cfi_endproc
  81 END(_key_expansion_192a)
  82
  83 ENTRY(_key_expansion_192b)
  84         .cfi_startproc
  85         pshufd  $0b01010101,%xmm1,%xmm1
  86         shufps  $0b00010000,%xmm0,%xmm4
  87         pxor    %xmm4,%xmm0
  88         shufps  $0b10001100,%xmm0,%xmm4
  89         pxor    %xmm4,%xmm0
  90         pxor    %xmm1,%xmm0
  91         movaps  %xmm2,%xmm5
  92         pslldq  $4,%xmm5
  93         pshufd  $0b11111111,%xmm0,%xmm3
  94         pxor    %xmm3,%xmm2
  95         pxor    %xmm5,%xmm2
  96         movaps  %xmm0,(%rsi)
  97         addq    $0x10,%rsi
  98         retq
  99         .cfi_endproc
 100 END(_key_expansion_192b)
 101
 102 ENTRY(_key_expansion_256b)
 103         .cfi_startproc
 104         pshufd  $0b10101010,%xmm1,%xmm1
 105         shufps  $0b00010000,%xmm2,%xmm4
 106         pxor    %xmm4,%xmm2
 107         shufps  $0b10001100,%xmm2,%xmm4
 108         pxor    %xmm4,%xmm2
 109         pxor    %xmm1,%xmm2
 110         movaps  %xmm2,(%rsi)
 111         addq    $0x10,%rsi
 112         retq
 113         .cfi_endproc
 114 END(_key_expansion_256b)
 115
 116 ENTRY(aesni_set_enckey)
 117         .cfi_startproc
 118         movups  (%rdi),%xmm0            # user key (first 16 bytes)
 119         movaps  %xmm0,(%rsi)
 120         addq    $0x10,%rsi              # key addr
 121         pxor    %xmm4,%xmm4             # xmm4 is assumed 0 in _key_expansion_x
 122         cmpl    $12,%edx
 123         jb      .Lenc_key128
 124         je      .Lenc_key192
 125         movups  0x10(%rdi),%xmm2        # other user key
 126         movaps  %xmm2,(%rsi)
 127         addq    $0x10,%rsi
 128         aeskeygenassist $0x1,%xmm2,%xmm1        # round 1
 129         call    _key_expansion_256a
 130         aeskeygenassist $0x1,%xmm0,%xmm1
 131         call    _key_expansion_256b
 132         aeskeygenassist $0x2,%xmm2,%xmm1        # round 2
 133         call    _key_expansion_256a
 134         aeskeygenassist $0x2,%xmm0,%xmm1
 135         call    _key_expansion_256b
 136         aeskeygenassist $0x4,%xmm2,%xmm1        # round 3
 137         call    _key_expansion_256a
 138         aeskeygenassist $0x4,%xmm0,%xmm1
 139         call    _key_expansion_256b
 140         aeskeygenassist $0x8,%xmm2,%xmm1        # round 4
 141         call    _key_expansion_256a
 142         aeskeygenassist $0x8,%xmm0,%xmm1
 143         call    _key_expansion_256b
 144         aeskeygenassist $0x10,%xmm2,%xmm1       # round 5
 145         call    _key_expansion_256a
 146         aeskeygenassist $0x10,%xmm0,%xmm1
 147         call    _key_expansion_256b
 148         aeskeygenassist $0x20,%xmm2,%xmm1       # round 6
 149         call    _key_expansion_256a
 150         aeskeygenassist $0x20,%xmm0,%xmm1
 151         call    _key_expansion_256b
 152         aeskeygenassist $0x40,%xmm2,%xmm1       # round 7
 153         call    _key_expansion_256a
 154         retq
 155 .Lenc_key192:
 156         movq    0x10(%rdi),%xmm2                # other user key
 157         aeskeygenassist $0x1,%xmm2,%xmm1        # round 1
 158         call    _key_expansion_192a
 159         aeskeygenassist $0x2,%xmm2,%xmm1        # round 2
 160         call    _key_expansion_192b
 161         aeskeygenassist $0x4,%xmm2,%xmm1        # round 3
 162         call    _key_expansion_192a
 163         aeskeygenassist $0x8,%xmm2,%xmm1        # round 4
 164         call    _key_expansion_192b
 165         aeskeygenassist $0x10,%xmm2,%xmm1       # round 5
 166         call    _key_expansion_192a
 167         aeskeygenassist $0x20,%xmm2,%xmm1       # round 6
 168         call    _key_expansion_192b
 169         aeskeygenassist $0x40,%xmm2,%xmm1       # round 7
 170         call    _key_expansion_192a
 171         aeskeygenassist $0x80,%xmm2,%xmm1       # round 8
 172         call    _key_expansion_192b
 173         retq
 174 .Lenc_key128:
 175         aeskeygenassist $0x1,%xmm0,%xmm1        # round 1
 176         call    _key_expansion_128
 177         aeskeygenassist $0x2,%xmm0,%xmm1        # round 2
 178         call    _key_expansion_128
 179         aeskeygenassist $0x4,%xmm0,%xmm1        # round 3
 180         call    _key_expansion_128
 181         aeskeygenassist $0x8,%xmm0,%xmm1        # round 4
 182         call    _key_expansion_128
 183         aeskeygenassist $0x10,%xmm0,%xmm1       # round 5
 184         call    _key_expansion_128
 185         aeskeygenassist $0x20,%xmm0,%xmm1       # round 6
 186         call    _key_expansion_128
 187         aeskeygenassist $0x40,%xmm0,%xmm1       # round 7
 188         call    _key_expansion_128
 189         aeskeygenassist $0x80,%xmm0,%xmm1       # round 8
 190         call    _key_expansion_128
 191         aeskeygenassist $0x1b,%xmm0,%xmm1       # round 9
 192         call    _key_expansion_128
 193         aeskeygenassist $0x36,%xmm0,%xmm1       # round 10
 194         call    _key_expansion_128
 195         retq
 196         .cfi_endproc
 197 END(aesni_set_enckey)
 198
 199 ENTRY(aesni_set_deckey)
 200         .cfi_startproc
 201         movslq  %edx,%rax
 202         shlq    $4,%rax
 203         addq    %rax,%rdi
 204         movdqa  (%rdi),%xmm0
 205         movdqa  %xmm0,(%rsi)
 206         decl    %edx
 207 1:
 208         addq    $0x10,%rsi
 209         subq    $0x10,%rdi
 210         aesimc  (%rdi),%xmm1
 211         movdqa  %xmm1,(%rsi)
 212         decl    %edx
 213         jne     1b
 214
 215         addq    $0x10,%rsi
 216         subq    $0x10,%rdi
 217         movdqa  (%rdi),%xmm0
 218         movdqa  %xmm0,(%rsi)
 219         retq
 220         .cfi_endproc
 221 END(aesni_set_deckey)
 222
 223         .ident  "$FreeBSD$"