1 /*******************************************************************************
2 * Copyright (c) 2013, Intel Corporation
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are
10 * * Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the
18 * * Neither the name of the Intel Corporation nor the names of its
19 * contributors may be used to endorse or promote products derived from
20 * this software without specific prior written permission.
23 * THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION ""AS IS"" AND ANY
24 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
26 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION OR
27 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
28 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
29 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
30 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
31 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
32 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
33 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 ********************************************************************************
36 * Intel SHA Extensions optimized implementation of a SHA-1 update function
38 * The function takes a pointer to the current hash values, a pointer to the
39 * input data, and a number of 64 byte blocks to process. Once all blocks have
40 * been processed, the digest pointer is updated with the resulting hash value.
41 * The function only processes complete blocks, there is no functionality to
42 * store partial blocks. All message padding and hash value initialization must
43 * be done outside the update function.
45 * The indented lines in the loop are instructions related to rounds processing.
46 * The non-indented lines are instructions related to the message schedule.
48 * Author: Sean Gulley <sean.m.gulley@intel.com>
51 ********************************************************************************
53 * Example complier command line:
54 * icc intel_sha_extensions_sha1_intrinsic.c
55 * gcc -msha -msse4 intel_sha_extensions_sha1_intrinsic.c
57 *******************************************************************************/
58 #include <sys/cdefs.h>
59 __FBSDID("$FreeBSD$");
61 #include <sys/types.h>
62 #include <crypto/aesni/aesni_os.h>
63 #include <crypto/aesni/sha_sse.h>
65 #include <immintrin.h>
67 void intel_sha1_step(uint32_t *digest, const char *data, uint32_t num_blks) {
69 __m128i abcd_save, e_save;
70 __m128i msg0, msg1, msg2, msg3;
71 __m128i shuf_mask, e_mask;
74 e_mask = _mm_set_epi64x(0xFFFFFFFF00000000ull, 0x0000000000000000ull);
77 e0 = _mm_set_epi64x(0, 0);
79 shuf_mask = _mm_set_epi64x(0x0001020304050607ull, 0x08090a0b0c0d0e0full);
81 // Load initial hash values
82 abcd = _mm_loadu_si128((__m128i*) digest);
83 e0 = _mm_insert_epi32(e0, *(digest+4), 3);
84 abcd = _mm_shuffle_epi32(abcd, 0x1B);
86 e0 = _mm_and_si128(e0, e_mask);
89 while (num_blks > 0) {
90 // Save hash values for addition after rounds
95 msg0 = _mm_loadu_si128((const __m128i*) data);
96 msg0 = _mm_shuffle_epi8(msg0, shuf_mask);
97 e0 = _mm_add_epi32(e0, msg0);
99 abcd = _mm_sha1rnds4_epu32(abcd, e0, 0);
102 msg1 = _mm_loadu_si128((const __m128i*) (data+16));
103 msg1 = _mm_shuffle_epi8(msg1, shuf_mask);
104 e1 = _mm_sha1nexte_epu32(e1, msg1);
106 abcd = _mm_sha1rnds4_epu32(abcd, e1, 0);
107 msg0 = _mm_sha1msg1_epu32(msg0, msg1);
110 msg2 = _mm_loadu_si128((const __m128i*) (data+32));
111 msg2 = _mm_shuffle_epi8(msg2, shuf_mask);
112 e0 = _mm_sha1nexte_epu32(e0, msg2);
114 abcd = _mm_sha1rnds4_epu32(abcd, e0, 0);
115 msg1 = _mm_sha1msg1_epu32(msg1, msg2);
116 msg0 = _mm_xor_si128(msg0, msg2);
119 msg3 = _mm_loadu_si128((const __m128i*) (data+48));
120 msg3 = _mm_shuffle_epi8(msg3, shuf_mask);
121 e1 = _mm_sha1nexte_epu32(e1, msg3);
123 msg0 = _mm_sha1msg2_epu32(msg0, msg3);
124 abcd = _mm_sha1rnds4_epu32(abcd, e1, 0);
125 msg2 = _mm_sha1msg1_epu32(msg2, msg3);
126 msg1 = _mm_xor_si128(msg1, msg3);
129 e0 = _mm_sha1nexte_epu32(e0, msg0);
131 msg1 = _mm_sha1msg2_epu32(msg1, msg0);
132 abcd = _mm_sha1rnds4_epu32(abcd, e0, 0);
133 msg3 = _mm_sha1msg1_epu32(msg3, msg0);
134 msg2 = _mm_xor_si128(msg2, msg0);
137 e1 = _mm_sha1nexte_epu32(e1, msg1);
139 msg2 = _mm_sha1msg2_epu32(msg2, msg1);
140 abcd = _mm_sha1rnds4_epu32(abcd, e1, 1);
141 msg0 = _mm_sha1msg1_epu32(msg0, msg1);
142 msg3 = _mm_xor_si128(msg3, msg1);
145 e0 = _mm_sha1nexte_epu32(e0, msg2);
147 msg3 = _mm_sha1msg2_epu32(msg3, msg2);
148 abcd = _mm_sha1rnds4_epu32(abcd, e0, 1);
149 msg1 = _mm_sha1msg1_epu32(msg1, msg2);
150 msg0 = _mm_xor_si128(msg0, msg2);
153 e1 = _mm_sha1nexte_epu32(e1, msg3);
155 msg0 = _mm_sha1msg2_epu32(msg0, msg3);
156 abcd = _mm_sha1rnds4_epu32(abcd, e1, 1);
157 msg2 = _mm_sha1msg1_epu32(msg2, msg3);
158 msg1 = _mm_xor_si128(msg1, msg3);
161 e0 = _mm_sha1nexte_epu32(e0, msg0);
163 msg1 = _mm_sha1msg2_epu32(msg1, msg0);
164 abcd = _mm_sha1rnds4_epu32(abcd, e0, 1);
165 msg3 = _mm_sha1msg1_epu32(msg3, msg0);
166 msg2 = _mm_xor_si128(msg2, msg0);
169 e1 = _mm_sha1nexte_epu32(e1, msg1);
171 msg2 = _mm_sha1msg2_epu32(msg2, msg1);
172 abcd = _mm_sha1rnds4_epu32(abcd, e1, 1);
173 msg0 = _mm_sha1msg1_epu32(msg0, msg1);
174 msg3 = _mm_xor_si128(msg3, msg1);
177 e0 = _mm_sha1nexte_epu32(e0, msg2);
179 msg3 = _mm_sha1msg2_epu32(msg3, msg2);
180 abcd = _mm_sha1rnds4_epu32(abcd, e0, 2);
181 msg1 = _mm_sha1msg1_epu32(msg1, msg2);
182 msg0 = _mm_xor_si128(msg0, msg2);
185 e1 = _mm_sha1nexte_epu32(e1, msg3);
187 msg0 = _mm_sha1msg2_epu32(msg0, msg3);
188 abcd = _mm_sha1rnds4_epu32(abcd, e1, 2);
189 msg2 = _mm_sha1msg1_epu32(msg2, msg3);
190 msg1 = _mm_xor_si128(msg1, msg3);
193 e0 = _mm_sha1nexte_epu32(e0, msg0);
195 msg1 = _mm_sha1msg2_epu32(msg1, msg0);
196 abcd = _mm_sha1rnds4_epu32(abcd, e0, 2);
197 msg3 = _mm_sha1msg1_epu32(msg3, msg0);
198 msg2 = _mm_xor_si128(msg2, msg0);
201 e1 = _mm_sha1nexte_epu32(e1, msg1);
203 msg2 = _mm_sha1msg2_epu32(msg2, msg1);
204 abcd = _mm_sha1rnds4_epu32(abcd, e1, 2);
205 msg0 = _mm_sha1msg1_epu32(msg0, msg1);
206 msg3 = _mm_xor_si128(msg3, msg1);
209 e0 = _mm_sha1nexte_epu32(e0, msg2);
211 msg3 = _mm_sha1msg2_epu32(msg3, msg2);
212 abcd = _mm_sha1rnds4_epu32(abcd, e0, 2);
213 msg1 = _mm_sha1msg1_epu32(msg1, msg2);
214 msg0 = _mm_xor_si128(msg0, msg2);
217 e1 = _mm_sha1nexte_epu32(e1, msg3);
219 msg0 = _mm_sha1msg2_epu32(msg0, msg3);
220 abcd = _mm_sha1rnds4_epu32(abcd, e1, 3);
221 msg2 = _mm_sha1msg1_epu32(msg2, msg3);
222 msg1 = _mm_xor_si128(msg1, msg3);
225 e0 = _mm_sha1nexte_epu32(e0, msg0);
227 msg1 = _mm_sha1msg2_epu32(msg1, msg0);
228 abcd = _mm_sha1rnds4_epu32(abcd, e0, 3);
229 msg3 = _mm_sha1msg1_epu32(msg3, msg0);
230 msg2 = _mm_xor_si128(msg2, msg0);
233 e1 = _mm_sha1nexte_epu32(e1, msg1);
235 msg2 = _mm_sha1msg2_epu32(msg2, msg1);
236 abcd = _mm_sha1rnds4_epu32(abcd, e1, 3);
237 msg3 = _mm_xor_si128(msg3, msg1);
240 e0 = _mm_sha1nexte_epu32(e0, msg2);
242 msg3 = _mm_sha1msg2_epu32(msg3, msg2);
243 abcd = _mm_sha1rnds4_epu32(abcd, e0, 3);
246 e1 = _mm_sha1nexte_epu32(e1, msg3);
248 abcd = _mm_sha1rnds4_epu32(abcd, e1, 3);
250 // Add current hash values with previously saved
251 e0 = _mm_sha1nexte_epu32(e0, e_save);
252 abcd = _mm_add_epi32(abcd, abcd_save);
258 abcd = _mm_shuffle_epi32(abcd, 0x1B);
259 _mm_store_si128((__m128i*) digest, abcd);
260 *(digest+4) = _mm_extract_epi32(e0, 3);