1 /*******************************************************************************
2 * Copyright (c) 2013, Intel Corporation
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are
10 * * Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the
18 * * Neither the name of the Intel Corporation nor the names of its
19 * contributors may be used to endorse or promote products derived from
20 * this software without specific prior written permission.
23 * THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION ""AS IS"" AND ANY
24 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
26 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION OR
27 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
28 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
29 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
30 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
31 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
32 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
33 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 ********************************************************************************
36 * Intel SHA Extensions optimized implementation of a SHA-256 update function
38 * The function takes a pointer to the current hash values, a pointer to the
39 * input data, and a number of 64 byte blocks to process. Once all blocks have
40 * been processed, the digest pointer is updated with the resulting hash value.
41 * The function only processes complete blocks, there is no functionality to
42 * store partial blocks. All message padding and hash value initialization must
43 * be done outside the update function.
45 * The indented lines in the loop are instructions related to rounds processing.
46 * The non-indented lines are instructions related to the message schedule.
48 * Author: Sean Gulley <sean.m.gulley@intel.com>
51 ********************************************************************************
53 * Example complier command line:
54 * icc intel_sha_extensions_sha256_intrinsic.c
55 * gcc -msha -msse4 intel_sha_extensions_sha256_intrinsic.c
57 *******************************************************************************/
58 #include <sys/cdefs.h>
59 __FBSDID("$FreeBSD$");
61 #include <sys/types.h>
62 #include <crypto/aesni/aesni_os.h>
63 #include <crypto/aesni/sha_sse.h>
65 #include <immintrin.h>
67 void intel_sha256_step(uint32_t *digest, const char *data, uint32_t num_blks) {
68 __m128i state0, state1;
70 __m128i msgtmp0, msgtmp1, msgtmp2, msgtmp3;
73 __m128i abef_save, cdgh_save;
75 // Load initial hash values
76 // Need to reorder these appropriately
77 // DCBA, HGFE -> ABEF, CDGH
78 tmp = _mm_loadu_si128((__m128i*) digest);
79 state1 = _mm_loadu_si128((__m128i*) (digest+4));
81 tmp = _mm_shuffle_epi32(tmp, 0xB1); // CDAB
82 state1 = _mm_shuffle_epi32(state1, 0x1B); // EFGH
83 state0 = _mm_alignr_epi8(tmp, state1, 8); // ABEF
84 state1 = _mm_blend_epi16(state1, tmp, 0xF0); // CDGH
86 shuf_mask = _mm_set_epi64x(0x0c0d0e0f08090a0bull, 0x0405060700010203ull);
88 while (num_blks > 0) {
89 // Save hash values for addition after rounds
94 msg = _mm_loadu_si128((const __m128i*) data);
95 msgtmp0 = _mm_shuffle_epi8(msg, shuf_mask);
96 msg = _mm_add_epi32(msgtmp0,
97 _mm_set_epi64x(0xE9B5DBA5B5C0FBCFull, 0x71374491428A2F98ull));
98 state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
99 msg = _mm_shuffle_epi32(msg, 0x0E);
100 state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
103 msgtmp1 = _mm_loadu_si128((const __m128i*) (data+16));
104 msgtmp1 = _mm_shuffle_epi8(msgtmp1, shuf_mask);
105 msg = _mm_add_epi32(msgtmp1,
106 _mm_set_epi64x(0xAB1C5ED5923F82A4ull, 0x59F111F13956C25Bull));
107 state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
108 msg = _mm_shuffle_epi32(msg, 0x0E);
109 state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
110 msgtmp0 = _mm_sha256msg1_epu32(msgtmp0, msgtmp1);
113 msgtmp2 = _mm_loadu_si128((const __m128i*) (data+32));
114 msgtmp2 = _mm_shuffle_epi8(msgtmp2, shuf_mask);
115 msg = _mm_add_epi32(msgtmp2,
116 _mm_set_epi64x(0x550C7DC3243185BEull, 0x12835B01D807AA98ull));
117 state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
118 msg = _mm_shuffle_epi32(msg, 0x0E);
119 state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
120 msgtmp1 = _mm_sha256msg1_epu32(msgtmp1, msgtmp2);
123 msgtmp3 = _mm_loadu_si128((const __m128i*) (data+48));
124 msgtmp3 = _mm_shuffle_epi8(msgtmp3, shuf_mask);
125 msg = _mm_add_epi32(msgtmp3,
126 _mm_set_epi64x(0xC19BF1749BDC06A7ull, 0x80DEB1FE72BE5D74ull));
127 state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
128 tmp = _mm_alignr_epi8(msgtmp3, msgtmp2, 4);
129 msgtmp0 = _mm_add_epi32(msgtmp0, tmp);
130 msgtmp0 = _mm_sha256msg2_epu32(msgtmp0, msgtmp3);
131 msg = _mm_shuffle_epi32(msg, 0x0E);
132 state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
133 msgtmp2 = _mm_sha256msg1_epu32(msgtmp2, msgtmp3);
136 msg = _mm_add_epi32(msgtmp0,
137 _mm_set_epi64x(0x240CA1CC0FC19DC6ull, 0xEFBE4786E49B69C1ull));
138 state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
139 tmp = _mm_alignr_epi8(msgtmp0, msgtmp3, 4);
140 msgtmp1 = _mm_add_epi32(msgtmp1, tmp);
141 msgtmp1 = _mm_sha256msg2_epu32(msgtmp1, msgtmp0);
142 msg = _mm_shuffle_epi32(msg, 0x0E);
143 state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
144 msgtmp3 = _mm_sha256msg1_epu32(msgtmp3, msgtmp0);
147 msg = _mm_add_epi32(msgtmp1,
148 _mm_set_epi64x(0x76F988DA5CB0A9DCull, 0x4A7484AA2DE92C6Full));
149 state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
150 tmp = _mm_alignr_epi8(msgtmp1, msgtmp0, 4);
151 msgtmp2 = _mm_add_epi32(msgtmp2, tmp);
152 msgtmp2 = _mm_sha256msg2_epu32(msgtmp2, msgtmp1);
153 msg = _mm_shuffle_epi32(msg, 0x0E);
154 state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
155 msgtmp0 = _mm_sha256msg1_epu32(msgtmp0, msgtmp1);
158 msg = _mm_add_epi32(msgtmp2,
159 _mm_set_epi64x(0xBF597FC7B00327C8ull, 0xA831C66D983E5152ull));
160 state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
161 tmp = _mm_alignr_epi8(msgtmp2, msgtmp1, 4);
162 msgtmp3 = _mm_add_epi32(msgtmp3, tmp);
163 msgtmp3 = _mm_sha256msg2_epu32(msgtmp3, msgtmp2);
164 msg = _mm_shuffle_epi32(msg, 0x0E);
165 state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
166 msgtmp1 = _mm_sha256msg1_epu32(msgtmp1, msgtmp2);
169 msg = _mm_add_epi32(msgtmp3,
170 _mm_set_epi64x(0x1429296706CA6351ull, 0xD5A79147C6E00BF3ull));
171 state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
172 tmp = _mm_alignr_epi8(msgtmp3, msgtmp2, 4);
173 msgtmp0 = _mm_add_epi32(msgtmp0, tmp);
174 msgtmp0 = _mm_sha256msg2_epu32(msgtmp0, msgtmp3);
175 msg = _mm_shuffle_epi32(msg, 0x0E);
176 state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
177 msgtmp2 = _mm_sha256msg1_epu32(msgtmp2, msgtmp3);
180 msg = _mm_add_epi32(msgtmp0,
181 _mm_set_epi64x(0x53380D134D2C6DFCull, 0x2E1B213827B70A85ull));
182 state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
183 tmp = _mm_alignr_epi8(msgtmp0, msgtmp3, 4);
184 msgtmp1 = _mm_add_epi32(msgtmp1, tmp);
185 msgtmp1 = _mm_sha256msg2_epu32(msgtmp1, msgtmp0);
186 msg = _mm_shuffle_epi32(msg, 0x0E);
187 state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
188 msgtmp3 = _mm_sha256msg1_epu32(msgtmp3, msgtmp0);
191 msg = _mm_add_epi32(msgtmp1,
192 _mm_set_epi64x(0x92722C8581C2C92Eull, 0x766A0ABB650A7354ull));
193 state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
194 tmp = _mm_alignr_epi8(msgtmp1, msgtmp0, 4);
195 msgtmp2 = _mm_add_epi32(msgtmp2, tmp);
196 msgtmp2 = _mm_sha256msg2_epu32(msgtmp2, msgtmp1);
197 msg = _mm_shuffle_epi32(msg, 0x0E);
198 state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
199 msgtmp0 = _mm_sha256msg1_epu32(msgtmp0, msgtmp1);
202 msg = _mm_add_epi32(msgtmp2,
203 _mm_set_epi64x(0xC76C51A3C24B8B70ull, 0xA81A664BA2BFE8A1ull));
204 state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
205 tmp = _mm_alignr_epi8(msgtmp2, msgtmp1, 4);
206 msgtmp3 = _mm_add_epi32(msgtmp3, tmp);
207 msgtmp3 = _mm_sha256msg2_epu32(msgtmp3, msgtmp2);
208 msg = _mm_shuffle_epi32(msg, 0x0E);
209 state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
210 msgtmp1 = _mm_sha256msg1_epu32(msgtmp1, msgtmp2);
213 msg = _mm_add_epi32(msgtmp3,
214 _mm_set_epi64x(0x106AA070F40E3585ull, 0xD6990624D192E819ull));
215 state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
216 tmp = _mm_alignr_epi8(msgtmp3, msgtmp2, 4);
217 msgtmp0 = _mm_add_epi32(msgtmp0, tmp);
218 msgtmp0 = _mm_sha256msg2_epu32(msgtmp0, msgtmp3);
219 msg = _mm_shuffle_epi32(msg, 0x0E);
220 state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
221 msgtmp2 = _mm_sha256msg1_epu32(msgtmp2, msgtmp3);
224 msg = _mm_add_epi32(msgtmp0,
225 _mm_set_epi64x(0x34B0BCB52748774Cull, 0x1E376C0819A4C116ull));
226 state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
227 tmp = _mm_alignr_epi8(msgtmp0, msgtmp3, 4);
228 msgtmp1 = _mm_add_epi32(msgtmp1, tmp);
229 msgtmp1 = _mm_sha256msg2_epu32(msgtmp1, msgtmp0);
230 msg = _mm_shuffle_epi32(msg, 0x0E);
231 state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
232 msgtmp3 = _mm_sha256msg1_epu32(msgtmp3, msgtmp0);
235 msg = _mm_add_epi32(msgtmp1,
236 _mm_set_epi64x(0x682E6FF35B9CCA4Full, 0x4ED8AA4A391C0CB3ull));
237 state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
238 tmp = _mm_alignr_epi8(msgtmp1, msgtmp0, 4);
239 msgtmp2 = _mm_add_epi32(msgtmp2, tmp);
240 msgtmp2 = _mm_sha256msg2_epu32(msgtmp2, msgtmp1);
241 msg = _mm_shuffle_epi32(msg, 0x0E);
242 state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
245 msg = _mm_add_epi32(msgtmp2,
246 _mm_set_epi64x(0x8CC7020884C87814ull, 0x78A5636F748F82EEull));
247 state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
248 tmp = _mm_alignr_epi8(msgtmp2, msgtmp1, 4);
249 msgtmp3 = _mm_add_epi32(msgtmp3, tmp);
250 msgtmp3 = _mm_sha256msg2_epu32(msgtmp3, msgtmp2);
251 msg = _mm_shuffle_epi32(msg, 0x0E);
252 state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
255 msg = _mm_add_epi32(msgtmp3,
256 _mm_set_epi64x(0xC67178F2BEF9A3F7ull, 0xA4506CEB90BEFFFAull));
257 state1 = _mm_sha256rnds2_epu32(state1, state0, msg);
258 msg = _mm_shuffle_epi32(msg, 0x0E);
259 state0 = _mm_sha256rnds2_epu32(state0, state1, msg);
261 // Add current hash values with previously saved
262 state0 = _mm_add_epi32(state0, abef_save);
263 state1 = _mm_add_epi32(state1, cdgh_save);
269 // Write hash values back in the correct order
270 tmp = _mm_shuffle_epi32(state0, 0x1B); // FEBA
271 state1 = _mm_shuffle_epi32(state1, 0xB1); // DCHG
272 state0 = _mm_blend_epi16(tmp, state1, 0xF0); // DCBA
273 state1 = _mm_alignr_epi8(state1, tmp, 8); // ABEF
275 _mm_store_si128((__m128i*) digest, state0);
276 _mm_store_si128((__m128i*) (digest+4), state1);