5 * Copyright (c) 2009 David McCullough <david.mccullough@securecomputing.com>
7 * Copyright (c) 2003-2007 Cavium Networks (support@cavium.com). All rights
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions are met:
12 * 1. Redistributions of source code must retain the above copyright notice,
13 * this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright notice,
15 * this list of conditions and the following disclaimer in the documentation
16 * and/or other materials provided with the distribution.
17 * 3. All advertising materials mentioning features or use of this software
18 * must display the following acknowledgement:
19 * This product includes software developed by Cavium Networks
20 * 4. Cavium Networks' name may not be used to endorse or promote products
21 * derived from this software without specific prior written permission.
23 * This Software, including technical data, may be subject to U.S. export
24 * control laws, including the U.S. Export Administration Act and its
25 * associated regulations, and may be subject to export or import regulations
26 * in other countries. You warrant that You will comply strictly in all
27 * respects with all such regulations and acknowledge that you have the
28 * responsibility to obtain licenses to export, re-export or import the
31 * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" AND
32 * WITH ALL FAULTS AND CAVIUM MAKES NO PROMISES, REPRESENTATIONS OR WARRANTIES,
33 * EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO THE
34 * SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR
35 * DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
36 * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
37 * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
38 * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
39 * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR
40 * PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
42 /****************************************************************************/
44 #include <sys/cdefs.h>
45 __FBSDID("$FreeBSD$");
47 #include <sys/param.h>
48 #include <sys/systm.h>
49 #include <sys/kernel.h>
50 #include <sys/module.h>
51 #include <sys/malloc.h>
54 #include <opencrypto/cryptodev.h>
56 #include <contrib/octeon-sdk/cvmx.h>
58 #include <mips/cavium/cryptocteon/cryptocteonvar.h>
60 /****************************************************************************/
62 #define IOV_INIT(iov, ptr, idx, len) \
65 (ptr) = (iov)[(idx)].iov_base; \
66 (len) = (iov)[(idx)].iov_len; \
71 * It would be better if this were an IOV_READ/IOV_WRITE macro instead so
72 * that we could detect overflow before it happens rather than right after,
73 * which is especially bad since there is usually no IOV_CONSUME after the
74 * final read or write.
76 #define IOV_CONSUME(iov, ptr, idx, len) \
78 if ((len) > sizeof *(ptr)) { \
79 (len) -= sizeof *(ptr); \
82 if ((len) != sizeof *(ptr)) \
83 panic("%s: went past end of iovec.", __func__); \
85 (ptr) = (iov)[(idx)].iov_base; \
86 (len) = (iov)[(idx)].iov_len; \
90 #define ESP_HEADER_LENGTH 8
91 #define DES_CBC_IV_LENGTH 8
92 #define AES_CBC_IV_LENGTH 16
93 #define ESP_HMAC_LEN 12
95 #define ESP_HEADER_LENGTH 8
96 #define DES_CBC_IV_LENGTH 8
98 /****************************************************************************/
100 #define CVM_LOAD_SHA_UNIT(dat, next) { \
103 CVMX_MT_HSH_DAT (dat, 0); \
104 } else if (next == 1) { \
106 CVMX_MT_HSH_DAT (dat, 1); \
107 } else if (next == 2) { \
109 CVMX_MT_HSH_DAT (dat, 2); \
110 } else if (next == 3) { \
112 CVMX_MT_HSH_DAT (dat, 3); \
113 } else if (next == 4) { \
115 CVMX_MT_HSH_DAT (dat, 4); \
116 } else if (next == 5) { \
118 CVMX_MT_HSH_DAT (dat, 5); \
119 } else if (next == 6) { \
121 CVMX_MT_HSH_DAT (dat, 6); \
123 CVMX_MT_HSH_STARTSHA (dat); \
128 #define CVM_LOAD2_SHA_UNIT(dat1, dat2, next) { \
130 CVMX_MT_HSH_DAT (dat1, 0); \
131 CVMX_MT_HSH_DAT (dat2, 1); \
133 } else if (next == 1) { \
134 CVMX_MT_HSH_DAT (dat1, 1); \
135 CVMX_MT_HSH_DAT (dat2, 2); \
137 } else if (next == 2) { \
138 CVMX_MT_HSH_DAT (dat1, 2); \
139 CVMX_MT_HSH_DAT (dat2, 3); \
141 } else if (next == 3) { \
142 CVMX_MT_HSH_DAT (dat1, 3); \
143 CVMX_MT_HSH_DAT (dat2, 4); \
145 } else if (next == 4) { \
146 CVMX_MT_HSH_DAT (dat1, 4); \
147 CVMX_MT_HSH_DAT (dat2, 5); \
149 } else if (next == 5) { \
150 CVMX_MT_HSH_DAT (dat1, 5); \
151 CVMX_MT_HSH_DAT (dat2, 6); \
153 } else if (next == 6) { \
154 CVMX_MT_HSH_DAT (dat1, 6); \
155 CVMX_MT_HSH_STARTSHA (dat2); \
158 CVMX_MT_HSH_STARTSHA (dat1); \
159 CVMX_MT_HSH_DAT (dat2, 0); \
164 /****************************************************************************/
166 #define CVM_LOAD_MD5_UNIT(dat, next) { \
169 CVMX_MT_HSH_DAT (dat, 0); \
170 } else if (next == 1) { \
172 CVMX_MT_HSH_DAT (dat, 1); \
173 } else if (next == 2) { \
175 CVMX_MT_HSH_DAT (dat, 2); \
176 } else if (next == 3) { \
178 CVMX_MT_HSH_DAT (dat, 3); \
179 } else if (next == 4) { \
181 CVMX_MT_HSH_DAT (dat, 4); \
182 } else if (next == 5) { \
184 CVMX_MT_HSH_DAT (dat, 5); \
185 } else if (next == 6) { \
187 CVMX_MT_HSH_DAT (dat, 6); \
189 CVMX_MT_HSH_STARTMD5 (dat); \
194 #define CVM_LOAD2_MD5_UNIT(dat1, dat2, next) { \
196 CVMX_MT_HSH_DAT (dat1, 0); \
197 CVMX_MT_HSH_DAT (dat2, 1); \
199 } else if (next == 1) { \
200 CVMX_MT_HSH_DAT (dat1, 1); \
201 CVMX_MT_HSH_DAT (dat2, 2); \
203 } else if (next == 2) { \
204 CVMX_MT_HSH_DAT (dat1, 2); \
205 CVMX_MT_HSH_DAT (dat2, 3); \
207 } else if (next == 3) { \
208 CVMX_MT_HSH_DAT (dat1, 3); \
209 CVMX_MT_HSH_DAT (dat2, 4); \
211 } else if (next == 4) { \
212 CVMX_MT_HSH_DAT (dat1, 4); \
213 CVMX_MT_HSH_DAT (dat2, 5); \
215 } else if (next == 5) { \
216 CVMX_MT_HSH_DAT (dat1, 5); \
217 CVMX_MT_HSH_DAT (dat2, 6); \
219 } else if (next == 6) { \
220 CVMX_MT_HSH_DAT (dat1, 6); \
221 CVMX_MT_HSH_STARTMD5 (dat2); \
224 CVMX_MT_HSH_STARTMD5 (dat1); \
225 CVMX_MT_HSH_DAT (dat2, 0); \
230 /****************************************************************************/
233 octo_calc_hash(uint8_t auth, unsigned char *key, uint64_t *inner, uint64_t *outer)
235 uint8_t hash_key[64];
237 register uint64_t xor1 = 0x3636363636363636ULL;
238 register uint64_t xor2 = 0x5c5c5c5c5c5c5c5cULL;
240 dprintf("%s()\n", __func__);
242 memset(hash_key, 0, sizeof(hash_key));
243 memcpy(hash_key, (uint8_t *) key, (auth ? 20 : 16));
244 key1 = (uint64_t *) hash_key;
246 CVMX_MT_HSH_IV(0x67452301EFCDAB89ULL, 0);
247 CVMX_MT_HSH_IV(0x98BADCFE10325476ULL, 1);
248 CVMX_MT_HSH_IV(0xC3D2E1F000000000ULL, 2);
250 CVMX_MT_HSH_IV(0x0123456789ABCDEFULL, 0);
251 CVMX_MT_HSH_IV(0xFEDCBA9876543210ULL, 1);
254 CVMX_MT_HSH_DAT((*key1 ^ xor1), 0);
256 CVMX_MT_HSH_DAT((*key1 ^ xor1), 1);
258 CVMX_MT_HSH_DAT((*key1 ^ xor1), 2);
260 CVMX_MT_HSH_DAT((*key1 ^ xor1), 3);
262 CVMX_MT_HSH_DAT((*key1 ^ xor1), 4);
264 CVMX_MT_HSH_DAT((*key1 ^ xor1), 5);
266 CVMX_MT_HSH_DAT((*key1 ^ xor1), 6);
269 CVMX_MT_HSH_STARTSHA((*key1 ^ xor1));
271 CVMX_MT_HSH_STARTMD5((*key1 ^ xor1));
273 CVMX_MF_HSH_IV(inner[0], 0);
274 CVMX_MF_HSH_IV(inner[1], 1);
277 CVMX_MF_HSH_IV(((uint64_t *) inner)[2], 2);
280 memset(hash_key, 0, sizeof(hash_key));
281 memcpy(hash_key, (uint8_t *) key, (auth ? 20 : 16));
282 key1 = (uint64_t *) hash_key;
284 CVMX_MT_HSH_IV(0x67452301EFCDAB89ULL, 0);
285 CVMX_MT_HSH_IV(0x98BADCFE10325476ULL, 1);
286 CVMX_MT_HSH_IV(0xC3D2E1F000000000ULL, 2);
288 CVMX_MT_HSH_IV(0x0123456789ABCDEFULL, 0);
289 CVMX_MT_HSH_IV(0xFEDCBA9876543210ULL, 1);
292 CVMX_MT_HSH_DAT((*key1 ^ xor2), 0);
294 CVMX_MT_HSH_DAT((*key1 ^ xor2), 1);
296 CVMX_MT_HSH_DAT((*key1 ^ xor2), 2);
298 CVMX_MT_HSH_DAT((*key1 ^ xor2), 3);
300 CVMX_MT_HSH_DAT((*key1 ^ xor2), 4);
302 CVMX_MT_HSH_DAT((*key1 ^ xor2), 5);
304 CVMX_MT_HSH_DAT((*key1 ^ xor2), 6);
307 CVMX_MT_HSH_STARTSHA((*key1 ^ xor2));
309 CVMX_MT_HSH_STARTMD5((*key1 ^ xor2));
311 CVMX_MF_HSH_IV(outer[0], 0);
312 CVMX_MF_HSH_IV(outer[1], 1);
315 CVMX_MF_HSH_IV(outer[2], 2);
320 /****************************************************************************/
324 octo_des_cbc_encrypt(
325 struct octo_sess *od,
326 struct iovec *iov, size_t iovcnt, size_t iovlen,
327 int auth_off, int auth_len,
328 int crypt_off, int crypt_len,
329 int icv_off, uint8_t *ivp)
334 dprintf("%s()\n", __func__);
336 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
337 (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) {
338 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
339 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
340 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
341 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
345 IOV_INIT(iov, data, data_i, data_l);
348 CVMX_PREFETCH0(od->octo_enckey);
352 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
353 if (od->octo_encklen == 24) {
354 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
355 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
356 } else if (od->octo_encklen == 8) {
357 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
358 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
360 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
364 CVMX_MT_3DES_IV(* (uint64_t *) ivp);
366 while (crypt_off > 0) {
367 IOV_CONSUME(iov, data, data_i, data_l);
371 while (crypt_len > 0) {
372 CVMX_MT_3DES_ENC_CBC(*data);
373 CVMX_MF_3DES_RESULT(*data);
374 IOV_CONSUME(iov, data, data_i, data_l);
383 octo_des_cbc_decrypt(
384 struct octo_sess *od,
385 struct iovec *iov, size_t iovcnt, size_t iovlen,
386 int auth_off, int auth_len,
387 int crypt_off, int crypt_len,
388 int icv_off, uint8_t *ivp)
393 dprintf("%s()\n", __func__);
395 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
396 (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) {
397 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
398 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
399 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
400 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
404 IOV_INIT(iov, data, data_i, data_l);
407 CVMX_PREFETCH0(od->octo_enckey);
410 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
411 if (od->octo_encklen == 24) {
412 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
413 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
414 } else if (od->octo_encklen == 8) {
415 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
416 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
418 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
422 CVMX_MT_3DES_IV(* (uint64_t *) ivp);
424 while (crypt_off > 0) {
425 IOV_CONSUME(iov, data, data_i, data_l);
429 while (crypt_len > 0) {
430 CVMX_MT_3DES_DEC_CBC(*data);
431 CVMX_MF_3DES_RESULT(*data);
432 IOV_CONSUME(iov, data, data_i, data_l);
439 /****************************************************************************/
443 octo_aes_cbc_encrypt(
444 struct octo_sess *od,
445 struct iovec *iov, size_t iovcnt, size_t iovlen,
446 int auth_off, int auth_len,
447 int crypt_off, int crypt_len,
448 int icv_off, uint8_t *ivp)
450 uint64_t *data, *pdata;
453 dprintf("%s()\n", __func__);
455 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
456 (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) {
457 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
458 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
459 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
460 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
464 IOV_INIT(iov, data, data_i, data_l);
467 CVMX_PREFETCH0(od->octo_enckey);
470 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
471 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
473 if (od->octo_encklen == 16) {
474 CVMX_MT_AES_KEY(0x0, 2);
475 CVMX_MT_AES_KEY(0x0, 3);
476 } else if (od->octo_encklen == 24) {
477 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
478 CVMX_MT_AES_KEY(0x0, 3);
479 } else if (od->octo_encklen == 32) {
480 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
481 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
483 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
486 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
488 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
489 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
491 while (crypt_off > 0) {
492 IOV_CONSUME(iov, data, data_i, data_l);
496 while (crypt_len > 0) {
498 CVMX_MT_AES_ENC_CBC0(*data);
499 IOV_CONSUME(iov, data, data_i, data_l);
500 CVMX_MT_AES_ENC_CBC1(*data);
501 CVMX_MF_AES_RESULT(*pdata, 0);
502 CVMX_MF_AES_RESULT(*data, 1);
503 IOV_CONSUME(iov, data, data_i, data_l);
512 octo_aes_cbc_decrypt(
513 struct octo_sess *od,
514 struct iovec *iov, size_t iovcnt, size_t iovlen,
515 int auth_off, int auth_len,
516 int crypt_off, int crypt_len,
517 int icv_off, uint8_t *ivp)
519 uint64_t *data, *pdata;
522 dprintf("%s()\n", __func__);
524 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
525 (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) {
526 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
527 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
528 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
529 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
533 IOV_INIT(iov, data, data_i, data_l);
536 CVMX_PREFETCH0(od->octo_enckey);
539 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
540 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
542 if (od->octo_encklen == 16) {
543 CVMX_MT_AES_KEY(0x0, 2);
544 CVMX_MT_AES_KEY(0x0, 3);
545 } else if (od->octo_encklen == 24) {
546 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
547 CVMX_MT_AES_KEY(0x0, 3);
548 } else if (od->octo_encklen == 32) {
549 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
550 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
552 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
555 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
557 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
558 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
560 while (crypt_off > 0) {
561 IOV_CONSUME(iov, data, data_i, data_l);
565 while (crypt_len > 0) {
567 CVMX_MT_AES_DEC_CBC0(*data);
568 IOV_CONSUME(iov, data, data_i, data_l);
569 CVMX_MT_AES_DEC_CBC1(*data);
570 CVMX_MF_AES_RESULT(*pdata, 0);
571 CVMX_MF_AES_RESULT(*data, 1);
572 IOV_CONSUME(iov, data, data_i, data_l);
579 /****************************************************************************/
583 octo_null_md5_encrypt(
584 struct octo_sess *od,
585 struct iovec *iov, size_t iovcnt, size_t iovlen,
586 int auth_off, int auth_len,
587 int crypt_off, int crypt_len,
588 int icv_off, uint8_t *ivp)
590 register int next = 0;
593 int data_i, data_l, alen = auth_len;
595 dprintf("%s()\n", __func__);
597 if (__predict_false(od == NULL || iov==NULL || iovlen==0 ||
598 (auth_off & 0x7) || (auth_off + auth_len > iovlen))) {
599 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
600 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
601 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
602 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
606 IOV_INIT(iov, data, data_i, data_l);
609 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
610 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
612 while (auth_off > 0) {
613 IOV_CONSUME(iov, data, data_i, data_l);
617 while (auth_len > 0) {
618 CVM_LOAD_MD5_UNIT(*data, next);
620 IOV_CONSUME(iov, data, data_i, data_l);
623 /* finish the hash */
624 CVMX_PREFETCH0(od->octo_hmouter);
626 if (__predict_false(inplen)) {
628 uint8_t *p = (uint8_t *) & tmp;
632 p[inplen] = ((uint8_t *) data)[inplen];
634 CVM_LOAD_MD5_UNIT(tmp, next);
636 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
639 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
642 /* Finish Inner hash */
644 CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
646 CVMX_ES64(tmp1, ((alen + 64) << 3));
647 CVM_LOAD_MD5_UNIT(tmp1, next);
649 /* Get the inner hash of HMAC */
650 CVMX_MF_HSH_IV(tmp1, 0);
651 CVMX_MF_HSH_IV(tmp2, 1);
653 /* Initialize hash unit */
654 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
655 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
657 CVMX_MT_HSH_DAT(tmp1, 0);
658 CVMX_MT_HSH_DAT(tmp2, 1);
659 CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
664 CVMX_ES64(tmp1, ((64 + 16) << 3));
665 CVMX_MT_HSH_STARTMD5(tmp1);
668 IOV_INIT(iov, data, data_i, data_l);
669 while (icv_off > 0) {
670 IOV_CONSUME(iov, data, data_i, data_l);
673 CVMX_MF_HSH_IV(*data, 0);
674 IOV_CONSUME(iov, data, data_i, data_l);
675 CVMX_MF_HSH_IV(tmp1, 1);
676 *(uint32_t *)data = (uint32_t) (tmp1 >> 32);
681 /****************************************************************************/
685 octo_null_sha1_encrypt(
686 struct octo_sess *od,
687 struct iovec *iov, size_t iovcnt, size_t iovlen,
688 int auth_off, int auth_len,
689 int crypt_off, int crypt_len,
690 int icv_off, uint8_t *ivp)
692 register int next = 0;
694 uint64_t tmp1, tmp2, tmp3;
695 int data_i, data_l, alen = auth_len;
697 dprintf("%s()\n", __func__);
699 if (__predict_false(od == NULL || iov==NULL || iovlen==0 ||
700 (auth_off & 0x7) || (auth_off + auth_len > iovlen))) {
701 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
702 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
703 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
704 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
708 IOV_INIT(iov, data, data_i, data_l);
711 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
712 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
713 CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
715 while (auth_off > 0) {
716 IOV_CONSUME(iov, data, data_i, data_l);
720 while (auth_len > 0) {
721 CVM_LOAD_SHA_UNIT(*data, next);
723 IOV_CONSUME(iov, data, data_i, data_l);
726 /* finish the hash */
727 CVMX_PREFETCH0(od->octo_hmouter);
729 if (__predict_false(inplen)) {
731 uint8_t *p = (uint8_t *) & tmp;
735 p[inplen] = ((uint8_t *) data)[inplen];
737 CVM_LOAD_MD5_UNIT(tmp, next);
739 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
742 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
745 /* Finish Inner hash */
747 CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
749 CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
751 /* Get the inner hash of HMAC */
752 CVMX_MF_HSH_IV(tmp1, 0);
753 CVMX_MF_HSH_IV(tmp2, 1);
755 CVMX_MF_HSH_IV(tmp3, 2);
757 /* Initialize hash unit */
758 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
759 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
760 CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
762 CVMX_MT_HSH_DAT(tmp1, 0);
763 CVMX_MT_HSH_DAT(tmp2, 1);
764 tmp3 |= 0x0000000080000000;
765 CVMX_MT_HSH_DAT(tmp3, 2);
770 CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
773 IOV_INIT(iov, data, data_i, data_l);
774 while (icv_off > 0) {
775 IOV_CONSUME(iov, data, data_i, data_l);
778 CVMX_MF_HSH_IV(*data, 0);
779 IOV_CONSUME(iov, data, data_i, data_l);
780 CVMX_MF_HSH_IV(tmp1, 1);
781 *(uint32_t *)data = (uint32_t) (tmp1 >> 32);
786 /****************************************************************************/
790 octo_des_cbc_md5_encrypt(
791 struct octo_sess *od,
792 struct iovec *iov, size_t iovcnt, size_t iovlen,
793 int auth_off, int auth_len,
794 int crypt_off, int crypt_len,
795 int icv_off, uint8_t *ivp)
797 register int next = 0;
802 uint64_t *data = &mydata.data64[0];
805 int data_i, data_l, alen = auth_len;
807 dprintf("%s()\n", __func__);
809 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
810 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
813 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
814 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
815 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
816 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
817 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
821 IOV_INIT(iov, data32, data_i, data_l);
824 CVMX_PREFETCH0(od->octo_enckey);
827 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
828 if (od->octo_encklen == 24) {
829 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
830 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
831 } else if (od->octo_encklen == 8) {
832 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
833 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
835 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
839 CVMX_MT_3DES_IV(* (uint64_t *) ivp);
842 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
843 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
845 while (crypt_off > 0 && auth_off > 0) {
846 IOV_CONSUME(iov, data32, data_i, data_l);
851 while (crypt_len > 0 || auth_len > 0) {
852 uint32_t *first = data32;
853 mydata.data32[0] = *first;
854 IOV_CONSUME(iov, data32, data_i, data_l);
855 mydata.data32[1] = *data32;
856 if (crypt_off <= 0) {
858 CVMX_MT_3DES_ENC_CBC(*data);
859 CVMX_MF_3DES_RESULT(*data);
866 CVM_LOAD_MD5_UNIT(*data, next);
871 *first = mydata.data32[0];
872 *data32 = mydata.data32[1];
873 IOV_CONSUME(iov, data32, data_i, data_l);
876 /* finish the hash */
877 CVMX_PREFETCH0(od->octo_hmouter);
879 if (__predict_false(inplen)) {
881 uint8_t *p = (uint8_t *) & tmp;
885 p[inplen] = ((uint8_t *) data)[inplen];
887 CVM_LOAD_MD5_UNIT(tmp, next);
889 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
892 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
895 /* Finish Inner hash */
897 CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
899 CVMX_ES64(tmp1, ((alen + 64) << 3));
900 CVM_LOAD_MD5_UNIT(tmp1, next);
902 /* Get the inner hash of HMAC */
903 CVMX_MF_HSH_IV(tmp1, 0);
904 CVMX_MF_HSH_IV(tmp2, 1);
906 /* Initialize hash unit */
907 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
908 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
910 CVMX_MT_HSH_DAT(tmp1, 0);
911 CVMX_MT_HSH_DAT(tmp2, 1);
912 CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
917 CVMX_ES64(tmp1, ((64 + 16) << 3));
918 CVMX_MT_HSH_STARTMD5(tmp1);
921 IOV_INIT(iov, data32, data_i, data_l);
922 while (icv_off > 0) {
923 IOV_CONSUME(iov, data32, data_i, data_l);
926 CVMX_MF_HSH_IV(tmp1, 0);
927 *data32 = (uint32_t) (tmp1 >> 32);
928 IOV_CONSUME(iov, data32, data_i, data_l);
929 *data32 = (uint32_t) tmp1;
930 IOV_CONSUME(iov, data32, data_i, data_l);
931 CVMX_MF_HSH_IV(tmp1, 1);
932 *data32 = (uint32_t) (tmp1 >> 32);
938 octo_des_cbc_md5_decrypt(
939 struct octo_sess *od,
940 struct iovec *iov, size_t iovcnt, size_t iovlen,
941 int auth_off, int auth_len,
942 int crypt_off, int crypt_len,
943 int icv_off, uint8_t *ivp)
945 register int next = 0;
950 uint64_t *data = &mydata.data64[0];
953 int data_i, data_l, alen = auth_len;
955 dprintf("%s()\n", __func__);
957 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
958 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
961 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
962 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
963 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
964 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
965 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
969 IOV_INIT(iov, data32, data_i, data_l);
972 CVMX_PREFETCH0(od->octo_enckey);
975 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
976 if (od->octo_encklen == 24) {
977 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
978 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
979 } else if (od->octo_encklen == 8) {
980 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
981 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
983 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
987 CVMX_MT_3DES_IV(* (uint64_t *) ivp);
990 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
991 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
993 while (crypt_off > 0 && auth_off > 0) {
994 IOV_CONSUME(iov, data32, data_i, data_l);
999 while (crypt_len > 0 || auth_len > 0) {
1000 uint32_t *first = data32;
1001 mydata.data32[0] = *first;
1002 IOV_CONSUME(iov, data32, data_i, data_l);
1003 mydata.data32[1] = *data32;
1004 if (auth_off <= 0) {
1006 CVM_LOAD_MD5_UNIT(*data, next);
1011 if (crypt_off <= 0) {
1012 if (crypt_len > 0) {
1013 CVMX_MT_3DES_DEC_CBC(*data);
1014 CVMX_MF_3DES_RESULT(*data);
1019 *first = mydata.data32[0];
1020 *data32 = mydata.data32[1];
1021 IOV_CONSUME(iov, data32, data_i, data_l);
1024 /* finish the hash */
1025 CVMX_PREFETCH0(od->octo_hmouter);
1027 if (__predict_false(inplen)) {
1029 uint8_t *p = (uint8_t *) & tmp;
1033 p[inplen] = ((uint8_t *) data)[inplen];
1035 CVM_LOAD_MD5_UNIT(tmp, next);
1037 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
1040 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
1043 /* Finish Inner hash */
1045 CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
1047 CVMX_ES64(tmp1, ((alen + 64) << 3));
1048 CVM_LOAD_MD5_UNIT(tmp1, next);
1050 /* Get the inner hash of HMAC */
1051 CVMX_MF_HSH_IV(tmp1, 0);
1052 CVMX_MF_HSH_IV(tmp2, 1);
1054 /* Initialize hash unit */
1055 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
1056 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
1058 CVMX_MT_HSH_DAT(tmp1, 0);
1059 CVMX_MT_HSH_DAT(tmp2, 1);
1060 CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
1061 CVMX_MT_HSH_DATZ(3);
1062 CVMX_MT_HSH_DATZ(4);
1063 CVMX_MT_HSH_DATZ(5);
1064 CVMX_MT_HSH_DATZ(6);
1065 CVMX_ES64(tmp1, ((64 + 16) << 3));
1066 CVMX_MT_HSH_STARTMD5(tmp1);
1069 IOV_INIT(iov, data32, data_i, data_l);
1070 while (icv_off > 0) {
1071 IOV_CONSUME(iov, data32, data_i, data_l);
1074 CVMX_MF_HSH_IV(tmp1, 0);
1075 *data32 = (uint32_t) (tmp1 >> 32);
1076 IOV_CONSUME(iov, data32, data_i, data_l);
1077 *data32 = (uint32_t) tmp1;
1078 IOV_CONSUME(iov, data32, data_i, data_l);
1079 CVMX_MF_HSH_IV(tmp1, 1);
1080 *data32 = (uint32_t) (tmp1 >> 32);
1085 /****************************************************************************/
1089 octo_des_cbc_sha1_encrypt(
1090 struct octo_sess *od,
1091 struct iovec *iov, size_t iovcnt, size_t iovlen,
1092 int auth_off, int auth_len,
1093 int crypt_off, int crypt_len,
1094 int icv_off, uint8_t *ivp)
1096 register int next = 0;
1101 uint64_t *data = &mydata.data64[0];
1103 uint64_t tmp1, tmp2, tmp3;
1104 int data_i, data_l, alen = auth_len;
1106 dprintf("%s()\n", __func__);
1108 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
1109 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
1110 (crypt_len & 0x7) ||
1112 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
1113 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
1114 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
1115 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
1116 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
1120 IOV_INIT(iov, data32, data_i, data_l);
1122 CVMX_PREFETCH0(ivp);
1123 CVMX_PREFETCH0(od->octo_enckey);
1126 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
1127 if (od->octo_encklen == 24) {
1128 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
1129 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1130 } else if (od->octo_encklen == 8) {
1131 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
1132 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
1134 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
1138 CVMX_MT_3DES_IV(* (uint64_t *) ivp);
1141 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
1142 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
1143 CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
1145 while (crypt_off > 0 && auth_off > 0) {
1146 IOV_CONSUME(iov, data32, data_i, data_l);
1151 while (crypt_len > 0 || auth_len > 0) {
1152 uint32_t *first = data32;
1153 mydata.data32[0] = *first;
1154 IOV_CONSUME(iov, data32, data_i, data_l);
1155 mydata.data32[1] = *data32;
1156 if (crypt_off <= 0) {
1157 if (crypt_len > 0) {
1158 CVMX_MT_3DES_ENC_CBC(*data);
1159 CVMX_MF_3DES_RESULT(*data);
1164 if (auth_off <= 0) {
1166 CVM_LOAD_SHA_UNIT(*data, next);
1171 *first = mydata.data32[0];
1172 *data32 = mydata.data32[1];
1173 IOV_CONSUME(iov, data32, data_i, data_l);
1176 /* finish the hash */
1177 CVMX_PREFETCH0(od->octo_hmouter);
1179 if (__predict_false(inplen)) {
1181 uint8_t *p = (uint8_t *) & tmp;
1185 p[inplen] = ((uint8_t *) data)[inplen];
1187 CVM_LOAD_SHA_UNIT(tmp, next);
1189 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
1192 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
1195 /* Finish Inner hash */
1197 CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
1199 CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
1201 /* Get the inner hash of HMAC */
1202 CVMX_MF_HSH_IV(tmp1, 0);
1203 CVMX_MF_HSH_IV(tmp2, 1);
1205 CVMX_MF_HSH_IV(tmp3, 2);
1207 /* Initialize hash unit */
1208 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
1209 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
1210 CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
1212 CVMX_MT_HSH_DAT(tmp1, 0);
1213 CVMX_MT_HSH_DAT(tmp2, 1);
1214 tmp3 |= 0x0000000080000000;
1215 CVMX_MT_HSH_DAT(tmp3, 2);
1216 CVMX_MT_HSH_DATZ(3);
1217 CVMX_MT_HSH_DATZ(4);
1218 CVMX_MT_HSH_DATZ(5);
1219 CVMX_MT_HSH_DATZ(6);
1220 CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
1223 IOV_INIT(iov, data32, data_i, data_l);
1224 while (icv_off > 0) {
1225 IOV_CONSUME(iov, data32, data_i, data_l);
1228 CVMX_MF_HSH_IV(tmp1, 0);
1229 *data32 = (uint32_t) (tmp1 >> 32);
1230 IOV_CONSUME(iov, data32, data_i, data_l);
1231 *data32 = (uint32_t) tmp1;
1232 IOV_CONSUME(iov, data32, data_i, data_l);
1233 CVMX_MF_HSH_IV(tmp1, 1);
1234 *data32 = (uint32_t) (tmp1 >> 32);
1240 octo_des_cbc_sha1_decrypt(
1241 struct octo_sess *od,
1242 struct iovec *iov, size_t iovcnt, size_t iovlen,
1243 int auth_off, int auth_len,
1244 int crypt_off, int crypt_len,
1245 int icv_off, uint8_t *ivp)
1247 register int next = 0;
1252 uint64_t *data = &mydata.data64[0];
1254 uint64_t tmp1, tmp2, tmp3;
1255 int data_i, data_l, alen = auth_len;
1257 dprintf("%s()\n", __func__);
1259 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
1260 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
1261 (crypt_len & 0x7) ||
1263 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
1264 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
1265 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
1266 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
1267 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
1271 IOV_INIT(iov, data32, data_i, data_l);
1273 CVMX_PREFETCH0(ivp);
1274 CVMX_PREFETCH0(od->octo_enckey);
1277 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
1278 if (od->octo_encklen == 24) {
1279 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
1280 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1281 } else if (od->octo_encklen == 8) {
1282 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
1283 CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
1285 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
1289 CVMX_MT_3DES_IV(* (uint64_t *) ivp);
1292 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
1293 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
1294 CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
1296 while (crypt_off > 0 && auth_off > 0) {
1297 IOV_CONSUME(iov, data32, data_i, data_l);
1302 while (crypt_len > 0 || auth_len > 0) {
1303 uint32_t *first = data32;
1304 mydata.data32[0] = *first;
1305 IOV_CONSUME(iov, data32, data_i, data_l);
1306 mydata.data32[1] = *data32;
1307 if (auth_off <= 0) {
1309 CVM_LOAD_SHA_UNIT(*data, next);
1314 if (crypt_off <= 0) {
1315 if (crypt_len > 0) {
1316 CVMX_MT_3DES_DEC_CBC(*data);
1317 CVMX_MF_3DES_RESULT(*data);
1322 *first = mydata.data32[0];
1323 *data32 = mydata.data32[1];
1324 IOV_CONSUME(iov, data32, data_i, data_l);
1327 /* finish the hash */
1328 CVMX_PREFETCH0(od->octo_hmouter);
1330 if (__predict_false(inplen)) {
1332 uint8_t *p = (uint8_t *) & tmp;
1336 p[inplen] = ((uint8_t *) data)[inplen];
1338 CVM_LOAD_SHA_UNIT(tmp, next);
1340 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
1343 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
1346 /* Finish Inner hash */
1348 CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
1350 CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
1352 /* Get the inner hash of HMAC */
1353 CVMX_MF_HSH_IV(tmp1, 0);
1354 CVMX_MF_HSH_IV(tmp2, 1);
1356 CVMX_MF_HSH_IV(tmp3, 2);
1358 /* Initialize hash unit */
1359 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
1360 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
1361 CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
1363 CVMX_MT_HSH_DAT(tmp1, 0);
1364 CVMX_MT_HSH_DAT(tmp2, 1);
1365 tmp3 |= 0x0000000080000000;
1366 CVMX_MT_HSH_DAT(tmp3, 2);
1367 CVMX_MT_HSH_DATZ(3);
1368 CVMX_MT_HSH_DATZ(4);
1369 CVMX_MT_HSH_DATZ(5);
1370 CVMX_MT_HSH_DATZ(6);
1371 CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
1373 IOV_INIT(iov, data32, data_i, data_l);
1374 while (icv_off > 0) {
1375 IOV_CONSUME(iov, data32, data_i, data_l);
1378 CVMX_MF_HSH_IV(tmp1, 0);
1379 *data32 = (uint32_t) (tmp1 >> 32);
1380 IOV_CONSUME(iov, data32, data_i, data_l);
1381 *data32 = (uint32_t) tmp1;
1382 IOV_CONSUME(iov, data32, data_i, data_l);
1383 CVMX_MF_HSH_IV(tmp1, 1);
1384 *data32 = (uint32_t) (tmp1 >> 32);
1389 /****************************************************************************/
1393 octo_aes_cbc_md5_encrypt(
1394 struct octo_sess *od,
1395 struct iovec *iov, size_t iovcnt, size_t iovlen,
1396 int auth_off, int auth_len,
1397 int crypt_off, int crypt_len,
1398 int icv_off, uint8_t *ivp)
1400 register int next = 0;
1405 uint64_t *pdata = &mydata[0].data64[0];
1406 uint64_t *data = &mydata[1].data64[0];
1408 uint64_t tmp1, tmp2;
1409 int data_i, data_l, alen = auth_len;
1411 dprintf("%s()\n", __func__);
1413 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
1414 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
1415 (crypt_len & 0x7) ||
1417 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
1418 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
1419 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
1420 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
1421 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
1425 IOV_INIT(iov, data32, data_i, data_l);
1427 CVMX_PREFETCH0(ivp);
1428 CVMX_PREFETCH0(od->octo_enckey);
1431 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
1432 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
1434 if (od->octo_encklen == 16) {
1435 CVMX_MT_AES_KEY(0x0, 2);
1436 CVMX_MT_AES_KEY(0x0, 3);
1437 } else if (od->octo_encklen == 24) {
1438 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1439 CVMX_MT_AES_KEY(0x0, 3);
1440 } else if (od->octo_encklen == 32) {
1441 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1442 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
1444 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
1447 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
1449 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
1450 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
1453 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
1454 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
1456 while (crypt_off > 0 && auth_off > 0) {
1457 IOV_CONSUME(iov, data32, data_i, data_l);
1462 while (crypt_len > 0 || auth_len > 0) {
1463 uint32_t *pdata32[3];
1465 pdata32[0] = data32;
1466 mydata[0].data32[0] = *data32;
1467 IOV_CONSUME(iov, data32, data_i, data_l);
1469 pdata32[1] = data32;
1470 mydata[0].data32[1] = *data32;
1471 IOV_CONSUME(iov, data32, data_i, data_l);
1473 pdata32[2] = data32;
1474 mydata[1].data32[0] = *data32;
1475 IOV_CONSUME(iov, data32, data_i, data_l);
1477 mydata[1].data32[1] = *data32;
1480 if (crypt_off <= 0) {
1481 if (crypt_len > 0) {
1482 CVMX_MT_AES_ENC_CBC0(*pdata);
1483 CVMX_MT_AES_ENC_CBC1(*data);
1484 CVMX_MF_AES_RESULT(*pdata, 0);
1485 CVMX_MF_AES_RESULT(*data, 1);
1491 if (auth_off <= 0) {
1493 CVM_LOAD_MD5_UNIT(*pdata, next);
1494 CVM_LOAD_MD5_UNIT(*data, next);
1500 *pdata32[0] = mydata[0].data32[0];
1501 *pdata32[1] = mydata[0].data32[1];
1502 *pdata32[2] = mydata[1].data32[0];
1503 *data32 = mydata[1].data32[1];
1505 IOV_CONSUME(iov, data32, data_i, data_l);
1508 /* finish the hash */
1509 CVMX_PREFETCH0(od->octo_hmouter);
1511 if (__predict_false(inplen)) {
1513 uint8_t *p = (uint8_t *) & tmp;
1517 p[inplen] = ((uint8_t *) data)[inplen];
1519 CVM_LOAD_MD5_UNIT(tmp, next);
1521 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
1524 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
1527 /* Finish Inner hash */
1529 CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
1531 CVMX_ES64(tmp1, ((alen + 64) << 3));
1532 CVM_LOAD_MD5_UNIT(tmp1, next);
1534 /* Get the inner hash of HMAC */
1535 CVMX_MF_HSH_IV(tmp1, 0);
1536 CVMX_MF_HSH_IV(tmp2, 1);
1538 /* Initialize hash unit */
1539 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
1540 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
1542 CVMX_MT_HSH_DAT(tmp1, 0);
1543 CVMX_MT_HSH_DAT(tmp2, 1);
1544 CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
1545 CVMX_MT_HSH_DATZ(3);
1546 CVMX_MT_HSH_DATZ(4);
1547 CVMX_MT_HSH_DATZ(5);
1548 CVMX_MT_HSH_DATZ(6);
1549 CVMX_ES64(tmp1, ((64 + 16) << 3));
1550 CVMX_MT_HSH_STARTMD5(tmp1);
1553 IOV_INIT(iov, data32, data_i, data_l);
1554 while (icv_off > 0) {
1555 IOV_CONSUME(iov, data32, data_i, data_l);
1558 CVMX_MF_HSH_IV(tmp1, 0);
1559 *data32 = (uint32_t) (tmp1 >> 32);
1560 IOV_CONSUME(iov, data32, data_i, data_l);
1561 *data32 = (uint32_t) tmp1;
1562 IOV_CONSUME(iov, data32, data_i, data_l);
1563 CVMX_MF_HSH_IV(tmp1, 1);
1564 *data32 = (uint32_t) (tmp1 >> 32);
1570 octo_aes_cbc_md5_decrypt(
1571 struct octo_sess *od,
1572 struct iovec *iov, size_t iovcnt, size_t iovlen,
1573 int auth_off, int auth_len,
1574 int crypt_off, int crypt_len,
1575 int icv_off, uint8_t *ivp)
1577 register int next = 0;
1582 uint64_t *pdata = &mydata[0].data64[0];
1583 uint64_t *data = &mydata[1].data64[0];
1585 uint64_t tmp1, tmp2;
1586 int data_i, data_l, alen = auth_len;
1588 dprintf("%s()\n", __func__);
1590 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
1591 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
1592 (crypt_len & 0x7) ||
1594 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
1595 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
1596 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
1597 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
1598 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
1602 IOV_INIT(iov, data32, data_i, data_l);
1604 CVMX_PREFETCH0(ivp);
1605 CVMX_PREFETCH0(od->octo_enckey);
1608 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
1609 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
1611 if (od->octo_encklen == 16) {
1612 CVMX_MT_AES_KEY(0x0, 2);
1613 CVMX_MT_AES_KEY(0x0, 3);
1614 } else if (od->octo_encklen == 24) {
1615 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1616 CVMX_MT_AES_KEY(0x0, 3);
1617 } else if (od->octo_encklen == 32) {
1618 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1619 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
1621 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
1624 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
1626 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
1627 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
1630 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
1631 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
1633 while (crypt_off > 0 && auth_off > 0) {
1634 IOV_CONSUME(iov, data32, data_i, data_l);
1639 while (crypt_len > 0 || auth_len > 0) {
1640 uint32_t *pdata32[3];
1642 pdata32[0] = data32;
1643 mydata[0].data32[0] = *data32;
1644 IOV_CONSUME(iov, data32, data_i, data_l);
1645 pdata32[1] = data32;
1646 mydata[0].data32[1] = *data32;
1647 IOV_CONSUME(iov, data32, data_i, data_l);
1648 pdata32[2] = data32;
1649 mydata[1].data32[0] = *data32;
1650 IOV_CONSUME(iov, data32, data_i, data_l);
1651 mydata[1].data32[1] = *data32;
1653 if (auth_off <= 0) {
1655 CVM_LOAD_MD5_UNIT(*pdata, next);
1656 CVM_LOAD_MD5_UNIT(*data, next);
1662 if (crypt_off <= 0) {
1663 if (crypt_len > 0) {
1664 CVMX_MT_AES_DEC_CBC0(*pdata);
1665 CVMX_MT_AES_DEC_CBC1(*data);
1666 CVMX_MF_AES_RESULT(*pdata, 0);
1667 CVMX_MF_AES_RESULT(*data, 1);
1673 *pdata32[0] = mydata[0].data32[0];
1674 *pdata32[1] = mydata[0].data32[1];
1675 *pdata32[2] = mydata[1].data32[0];
1676 *data32 = mydata[1].data32[1];
1678 IOV_CONSUME(iov, data32, data_i, data_l);
1681 /* finish the hash */
1682 CVMX_PREFETCH0(od->octo_hmouter);
1684 if (__predict_false(inplen)) {
1686 uint8_t *p = (uint8_t *) & tmp;
1690 p[inplen] = ((uint8_t *) data)[inplen];
1692 CVM_LOAD_MD5_UNIT(tmp, next);
1694 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
1697 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
1700 /* Finish Inner hash */
1702 CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
1704 CVMX_ES64(tmp1, ((alen + 64) << 3));
1705 CVM_LOAD_MD5_UNIT(tmp1, next);
1707 /* Get the inner hash of HMAC */
1708 CVMX_MF_HSH_IV(tmp1, 0);
1709 CVMX_MF_HSH_IV(tmp2, 1);
1711 /* Initialize hash unit */
1712 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
1713 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
1715 CVMX_MT_HSH_DAT(tmp1, 0);
1716 CVMX_MT_HSH_DAT(tmp2, 1);
1717 CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
1718 CVMX_MT_HSH_DATZ(3);
1719 CVMX_MT_HSH_DATZ(4);
1720 CVMX_MT_HSH_DATZ(5);
1721 CVMX_MT_HSH_DATZ(6);
1722 CVMX_ES64(tmp1, ((64 + 16) << 3));
1723 CVMX_MT_HSH_STARTMD5(tmp1);
1726 IOV_INIT(iov, data32, data_i, data_l);
1727 while (icv_off > 0) {
1728 IOV_CONSUME(iov, data32, data_i, data_l);
1731 CVMX_MF_HSH_IV(tmp1, 0);
1732 *data32 = (uint32_t) (tmp1 >> 32);
1733 IOV_CONSUME(iov, data32, data_i, data_l);
1734 *data32 = (uint32_t) tmp1;
1735 IOV_CONSUME(iov, data32, data_i, data_l);
1736 CVMX_MF_HSH_IV(tmp1, 1);
1737 *data32 = (uint32_t) (tmp1 >> 32);
1742 /****************************************************************************/
1746 octo_aes_cbc_sha1_encrypt(
1747 struct octo_sess *od,
1748 struct iovec *iov, size_t iovcnt, size_t iovlen,
1749 int auth_off, int auth_len,
1750 int crypt_off, int crypt_len,
1751 int icv_off, uint8_t *ivp)
1753 register int next = 0;
1758 uint64_t *pdata = &mydata[0].data64[0];
1759 uint64_t *data = &mydata[1].data64[0];
1761 uint64_t tmp1, tmp2, tmp3;
1762 int data_i, data_l, alen = auth_len;
1764 dprintf("%s()\n", __func__);
1766 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
1767 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
1768 (crypt_len & 0x7) ||
1770 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
1771 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
1772 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
1773 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
1774 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
1778 IOV_INIT(iov, data32, data_i, data_l);
1780 CVMX_PREFETCH0(ivp);
1781 CVMX_PREFETCH0(od->octo_enckey);
1784 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
1785 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
1787 if (od->octo_encklen == 16) {
1788 CVMX_MT_AES_KEY(0x0, 2);
1789 CVMX_MT_AES_KEY(0x0, 3);
1790 } else if (od->octo_encklen == 24) {
1791 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1792 CVMX_MT_AES_KEY(0x0, 3);
1793 } else if (od->octo_encklen == 32) {
1794 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1795 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
1797 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
1800 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
1802 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
1803 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
1806 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
1807 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
1808 CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
1810 while (crypt_off > 0 && auth_off > 0) {
1811 IOV_CONSUME(iov, data32, data_i, data_l);
1816 while (crypt_len > 0 || auth_len > 0) {
1817 uint32_t *pdata32[3];
1819 pdata32[0] = data32;
1820 mydata[0].data32[0] = *data32;
1821 IOV_CONSUME(iov, data32, data_i, data_l);
1822 pdata32[1] = data32;
1823 mydata[0].data32[1] = *data32;
1824 IOV_CONSUME(iov, data32, data_i, data_l);
1825 pdata32[2] = data32;
1826 mydata[1].data32[0] = *data32;
1827 IOV_CONSUME(iov, data32, data_i, data_l);
1828 mydata[1].data32[1] = *data32;
1831 if (crypt_off <= 0) {
1832 if (crypt_len > 0) {
1833 CVMX_MT_AES_ENC_CBC0(*pdata);
1834 CVMX_MT_AES_ENC_CBC1(*data);
1835 CVMX_MF_AES_RESULT(*pdata, 0);
1836 CVMX_MF_AES_RESULT(*data, 1);
1842 if (auth_off <= 0) {
1844 CVM_LOAD_SHA_UNIT(*pdata, next);
1845 CVM_LOAD_SHA_UNIT(*data, next);
1851 *pdata32[0] = mydata[0].data32[0];
1852 *pdata32[1] = mydata[0].data32[1];
1853 *pdata32[2] = mydata[1].data32[0];
1854 *data32 = mydata[1].data32[1];
1856 IOV_CONSUME(iov, data32, data_i, data_l);
1859 /* finish the hash */
1860 CVMX_PREFETCH0(od->octo_hmouter);
1862 if (__predict_false(inplen)) {
1864 uint8_t *p = (uint8_t *) & tmp;
1868 p[inplen] = ((uint8_t *) data)[inplen];
1870 CVM_LOAD_SHA_UNIT(tmp, next);
1872 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
1875 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
1878 /* Finish Inner hash */
1880 CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
1882 CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
1884 /* Get the inner hash of HMAC */
1885 CVMX_MF_HSH_IV(tmp1, 0);
1886 CVMX_MF_HSH_IV(tmp2, 1);
1888 CVMX_MF_HSH_IV(tmp3, 2);
1890 /* Initialize hash unit */
1891 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
1892 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
1893 CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
1895 CVMX_MT_HSH_DAT(tmp1, 0);
1896 CVMX_MT_HSH_DAT(tmp2, 1);
1897 tmp3 |= 0x0000000080000000;
1898 CVMX_MT_HSH_DAT(tmp3, 2);
1899 CVMX_MT_HSH_DATZ(3);
1900 CVMX_MT_HSH_DATZ(4);
1901 CVMX_MT_HSH_DATZ(5);
1902 CVMX_MT_HSH_DATZ(6);
1903 CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
1905 /* finish the hash */
1906 CVMX_PREFETCH0(od->octo_hmouter);
1908 if (__predict_false(inplen)) {
1910 uint8_t *p = (uint8_t *) & tmp;
1914 p[inplen] = ((uint8_t *) data)[inplen];
1916 CVM_LOAD_MD5_UNIT(tmp, next);
1918 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
1921 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
1925 IOV_INIT(iov, data32, data_i, data_l);
1926 while (icv_off > 0) {
1927 IOV_CONSUME(iov, data32, data_i, data_l);
1930 CVMX_MF_HSH_IV(tmp1, 0);
1931 *data32 = (uint32_t) (tmp1 >> 32);
1932 IOV_CONSUME(iov, data32, data_i, data_l);
1933 *data32 = (uint32_t) tmp1;
1934 IOV_CONSUME(iov, data32, data_i, data_l);
1935 CVMX_MF_HSH_IV(tmp1, 1);
1936 *data32 = (uint32_t) (tmp1 >> 32);
1942 octo_aes_cbc_sha1_decrypt(
1943 struct octo_sess *od,
1944 struct iovec *iov, size_t iovcnt, size_t iovlen,
1945 int auth_off, int auth_len,
1946 int crypt_off, int crypt_len,
1947 int icv_off, uint8_t *ivp)
1949 register int next = 0;
1954 uint64_t *pdata = &mydata[0].data64[0];
1955 uint64_t *data = &mydata[1].data64[0];
1957 uint64_t tmp1, tmp2, tmp3;
1958 int data_i, data_l, alen = auth_len;
1960 dprintf("%s()\n", __func__);
1962 if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
1963 (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
1964 (crypt_len & 0x7) ||
1966 (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
1967 dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
1968 "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
1969 "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
1970 auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
1974 IOV_INIT(iov, data32, data_i, data_l);
1976 CVMX_PREFETCH0(ivp);
1977 CVMX_PREFETCH0(od->octo_enckey);
1980 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
1981 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
1983 if (od->octo_encklen == 16) {
1984 CVMX_MT_AES_KEY(0x0, 2);
1985 CVMX_MT_AES_KEY(0x0, 3);
1986 } else if (od->octo_encklen == 24) {
1987 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1988 CVMX_MT_AES_KEY(0x0, 3);
1989 } else if (od->octo_encklen == 32) {
1990 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
1991 CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
1993 dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
1996 CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
1998 CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
1999 CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
2002 CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
2003 CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
2004 CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
2006 while (crypt_off > 0 && auth_off > 0) {
2007 IOV_CONSUME(iov, data32, data_i, data_l);
2012 while (crypt_len > 0 || auth_len > 0) {
2013 uint32_t *pdata32[3];
2015 pdata32[0] = data32;
2016 mydata[0].data32[0] = *data32;
2017 IOV_CONSUME(iov, data32, data_i, data_l);
2018 pdata32[1] = data32;
2019 mydata[0].data32[1] = *data32;
2020 IOV_CONSUME(iov, data32, data_i, data_l);
2021 pdata32[2] = data32;
2022 mydata[1].data32[0] = *data32;
2023 IOV_CONSUME(iov, data32, data_i, data_l);
2024 mydata[1].data32[1] = *data32;
2026 if (auth_off <= 0) {
2028 CVM_LOAD_SHA_UNIT(*pdata, next);
2029 CVM_LOAD_SHA_UNIT(*data, next);
2035 if (crypt_off <= 0) {
2036 if (crypt_len > 0) {
2037 CVMX_MT_AES_DEC_CBC0(*pdata);
2038 CVMX_MT_AES_DEC_CBC1(*data);
2039 CVMX_MF_AES_RESULT(*pdata, 0);
2040 CVMX_MF_AES_RESULT(*data, 1);
2046 *pdata32[0] = mydata[0].data32[0];
2047 *pdata32[1] = mydata[0].data32[1];
2048 *pdata32[2] = mydata[1].data32[0];
2049 *data32 = mydata[1].data32[1];
2051 IOV_CONSUME(iov, data32, data_i, data_l);
2054 /* finish the hash */
2055 CVMX_PREFETCH0(od->octo_hmouter);
2057 if (__predict_false(inplen)) {
2059 uint8_t *p = (uint8_t *) & tmp;
2063 p[inplen] = ((uint8_t *) data)[inplen];
2065 CVM_LOAD_SHA_UNIT(tmp, next);
2067 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
2070 CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
2073 /* Finish Inner hash */
2075 CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
2077 CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
2079 /* Get the inner hash of HMAC */
2080 CVMX_MF_HSH_IV(tmp1, 0);
2081 CVMX_MF_HSH_IV(tmp2, 1);
2083 CVMX_MF_HSH_IV(tmp3, 2);
2085 /* Initialize hash unit */
2086 CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
2087 CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
2088 CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
2090 CVMX_MT_HSH_DAT(tmp1, 0);
2091 CVMX_MT_HSH_DAT(tmp2, 1);
2092 tmp3 |= 0x0000000080000000;
2093 CVMX_MT_HSH_DAT(tmp3, 2);
2094 CVMX_MT_HSH_DATZ(3);
2095 CVMX_MT_HSH_DATZ(4);
2096 CVMX_MT_HSH_DATZ(5);
2097 CVMX_MT_HSH_DATZ(6);
2098 CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
2100 /* finish the hash */
2101 CVMX_PREFETCH0(od->octo_hmouter);
2103 if (__predict_false(inplen)) {
2105 uint8_t *p = (uint8_t *) & tmp;
2109 p[inplen] = ((uint8_t *) data)[inplen];
2111 CVM_LOAD_MD5_UNIT(tmp, next);
2113 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
2116 CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
2120 IOV_INIT(iov, data32, data_i, data_l);
2121 while (icv_off > 0) {
2122 IOV_CONSUME(iov, data32, data_i, data_l);
2125 CVMX_MF_HSH_IV(tmp1, 0);
2126 *data32 = (uint32_t) (tmp1 >> 32);
2127 IOV_CONSUME(iov, data32, data_i, data_l);
2128 *data32 = (uint32_t) tmp1;
2129 IOV_CONSUME(iov, data32, data_i, data_l);
2130 CVMX_MF_HSH_IV(tmp1, 1);
2131 *data32 = (uint32_t) (tmp1 >> 32);
2136 /****************************************************************************/