/* * vim:sw=4 ts=8 */ /* * Copyright (c) 2009 David McCullough * * Copyright (c) 2003-2007 Cavium Networks (support@cavium.com). All rights * reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Cavium Networks * 4. Cavium Networks' name may not be used to endorse or promote products * derived from this software without specific prior written permission. * * This Software, including technical data, may be subject to U.S. export * control laws, including the U.S. Export Administration Act and its * associated regulations, and may be subject to export or import regulations * in other countries. You warrant that You will comply strictly in all * respects with all such regulations and acknowledge that you have the * responsibility to obtain licenses to export, re-export or import the * Software. * * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" AND * WITH ALL FAULTS AND CAVIUM MAKES NO PROMISES, REPRESENTATIONS OR WARRANTIES, * EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO THE * SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR * DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE, * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR * PERFORMANCE OF THE SOFTWARE LIES WITH YOU. */ /****************************************************************************/ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include /****************************************************************************/ #define IOV_INIT(iov, ptr, idx, len) \ do { \ (idx) = 0; \ (ptr) = (iov)[(idx)].iov_base; \ (len) = (iov)[(idx)].iov_len; \ } while (0) /* * XXX * It would be better if this were an IOV_READ/IOV_WRITE macro instead so * that we could detect overflow before it happens rather than right after, * which is especially bad since there is usually no IOV_CONSUME after the * final read or write. */ #define IOV_CONSUME(iov, ptr, idx, len) \ do { \ if ((len) > sizeof *(ptr)) { \ (len) -= sizeof *(ptr); \ (ptr)++; \ } else { \ if ((len) != sizeof *(ptr)) \ panic("%s: went past end of iovec.", __func__); \ (idx)++; \ (ptr) = (iov)[(idx)].iov_base; \ (len) = (iov)[(idx)].iov_len; \ } \ } while (0) static inline unsigned long octeon_crypto_enable(void) { register_t s; s = intr_disable(); mips_wr_status(mips_rd_status() | MIPS_SR_COP_2_BIT); return (s); } static inline void octeon_crypto_disable(register_t s) { mips_wr_status(mips_rd_status() & ~MIPS_SR_COP_2_BIT); intr_restore(s); } #define ESP_HEADER_LENGTH 8 #define DES_CBC_IV_LENGTH 8 #define AES_CBC_IV_LENGTH 16 #define ESP_HMAC_LEN 12 #define ESP_HEADER_LENGTH 8 #define DES_CBC_IV_LENGTH 8 /****************************************************************************/ #define CVM_LOAD_SHA_UNIT(dat, next) { \ if (next == 0) { \ next = 1; \ CVMX_MT_HSH_DAT (dat, 0); \ } else if (next == 1) { \ next = 2; \ CVMX_MT_HSH_DAT (dat, 1); \ } else if (next == 2) { \ next = 3; \ CVMX_MT_HSH_DAT (dat, 2); \ } else if (next == 3) { \ next = 4; \ CVMX_MT_HSH_DAT (dat, 3); \ } else if (next == 4) { \ next = 5; \ CVMX_MT_HSH_DAT (dat, 4); \ } else if (next == 5) { \ next = 6; \ CVMX_MT_HSH_DAT (dat, 5); \ } else if (next == 6) { \ next = 7; \ CVMX_MT_HSH_DAT (dat, 6); \ } else { \ CVMX_MT_HSH_STARTSHA (dat); \ next = 0; \ } \ } #define CVM_LOAD2_SHA_UNIT(dat1, dat2, next) { \ if (next == 0) { \ CVMX_MT_HSH_DAT (dat1, 0); \ CVMX_MT_HSH_DAT (dat2, 1); \ next = 2; \ } else if (next == 1) { \ CVMX_MT_HSH_DAT (dat1, 1); \ CVMX_MT_HSH_DAT (dat2, 2); \ next = 3; \ } else if (next == 2) { \ CVMX_MT_HSH_DAT (dat1, 2); \ CVMX_MT_HSH_DAT (dat2, 3); \ next = 4; \ } else if (next == 3) { \ CVMX_MT_HSH_DAT (dat1, 3); \ CVMX_MT_HSH_DAT (dat2, 4); \ next = 5; \ } else if (next == 4) { \ CVMX_MT_HSH_DAT (dat1, 4); \ CVMX_MT_HSH_DAT (dat2, 5); \ next = 6; \ } else if (next == 5) { \ CVMX_MT_HSH_DAT (dat1, 5); \ CVMX_MT_HSH_DAT (dat2, 6); \ next = 7; \ } else if (next == 6) { \ CVMX_MT_HSH_DAT (dat1, 6); \ CVMX_MT_HSH_STARTSHA (dat2); \ next = 0; \ } else { \ CVMX_MT_HSH_STARTSHA (dat1); \ CVMX_MT_HSH_DAT (dat2, 0); \ next = 1; \ } \ } /****************************************************************************/ #define CVM_LOAD_MD5_UNIT(dat, next) { \ if (next == 0) { \ next = 1; \ CVMX_MT_HSH_DAT (dat, 0); \ } else if (next == 1) { \ next = 2; \ CVMX_MT_HSH_DAT (dat, 1); \ } else if (next == 2) { \ next = 3; \ CVMX_MT_HSH_DAT (dat, 2); \ } else if (next == 3) { \ next = 4; \ CVMX_MT_HSH_DAT (dat, 3); \ } else if (next == 4) { \ next = 5; \ CVMX_MT_HSH_DAT (dat, 4); \ } else if (next == 5) { \ next = 6; \ CVMX_MT_HSH_DAT (dat, 5); \ } else if (next == 6) { \ next = 7; \ CVMX_MT_HSH_DAT (dat, 6); \ } else { \ CVMX_MT_HSH_STARTMD5 (dat); \ next = 0; \ } \ } #define CVM_LOAD2_MD5_UNIT(dat1, dat2, next) { \ if (next == 0) { \ CVMX_MT_HSH_DAT (dat1, 0); \ CVMX_MT_HSH_DAT (dat2, 1); \ next = 2; \ } else if (next == 1) { \ CVMX_MT_HSH_DAT (dat1, 1); \ CVMX_MT_HSH_DAT (dat2, 2); \ next = 3; \ } else if (next == 2) { \ CVMX_MT_HSH_DAT (dat1, 2); \ CVMX_MT_HSH_DAT (dat2, 3); \ next = 4; \ } else if (next == 3) { \ CVMX_MT_HSH_DAT (dat1, 3); \ CVMX_MT_HSH_DAT (dat2, 4); \ next = 5; \ } else if (next == 4) { \ CVMX_MT_HSH_DAT (dat1, 4); \ CVMX_MT_HSH_DAT (dat2, 5); \ next = 6; \ } else if (next == 5) { \ CVMX_MT_HSH_DAT (dat1, 5); \ CVMX_MT_HSH_DAT (dat2, 6); \ next = 7; \ } else if (next == 6) { \ CVMX_MT_HSH_DAT (dat1, 6); \ CVMX_MT_HSH_STARTMD5 (dat2); \ next = 0; \ } else { \ CVMX_MT_HSH_STARTMD5 (dat1); \ CVMX_MT_HSH_DAT (dat2, 0); \ next = 1; \ } \ } /****************************************************************************/ void octo_calc_hash(uint8_t auth, unsigned char *key, uint64_t *inner, uint64_t *outer) { uint8_t hash_key[64]; uint64_t *key1; register uint64_t xor1 = 0x3636363636363636ULL; register uint64_t xor2 = 0x5c5c5c5c5c5c5c5cULL; register_t s; dprintf("%s()\n", __func__); memset(hash_key, 0, sizeof(hash_key)); memcpy(hash_key, (uint8_t *) key, (auth ? 20 : 16)); key1 = (uint64_t *) hash_key; s = octeon_crypto_enable(); if (auth) { CVMX_MT_HSH_IV(0x67452301EFCDAB89ULL, 0); CVMX_MT_HSH_IV(0x98BADCFE10325476ULL, 1); CVMX_MT_HSH_IV(0xC3D2E1F000000000ULL, 2); } else { CVMX_MT_HSH_IV(0x0123456789ABCDEFULL, 0); CVMX_MT_HSH_IV(0xFEDCBA9876543210ULL, 1); } CVMX_MT_HSH_DAT((*key1 ^ xor1), 0); key1++; CVMX_MT_HSH_DAT((*key1 ^ xor1), 1); key1++; CVMX_MT_HSH_DAT((*key1 ^ xor1), 2); key1++; CVMX_MT_HSH_DAT((*key1 ^ xor1), 3); key1++; CVMX_MT_HSH_DAT((*key1 ^ xor1), 4); key1++; CVMX_MT_HSH_DAT((*key1 ^ xor1), 5); key1++; CVMX_MT_HSH_DAT((*key1 ^ xor1), 6); key1++; if (auth) CVMX_MT_HSH_STARTSHA((*key1 ^ xor1)); else CVMX_MT_HSH_STARTMD5((*key1 ^ xor1)); CVMX_MF_HSH_IV(inner[0], 0); CVMX_MF_HSH_IV(inner[1], 1); if (auth) { inner[2] = 0; CVMX_MF_HSH_IV(((uint64_t *) inner)[2], 2); } memset(hash_key, 0, sizeof(hash_key)); memcpy(hash_key, (uint8_t *) key, (auth ? 20 : 16)); key1 = (uint64_t *) hash_key; if (auth) { CVMX_MT_HSH_IV(0x67452301EFCDAB89ULL, 0); CVMX_MT_HSH_IV(0x98BADCFE10325476ULL, 1); CVMX_MT_HSH_IV(0xC3D2E1F000000000ULL, 2); } else { CVMX_MT_HSH_IV(0x0123456789ABCDEFULL, 0); CVMX_MT_HSH_IV(0xFEDCBA9876543210ULL, 1); } CVMX_MT_HSH_DAT((*key1 ^ xor2), 0); key1++; CVMX_MT_HSH_DAT((*key1 ^ xor2), 1); key1++; CVMX_MT_HSH_DAT((*key1 ^ xor2), 2); key1++; CVMX_MT_HSH_DAT((*key1 ^ xor2), 3); key1++; CVMX_MT_HSH_DAT((*key1 ^ xor2), 4); key1++; CVMX_MT_HSH_DAT((*key1 ^ xor2), 5); key1++; CVMX_MT_HSH_DAT((*key1 ^ xor2), 6); key1++; if (auth) CVMX_MT_HSH_STARTSHA((*key1 ^ xor2)); else CVMX_MT_HSH_STARTMD5((*key1 ^ xor2)); CVMX_MF_HSH_IV(outer[0], 0); CVMX_MF_HSH_IV(outer[1], 1); if (auth) { outer[2] = 0; CVMX_MF_HSH_IV(outer[2], 2); } octeon_crypto_disable(s); return; } /****************************************************************************/ /* DES functions */ int octo_des_cbc_encrypt( struct octo_sess *od, struct iovec *iov, size_t iovcnt, size_t iovlen, int auth_off, int auth_len, int crypt_off, int crypt_len, int icv_off, uint8_t *ivp) { uint64_t *data; int data_i, data_l; register_t s; dprintf("%s()\n", __func__); if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) { dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); return -EINVAL; } IOV_INIT(iov, data, data_i, data_l); CVMX_PREFETCH0(ivp); CVMX_PREFETCH0(od->octo_enckey); s = octeon_crypto_enable(); /* load 3DES Key */ CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); if (od->octo_encklen == 24) { CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); } else if (od->octo_encklen == 8) { CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); } else { octeon_crypto_disable(s); dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); return -EINVAL; } CVMX_MT_3DES_IV(* (uint64_t *) ivp); while (crypt_off > 0) { IOV_CONSUME(iov, data, data_i, data_l); crypt_off -= 8; } while (crypt_len > 0) { CVMX_MT_3DES_ENC_CBC(*data); CVMX_MF_3DES_RESULT(*data); IOV_CONSUME(iov, data, data_i, data_l); crypt_len -= 8; } octeon_crypto_disable(s); return 0; } int octo_des_cbc_decrypt( struct octo_sess *od, struct iovec *iov, size_t iovcnt, size_t iovlen, int auth_off, int auth_len, int crypt_off, int crypt_len, int icv_off, uint8_t *ivp) { uint64_t *data; int data_i, data_l; register_t s; dprintf("%s()\n", __func__); if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) { dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); return -EINVAL; } IOV_INIT(iov, data, data_i, data_l); CVMX_PREFETCH0(ivp); CVMX_PREFETCH0(od->octo_enckey); s = octeon_crypto_enable(); /* load 3DES Key */ CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); if (od->octo_encklen == 24) { CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); } else if (od->octo_encklen == 8) { CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); } else { octeon_crypto_disable(s); dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); return -EINVAL; } CVMX_MT_3DES_IV(* (uint64_t *) ivp); while (crypt_off > 0) { IOV_CONSUME(iov, data, data_i, data_l); crypt_off -= 8; } while (crypt_len > 0) { CVMX_MT_3DES_DEC_CBC(*data); CVMX_MF_3DES_RESULT(*data); IOV_CONSUME(iov, data, data_i, data_l); crypt_len -= 8; } octeon_crypto_disable(s); return 0; } /****************************************************************************/ /* AES functions */ int octo_aes_cbc_encrypt( struct octo_sess *od, struct iovec *iov, size_t iovcnt, size_t iovlen, int auth_off, int auth_len, int crypt_off, int crypt_len, int icv_off, uint8_t *ivp) { uint64_t *data, *pdata; int data_i, data_l; register_t s; dprintf("%s()\n", __func__); if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) { dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); return -EINVAL; } IOV_INIT(iov, data, data_i, data_l); CVMX_PREFETCH0(ivp); CVMX_PREFETCH0(od->octo_enckey); s = octeon_crypto_enable(); /* load AES Key */ CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); if (od->octo_encklen == 16) { CVMX_MT_AES_KEY(0x0, 2); CVMX_MT_AES_KEY(0x0, 3); } else if (od->octo_encklen == 24) { CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); CVMX_MT_AES_KEY(0x0, 3); } else if (od->octo_encklen == 32) { CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); } else { octeon_crypto_disable(s); dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); return -EINVAL; } CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); while (crypt_off > 0) { IOV_CONSUME(iov, data, data_i, data_l); crypt_off -= 8; } while (crypt_len > 0) { pdata = data; CVMX_MT_AES_ENC_CBC0(*data); IOV_CONSUME(iov, data, data_i, data_l); CVMX_MT_AES_ENC_CBC1(*data); CVMX_MF_AES_RESULT(*pdata, 0); CVMX_MF_AES_RESULT(*data, 1); IOV_CONSUME(iov, data, data_i, data_l); crypt_len -= 16; } octeon_crypto_disable(s); return 0; } int octo_aes_cbc_decrypt( struct octo_sess *od, struct iovec *iov, size_t iovcnt, size_t iovlen, int auth_off, int auth_len, int crypt_off, int crypt_len, int icv_off, uint8_t *ivp) { uint64_t *data, *pdata; int data_i, data_l; register_t s; dprintf("%s()\n", __func__); if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) { dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); return -EINVAL; } IOV_INIT(iov, data, data_i, data_l); CVMX_PREFETCH0(ivp); CVMX_PREFETCH0(od->octo_enckey); s = octeon_crypto_enable(); /* load AES Key */ CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); if (od->octo_encklen == 16) { CVMX_MT_AES_KEY(0x0, 2); CVMX_MT_AES_KEY(0x0, 3); } else if (od->octo_encklen == 24) { CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); CVMX_MT_AES_KEY(0x0, 3); } else if (od->octo_encklen == 32) { CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); } else { octeon_crypto_disable(s); dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); return -EINVAL; } CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); while (crypt_off > 0) { IOV_CONSUME(iov, data, data_i, data_l); crypt_off -= 8; } while (crypt_len > 0) { pdata = data; CVMX_MT_AES_DEC_CBC0(*data); IOV_CONSUME(iov, data, data_i, data_l); CVMX_MT_AES_DEC_CBC1(*data); CVMX_MF_AES_RESULT(*pdata, 0); CVMX_MF_AES_RESULT(*data, 1); IOV_CONSUME(iov, data, data_i, data_l); crypt_len -= 16; } octeon_crypto_disable(s); return 0; } /****************************************************************************/ /* MD5 */ int octo_null_md5_encrypt( struct octo_sess *od, struct iovec *iov, size_t iovcnt, size_t iovlen, int auth_off, int auth_len, int crypt_off, int crypt_len, int icv_off, uint8_t *ivp) { register int next = 0; uint64_t *data; uint64_t tmp1, tmp2; int data_i, data_l, alen = auth_len; register_t s; dprintf("%s()\n", __func__); if (__predict_false(od == NULL || iov==NULL || iovlen==0 || (auth_off & 0x7) || (auth_off + auth_len > iovlen))) { dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); return -EINVAL; } IOV_INIT(iov, data, data_i, data_l); s = octeon_crypto_enable(); /* Load MD5 IV */ CVMX_MT_HSH_IV(od->octo_hminner[0], 0); CVMX_MT_HSH_IV(od->octo_hminner[1], 1); while (auth_off > 0) { IOV_CONSUME(iov, data, data_i, data_l); auth_off -= 8; } while (auth_len > 0) { CVM_LOAD_MD5_UNIT(*data, next); auth_len -= 8; IOV_CONSUME(iov, data, data_i, data_l); } /* finish the hash */ CVMX_PREFETCH0(od->octo_hmouter); #if 0 if (__predict_false(inplen)) { uint64_t tmp = 0; uint8_t *p = (uint8_t *) & tmp; p[inplen] = 0x80; do { inplen--; p[inplen] = ((uint8_t *) data)[inplen]; } while (inplen); CVM_LOAD_MD5_UNIT(tmp, next); } else { CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); } #else CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); #endif /* Finish Inner hash */ while (next != 7) { CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next); } CVMX_ES64(tmp1, ((alen + 64) << 3)); CVM_LOAD_MD5_UNIT(tmp1, next); /* Get the inner hash of HMAC */ CVMX_MF_HSH_IV(tmp1, 0); CVMX_MF_HSH_IV(tmp2, 1); /* Initialize hash unit */ CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); CVMX_MT_HSH_DAT(tmp1, 0); CVMX_MT_HSH_DAT(tmp2, 1); CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2); CVMX_MT_HSH_DATZ(3); CVMX_MT_HSH_DATZ(4); CVMX_MT_HSH_DATZ(5); CVMX_MT_HSH_DATZ(6); CVMX_ES64(tmp1, ((64 + 16) << 3)); CVMX_MT_HSH_STARTMD5(tmp1); /* save the HMAC */ IOV_INIT(iov, data, data_i, data_l); while (icv_off > 0) { IOV_CONSUME(iov, data, data_i, data_l); icv_off -= 8; } CVMX_MF_HSH_IV(*data, 0); IOV_CONSUME(iov, data, data_i, data_l); CVMX_MF_HSH_IV(tmp1, 1); *(uint32_t *)data = (uint32_t) (tmp1 >> 32); octeon_crypto_disable(s); return 0; } /****************************************************************************/ /* SHA1 */ int octo_null_sha1_encrypt( struct octo_sess *od, struct iovec *iov, size_t iovcnt, size_t iovlen, int auth_off, int auth_len, int crypt_off, int crypt_len, int icv_off, uint8_t *ivp) { register int next = 0; uint64_t *data; uint64_t tmp1, tmp2, tmp3; int data_i, data_l, alen = auth_len; register_t s; dprintf("%s()\n", __func__); if (__predict_false(od == NULL || iov==NULL || iovlen==0 || (auth_off & 0x7) || (auth_off + auth_len > iovlen))) { dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); return -EINVAL; } IOV_INIT(iov, data, data_i, data_l); s = octeon_crypto_enable(); /* Load SHA1 IV */ CVMX_MT_HSH_IV(od->octo_hminner[0], 0); CVMX_MT_HSH_IV(od->octo_hminner[1], 1); CVMX_MT_HSH_IV(od->octo_hminner[2], 2); while (auth_off > 0) { IOV_CONSUME(iov, data, data_i, data_l); auth_off -= 8; } while (auth_len > 0) { CVM_LOAD_SHA_UNIT(*data, next); auth_len -= 8; IOV_CONSUME(iov, data, data_i, data_l); } /* finish the hash */ CVMX_PREFETCH0(od->octo_hmouter); #if 0 if (__predict_false(inplen)) { uint64_t tmp = 0; uint8_t *p = (uint8_t *) & tmp; p[inplen] = 0x80; do { inplen--; p[inplen] = ((uint8_t *) data)[inplen]; } while (inplen); CVM_LOAD_MD5_UNIT(tmp, next); } else { CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); } #else CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); #endif /* Finish Inner hash */ while (next != 7) { CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next); } CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next); /* Get the inner hash of HMAC */ CVMX_MF_HSH_IV(tmp1, 0); CVMX_MF_HSH_IV(tmp2, 1); tmp3 = 0; CVMX_MF_HSH_IV(tmp3, 2); /* Initialize hash unit */ CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); CVMX_MT_HSH_IV(od->octo_hmouter[2], 2); CVMX_MT_HSH_DAT(tmp1, 0); CVMX_MT_HSH_DAT(tmp2, 1); tmp3 |= 0x0000000080000000; CVMX_MT_HSH_DAT(tmp3, 2); CVMX_MT_HSH_DATZ(3); CVMX_MT_HSH_DATZ(4); CVMX_MT_HSH_DATZ(5); CVMX_MT_HSH_DATZ(6); CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3)); /* save the HMAC */ IOV_INIT(iov, data, data_i, data_l); while (icv_off > 0) { IOV_CONSUME(iov, data, data_i, data_l); icv_off -= 8; } CVMX_MF_HSH_IV(*data, 0); IOV_CONSUME(iov, data, data_i, data_l); CVMX_MF_HSH_IV(tmp1, 1); *(uint32_t *)data = (uint32_t) (tmp1 >> 32); octeon_crypto_disable(s); return 0; } /****************************************************************************/ /* DES MD5 */ int octo_des_cbc_md5_encrypt( struct octo_sess *od, struct iovec *iov, size_t iovcnt, size_t iovlen, int auth_off, int auth_len, int crypt_off, int crypt_len, int icv_off, uint8_t *ivp) { register int next = 0; union { uint32_t data32[2]; uint64_t data64[1]; } mydata; uint64_t *data = &mydata.data64[0]; uint32_t *data32; uint64_t tmp1, tmp2; int data_i, data_l, alen = auth_len; register_t s; dprintf("%s()\n", __func__); if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || (crypt_len & 0x7) || (auth_len & 0x7) || (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); return -EINVAL; } IOV_INIT(iov, data32, data_i, data_l); CVMX_PREFETCH0(ivp); CVMX_PREFETCH0(od->octo_enckey); s = octeon_crypto_enable(); /* load 3DES Key */ CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); if (od->octo_encklen == 24) { CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); } else if (od->octo_encklen == 8) { CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); } else { octeon_crypto_disable(s); dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); return -EINVAL; } CVMX_MT_3DES_IV(* (uint64_t *) ivp); /* Load MD5 IV */ CVMX_MT_HSH_IV(od->octo_hminner[0], 0); CVMX_MT_HSH_IV(od->octo_hminner[1], 1); while (crypt_off > 0 && auth_off > 0) { IOV_CONSUME(iov, data32, data_i, data_l); crypt_off -= 4; auth_off -= 4; } while (crypt_len > 0 || auth_len > 0) { uint32_t *first = data32; mydata.data32[0] = *first; IOV_CONSUME(iov, data32, data_i, data_l); mydata.data32[1] = *data32; if (crypt_off <= 0) { if (crypt_len > 0) { CVMX_MT_3DES_ENC_CBC(*data); CVMX_MF_3DES_RESULT(*data); crypt_len -= 8; } } else crypt_off -= 8; if (auth_off <= 0) { if (auth_len > 0) { CVM_LOAD_MD5_UNIT(*data, next); auth_len -= 8; } } else auth_off -= 8; *first = mydata.data32[0]; *data32 = mydata.data32[1]; IOV_CONSUME(iov, data32, data_i, data_l); } /* finish the hash */ CVMX_PREFETCH0(od->octo_hmouter); #if 0 if (__predict_false(inplen)) { uint64_t tmp = 0; uint8_t *p = (uint8_t *) & tmp; p[inplen] = 0x80; do { inplen--; p[inplen] = ((uint8_t *) data)[inplen]; } while (inplen); CVM_LOAD_MD5_UNIT(tmp, next); } else { CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); } #else CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); #endif /* Finish Inner hash */ while (next != 7) { CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next); } CVMX_ES64(tmp1, ((alen + 64) << 3)); CVM_LOAD_MD5_UNIT(tmp1, next); /* Get the inner hash of HMAC */ CVMX_MF_HSH_IV(tmp1, 0); CVMX_MF_HSH_IV(tmp2, 1); /* Initialize hash unit */ CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); CVMX_MT_HSH_DAT(tmp1, 0); CVMX_MT_HSH_DAT(tmp2, 1); CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2); CVMX_MT_HSH_DATZ(3); CVMX_MT_HSH_DATZ(4); CVMX_MT_HSH_DATZ(5); CVMX_MT_HSH_DATZ(6); CVMX_ES64(tmp1, ((64 + 16) << 3)); CVMX_MT_HSH_STARTMD5(tmp1); /* save the HMAC */ IOV_INIT(iov, data32, data_i, data_l); while (icv_off > 0) { IOV_CONSUME(iov, data32, data_i, data_l); icv_off -= 4; } CVMX_MF_HSH_IV(tmp1, 0); *data32 = (uint32_t) (tmp1 >> 32); IOV_CONSUME(iov, data32, data_i, data_l); *data32 = (uint32_t) tmp1; IOV_CONSUME(iov, data32, data_i, data_l); CVMX_MF_HSH_IV(tmp1, 1); *data32 = (uint32_t) (tmp1 >> 32); octeon_crypto_disable(s); return 0; } int octo_des_cbc_md5_decrypt( struct octo_sess *od, struct iovec *iov, size_t iovcnt, size_t iovlen, int auth_off, int auth_len, int crypt_off, int crypt_len, int icv_off, uint8_t *ivp) { register int next = 0; union { uint32_t data32[2]; uint64_t data64[1]; } mydata; uint64_t *data = &mydata.data64[0]; uint32_t *data32; uint64_t tmp1, tmp2; int data_i, data_l, alen = auth_len; register_t s; dprintf("%s()\n", __func__); if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || (crypt_len & 0x7) || (auth_len & 0x7) || (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); return -EINVAL; } IOV_INIT(iov, data32, data_i, data_l); CVMX_PREFETCH0(ivp); CVMX_PREFETCH0(od->octo_enckey); s = octeon_crypto_enable(); /* load 3DES Key */ CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); if (od->octo_encklen == 24) { CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); } else if (od->octo_encklen == 8) { CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); } else { octeon_crypto_disable(s); dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); return -EINVAL; } CVMX_MT_3DES_IV(* (uint64_t *) ivp); /* Load MD5 IV */ CVMX_MT_HSH_IV(od->octo_hminner[0], 0); CVMX_MT_HSH_IV(od->octo_hminner[1], 1); while (crypt_off > 0 && auth_off > 0) { IOV_CONSUME(iov, data32, data_i, data_l); crypt_off -= 4; auth_off -= 4; } while (crypt_len > 0 || auth_len > 0) { uint32_t *first = data32; mydata.data32[0] = *first; IOV_CONSUME(iov, data32, data_i, data_l); mydata.data32[1] = *data32; if (auth_off <= 0) { if (auth_len > 0) { CVM_LOAD_MD5_UNIT(*data, next); auth_len -= 8; } } else auth_off -= 8; if (crypt_off <= 0) { if (crypt_len > 0) { CVMX_MT_3DES_DEC_CBC(*data); CVMX_MF_3DES_RESULT(*data); crypt_len -= 8; } } else crypt_off -= 8; *first = mydata.data32[0]; *data32 = mydata.data32[1]; IOV_CONSUME(iov, data32, data_i, data_l); } /* finish the hash */ CVMX_PREFETCH0(od->octo_hmouter); #if 0 if (__predict_false(inplen)) { uint64_t tmp = 0; uint8_t *p = (uint8_t *) & tmp; p[inplen] = 0x80; do { inplen--; p[inplen] = ((uint8_t *) data)[inplen]; } while (inplen); CVM_LOAD_MD5_UNIT(tmp, next); } else { CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); } #else CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); #endif /* Finish Inner hash */ while (next != 7) { CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next); } CVMX_ES64(tmp1, ((alen + 64) << 3)); CVM_LOAD_MD5_UNIT(tmp1, next); /* Get the inner hash of HMAC */ CVMX_MF_HSH_IV(tmp1, 0); CVMX_MF_HSH_IV(tmp2, 1); /* Initialize hash unit */ CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); CVMX_MT_HSH_DAT(tmp1, 0); CVMX_MT_HSH_DAT(tmp2, 1); CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2); CVMX_MT_HSH_DATZ(3); CVMX_MT_HSH_DATZ(4); CVMX_MT_HSH_DATZ(5); CVMX_MT_HSH_DATZ(6); CVMX_ES64(tmp1, ((64 + 16) << 3)); CVMX_MT_HSH_STARTMD5(tmp1); /* save the HMAC */ IOV_INIT(iov, data32, data_i, data_l); while (icv_off > 0) { IOV_CONSUME(iov, data32, data_i, data_l); icv_off -= 4; } CVMX_MF_HSH_IV(tmp1, 0); *data32 = (uint32_t) (tmp1 >> 32); IOV_CONSUME(iov, data32, data_i, data_l); *data32 = (uint32_t) tmp1; IOV_CONSUME(iov, data32, data_i, data_l); CVMX_MF_HSH_IV(tmp1, 1); *data32 = (uint32_t) (tmp1 >> 32); octeon_crypto_disable(s); return 0; } /****************************************************************************/ /* DES SHA */ int octo_des_cbc_sha1_encrypt( struct octo_sess *od, struct iovec *iov, size_t iovcnt, size_t iovlen, int auth_off, int auth_len, int crypt_off, int crypt_len, int icv_off, uint8_t *ivp) { register int next = 0; union { uint32_t data32[2]; uint64_t data64[1]; } mydata; uint64_t *data = &mydata.data64[0]; uint32_t *data32; uint64_t tmp1, tmp2, tmp3; int data_i, data_l, alen = auth_len; register_t s; dprintf("%s()\n", __func__); if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || (crypt_len & 0x7) || (auth_len & 0x7) || (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); return -EINVAL; } IOV_INIT(iov, data32, data_i, data_l); CVMX_PREFETCH0(ivp); CVMX_PREFETCH0(od->octo_enckey); s = octeon_crypto_enable(); /* load 3DES Key */ CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); if (od->octo_encklen == 24) { CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); } else if (od->octo_encklen == 8) { CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); } else { octeon_crypto_disable(s); dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); return -EINVAL; } CVMX_MT_3DES_IV(* (uint64_t *) ivp); /* Load SHA1 IV */ CVMX_MT_HSH_IV(od->octo_hminner[0], 0); CVMX_MT_HSH_IV(od->octo_hminner[1], 1); CVMX_MT_HSH_IV(od->octo_hminner[2], 2); while (crypt_off > 0 && auth_off > 0) { IOV_CONSUME(iov, data32, data_i, data_l); crypt_off -= 4; auth_off -= 4; } while (crypt_len > 0 || auth_len > 0) { uint32_t *first = data32; mydata.data32[0] = *first; IOV_CONSUME(iov, data32, data_i, data_l); mydata.data32[1] = *data32; if (crypt_off <= 0) { if (crypt_len > 0) { CVMX_MT_3DES_ENC_CBC(*data); CVMX_MF_3DES_RESULT(*data); crypt_len -= 8; } } else crypt_off -= 8; if (auth_off <= 0) { if (auth_len > 0) { CVM_LOAD_SHA_UNIT(*data, next); auth_len -= 8; } } else auth_off -= 8; *first = mydata.data32[0]; *data32 = mydata.data32[1]; IOV_CONSUME(iov, data32, data_i, data_l); } /* finish the hash */ CVMX_PREFETCH0(od->octo_hmouter); #if 0 if (__predict_false(inplen)) { uint64_t tmp = 0; uint8_t *p = (uint8_t *) & tmp; p[inplen] = 0x80; do { inplen--; p[inplen] = ((uint8_t *) data)[inplen]; } while (inplen); CVM_LOAD_SHA_UNIT(tmp, next); } else { CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); } #else CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); #endif /* Finish Inner hash */ while (next != 7) { CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next); } CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next); /* Get the inner hash of HMAC */ CVMX_MF_HSH_IV(tmp1, 0); CVMX_MF_HSH_IV(tmp2, 1); tmp3 = 0; CVMX_MF_HSH_IV(tmp3, 2); /* Initialize hash unit */ CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); CVMX_MT_HSH_IV(od->octo_hmouter[2], 2); CVMX_MT_HSH_DAT(tmp1, 0); CVMX_MT_HSH_DAT(tmp2, 1); tmp3 |= 0x0000000080000000; CVMX_MT_HSH_DAT(tmp3, 2); CVMX_MT_HSH_DATZ(3); CVMX_MT_HSH_DATZ(4); CVMX_MT_HSH_DATZ(5); CVMX_MT_HSH_DATZ(6); CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3)); /* save the HMAC */ IOV_INIT(iov, data32, data_i, data_l); while (icv_off > 0) { IOV_CONSUME(iov, data32, data_i, data_l); icv_off -= 4; } CVMX_MF_HSH_IV(tmp1, 0); *data32 = (uint32_t) (tmp1 >> 32); IOV_CONSUME(iov, data32, data_i, data_l); *data32 = (uint32_t) tmp1; IOV_CONSUME(iov, data32, data_i, data_l); CVMX_MF_HSH_IV(tmp1, 1); *data32 = (uint32_t) (tmp1 >> 32); octeon_crypto_disable(s); return 0; } int octo_des_cbc_sha1_decrypt( struct octo_sess *od, struct iovec *iov, size_t iovcnt, size_t iovlen, int auth_off, int auth_len, int crypt_off, int crypt_len, int icv_off, uint8_t *ivp) { register int next = 0; union { uint32_t data32[2]; uint64_t data64[1]; } mydata; uint64_t *data = &mydata.data64[0]; uint32_t *data32; uint64_t tmp1, tmp2, tmp3; int data_i, data_l, alen = auth_len; register_t s; dprintf("%s()\n", __func__); if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || (crypt_len & 0x7) || (auth_len & 0x7) || (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); return -EINVAL; } IOV_INIT(iov, data32, data_i, data_l); CVMX_PREFETCH0(ivp); CVMX_PREFETCH0(od->octo_enckey); s = octeon_crypto_enable(); /* load 3DES Key */ CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0); if (od->octo_encklen == 24) { CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1); CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2); } else if (od->octo_encklen == 8) { CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1); CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2); } else { octeon_crypto_disable(s); dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); return -EINVAL; } CVMX_MT_3DES_IV(* (uint64_t *) ivp); /* Load SHA1 IV */ CVMX_MT_HSH_IV(od->octo_hminner[0], 0); CVMX_MT_HSH_IV(od->octo_hminner[1], 1); CVMX_MT_HSH_IV(od->octo_hminner[2], 2); while (crypt_off > 0 && auth_off > 0) { IOV_CONSUME(iov, data32, data_i, data_l); crypt_off -= 4; auth_off -= 4; } while (crypt_len > 0 || auth_len > 0) { uint32_t *first = data32; mydata.data32[0] = *first; IOV_CONSUME(iov, data32, data_i, data_l); mydata.data32[1] = *data32; if (auth_off <= 0) { if (auth_len > 0) { CVM_LOAD_SHA_UNIT(*data, next); auth_len -= 8; } } else auth_off -= 8; if (crypt_off <= 0) { if (crypt_len > 0) { CVMX_MT_3DES_DEC_CBC(*data); CVMX_MF_3DES_RESULT(*data); crypt_len -= 8; } } else crypt_off -= 8; *first = mydata.data32[0]; *data32 = mydata.data32[1]; IOV_CONSUME(iov, data32, data_i, data_l); } /* finish the hash */ CVMX_PREFETCH0(od->octo_hmouter); #if 0 if (__predict_false(inplen)) { uint64_t tmp = 0; uint8_t *p = (uint8_t *) & tmp; p[inplen] = 0x80; do { inplen--; p[inplen] = ((uint8_t *) data)[inplen]; } while (inplen); CVM_LOAD_SHA_UNIT(tmp, next); } else { CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); } #else CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); #endif /* Finish Inner hash */ while (next != 7) { CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next); } CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next); /* Get the inner hash of HMAC */ CVMX_MF_HSH_IV(tmp1, 0); CVMX_MF_HSH_IV(tmp2, 1); tmp3 = 0; CVMX_MF_HSH_IV(tmp3, 2); /* Initialize hash unit */ CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); CVMX_MT_HSH_IV(od->octo_hmouter[2], 2); CVMX_MT_HSH_DAT(tmp1, 0); CVMX_MT_HSH_DAT(tmp2, 1); tmp3 |= 0x0000000080000000; CVMX_MT_HSH_DAT(tmp3, 2); CVMX_MT_HSH_DATZ(3); CVMX_MT_HSH_DATZ(4); CVMX_MT_HSH_DATZ(5); CVMX_MT_HSH_DATZ(6); CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3)); /* save the HMAC */ IOV_INIT(iov, data32, data_i, data_l); while (icv_off > 0) { IOV_CONSUME(iov, data32, data_i, data_l); icv_off -= 4; } CVMX_MF_HSH_IV(tmp1, 0); *data32 = (uint32_t) (tmp1 >> 32); IOV_CONSUME(iov, data32, data_i, data_l); *data32 = (uint32_t) tmp1; IOV_CONSUME(iov, data32, data_i, data_l); CVMX_MF_HSH_IV(tmp1, 1); *data32 = (uint32_t) (tmp1 >> 32); octeon_crypto_disable(s); return 0; } /****************************************************************************/ /* AES MD5 */ int octo_aes_cbc_md5_encrypt( struct octo_sess *od, struct iovec *iov, size_t iovcnt, size_t iovlen, int auth_off, int auth_len, int crypt_off, int crypt_len, int icv_off, uint8_t *ivp) { register int next = 0; union { uint32_t data32[2]; uint64_t data64[1]; } mydata[2]; uint64_t *pdata = &mydata[0].data64[0]; uint64_t *data = &mydata[1].data64[0]; uint32_t *data32; uint64_t tmp1, tmp2; int data_i, data_l, alen = auth_len; register_t s; dprintf("%s()\n", __func__); if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || (crypt_len & 0x7) || (auth_len & 0x7) || (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); return -EINVAL; } IOV_INIT(iov, data32, data_i, data_l); CVMX_PREFETCH0(ivp); CVMX_PREFETCH0(od->octo_enckey); s = octeon_crypto_enable(); /* load AES Key */ CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); if (od->octo_encklen == 16) { CVMX_MT_AES_KEY(0x0, 2); CVMX_MT_AES_KEY(0x0, 3); } else if (od->octo_encklen == 24) { CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); CVMX_MT_AES_KEY(0x0, 3); } else if (od->octo_encklen == 32) { CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); } else { octeon_crypto_disable(s); dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); return -EINVAL; } CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); /* Load MD5 IV */ CVMX_MT_HSH_IV(od->octo_hminner[0], 0); CVMX_MT_HSH_IV(od->octo_hminner[1], 1); while (crypt_off > 0 && auth_off > 0) { IOV_CONSUME(iov, data32, data_i, data_l); crypt_off -= 4; auth_off -= 4; } while (crypt_len > 0 || auth_len > 0) { uint32_t *pdata32[3]; pdata32[0] = data32; mydata[0].data32[0] = *data32; IOV_CONSUME(iov, data32, data_i, data_l); pdata32[1] = data32; mydata[0].data32[1] = *data32; IOV_CONSUME(iov, data32, data_i, data_l); pdata32[2] = data32; mydata[1].data32[0] = *data32; IOV_CONSUME(iov, data32, data_i, data_l); mydata[1].data32[1] = *data32; if (crypt_off <= 0) { if (crypt_len > 0) { CVMX_MT_AES_ENC_CBC0(*pdata); CVMX_MT_AES_ENC_CBC1(*data); CVMX_MF_AES_RESULT(*pdata, 0); CVMX_MF_AES_RESULT(*data, 1); crypt_len -= 16; } } else crypt_off -= 16; if (auth_off <= 0) { if (auth_len > 0) { CVM_LOAD_MD5_UNIT(*pdata, next); CVM_LOAD_MD5_UNIT(*data, next); auth_len -= 16; } } else auth_off -= 16; *pdata32[0] = mydata[0].data32[0]; *pdata32[1] = mydata[0].data32[1]; *pdata32[2] = mydata[1].data32[0]; *data32 = mydata[1].data32[1]; IOV_CONSUME(iov, data32, data_i, data_l); } /* finish the hash */ CVMX_PREFETCH0(od->octo_hmouter); #if 0 if (__predict_false(inplen)) { uint64_t tmp = 0; uint8_t *p = (uint8_t *) & tmp; p[inplen] = 0x80; do { inplen--; p[inplen] = ((uint8_t *) data)[inplen]; } while (inplen); CVM_LOAD_MD5_UNIT(tmp, next); } else { CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); } #else CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); #endif /* Finish Inner hash */ while (next != 7) { CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next); } CVMX_ES64(tmp1, ((alen + 64) << 3)); CVM_LOAD_MD5_UNIT(tmp1, next); /* Get the inner hash of HMAC */ CVMX_MF_HSH_IV(tmp1, 0); CVMX_MF_HSH_IV(tmp2, 1); /* Initialize hash unit */ CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); CVMX_MT_HSH_DAT(tmp1, 0); CVMX_MT_HSH_DAT(tmp2, 1); CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2); CVMX_MT_HSH_DATZ(3); CVMX_MT_HSH_DATZ(4); CVMX_MT_HSH_DATZ(5); CVMX_MT_HSH_DATZ(6); CVMX_ES64(tmp1, ((64 + 16) << 3)); CVMX_MT_HSH_STARTMD5(tmp1); /* save the HMAC */ IOV_INIT(iov, data32, data_i, data_l); while (icv_off > 0) { IOV_CONSUME(iov, data32, data_i, data_l); icv_off -= 4; } CVMX_MF_HSH_IV(tmp1, 0); *data32 = (uint32_t) (tmp1 >> 32); IOV_CONSUME(iov, data32, data_i, data_l); *data32 = (uint32_t) tmp1; IOV_CONSUME(iov, data32, data_i, data_l); CVMX_MF_HSH_IV(tmp1, 1); *data32 = (uint32_t) (tmp1 >> 32); octeon_crypto_disable(s); return 0; } int octo_aes_cbc_md5_decrypt( struct octo_sess *od, struct iovec *iov, size_t iovcnt, size_t iovlen, int auth_off, int auth_len, int crypt_off, int crypt_len, int icv_off, uint8_t *ivp) { register int next = 0; union { uint32_t data32[2]; uint64_t data64[1]; } mydata[2]; uint64_t *pdata = &mydata[0].data64[0]; uint64_t *data = &mydata[1].data64[0]; uint32_t *data32; uint64_t tmp1, tmp2; int data_i, data_l, alen = auth_len; register_t s; dprintf("%s()\n", __func__); if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || (crypt_len & 0x7) || (auth_len & 0x7) || (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); return -EINVAL; } IOV_INIT(iov, data32, data_i, data_l); CVMX_PREFETCH0(ivp); CVMX_PREFETCH0(od->octo_enckey); s = octeon_crypto_enable(); /* load AES Key */ CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); if (od->octo_encklen == 16) { CVMX_MT_AES_KEY(0x0, 2); CVMX_MT_AES_KEY(0x0, 3); } else if (od->octo_encklen == 24) { CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); CVMX_MT_AES_KEY(0x0, 3); } else if (od->octo_encklen == 32) { CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); } else { octeon_crypto_disable(s); dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); return -EINVAL; } CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); /* Load MD5 IV */ CVMX_MT_HSH_IV(od->octo_hminner[0], 0); CVMX_MT_HSH_IV(od->octo_hminner[1], 1); while (crypt_off > 0 && auth_off > 0) { IOV_CONSUME(iov, data32, data_i, data_l); crypt_off -= 4; auth_off -= 4; } while (crypt_len > 0 || auth_len > 0) { uint32_t *pdata32[3]; pdata32[0] = data32; mydata[0].data32[0] = *data32; IOV_CONSUME(iov, data32, data_i, data_l); pdata32[1] = data32; mydata[0].data32[1] = *data32; IOV_CONSUME(iov, data32, data_i, data_l); pdata32[2] = data32; mydata[1].data32[0] = *data32; IOV_CONSUME(iov, data32, data_i, data_l); mydata[1].data32[1] = *data32; if (auth_off <= 0) { if (auth_len > 0) { CVM_LOAD_MD5_UNIT(*pdata, next); CVM_LOAD_MD5_UNIT(*data, next); auth_len -= 16; } } else auth_off -= 16; if (crypt_off <= 0) { if (crypt_len > 0) { CVMX_MT_AES_DEC_CBC0(*pdata); CVMX_MT_AES_DEC_CBC1(*data); CVMX_MF_AES_RESULT(*pdata, 0); CVMX_MF_AES_RESULT(*data, 1); crypt_len -= 16; } } else crypt_off -= 16; *pdata32[0] = mydata[0].data32[0]; *pdata32[1] = mydata[0].data32[1]; *pdata32[2] = mydata[1].data32[0]; *data32 = mydata[1].data32[1]; IOV_CONSUME(iov, data32, data_i, data_l); } /* finish the hash */ CVMX_PREFETCH0(od->octo_hmouter); #if 0 if (__predict_false(inplen)) { uint64_t tmp = 0; uint8_t *p = (uint8_t *) & tmp; p[inplen] = 0x80; do { inplen--; p[inplen] = ((uint8_t *) data)[inplen]; } while (inplen); CVM_LOAD_MD5_UNIT(tmp, next); } else { CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); } #else CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); #endif /* Finish Inner hash */ while (next != 7) { CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next); } CVMX_ES64(tmp1, ((alen + 64) << 3)); CVM_LOAD_MD5_UNIT(tmp1, next); /* Get the inner hash of HMAC */ CVMX_MF_HSH_IV(tmp1, 0); CVMX_MF_HSH_IV(tmp2, 1); /* Initialize hash unit */ CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); CVMX_MT_HSH_DAT(tmp1, 0); CVMX_MT_HSH_DAT(tmp2, 1); CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2); CVMX_MT_HSH_DATZ(3); CVMX_MT_HSH_DATZ(4); CVMX_MT_HSH_DATZ(5); CVMX_MT_HSH_DATZ(6); CVMX_ES64(tmp1, ((64 + 16) << 3)); CVMX_MT_HSH_STARTMD5(tmp1); /* save the HMAC */ IOV_INIT(iov, data32, data_i, data_l); while (icv_off > 0) { IOV_CONSUME(iov, data32, data_i, data_l); icv_off -= 4; } CVMX_MF_HSH_IV(tmp1, 0); *data32 = (uint32_t) (tmp1 >> 32); IOV_CONSUME(iov, data32, data_i, data_l); *data32 = (uint32_t) tmp1; IOV_CONSUME(iov, data32, data_i, data_l); CVMX_MF_HSH_IV(tmp1, 1); *data32 = (uint32_t) (tmp1 >> 32); octeon_crypto_disable(s); return 0; } /****************************************************************************/ /* AES SHA1 */ int octo_aes_cbc_sha1_encrypt( struct octo_sess *od, struct iovec *iov, size_t iovcnt, size_t iovlen, int auth_off, int auth_len, int crypt_off, int crypt_len, int icv_off, uint8_t *ivp) { register int next = 0; union { uint32_t data32[2]; uint64_t data64[1]; } mydata[2]; uint64_t *pdata = &mydata[0].data64[0]; uint64_t *data = &mydata[1].data64[0]; uint32_t *data32; uint64_t tmp1, tmp2, tmp3; int data_i, data_l, alen = auth_len; register_t s; dprintf("%s()\n", __func__); if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || (crypt_len & 0x7) || (auth_len & 0x7) || (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); return -EINVAL; } IOV_INIT(iov, data32, data_i, data_l); CVMX_PREFETCH0(ivp); CVMX_PREFETCH0(od->octo_enckey); s = octeon_crypto_enable(); /* load AES Key */ CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); if (od->octo_encklen == 16) { CVMX_MT_AES_KEY(0x0, 2); CVMX_MT_AES_KEY(0x0, 3); } else if (od->octo_encklen == 24) { CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); CVMX_MT_AES_KEY(0x0, 3); } else if (od->octo_encklen == 32) { CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); } else { octeon_crypto_disable(s); dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); return -EINVAL; } CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); /* Load SHA IV */ CVMX_MT_HSH_IV(od->octo_hminner[0], 0); CVMX_MT_HSH_IV(od->octo_hminner[1], 1); CVMX_MT_HSH_IV(od->octo_hminner[2], 2); while (crypt_off > 0 && auth_off > 0) { IOV_CONSUME(iov, data32, data_i, data_l); crypt_off -= 4; auth_off -= 4; } while (crypt_len > 0 || auth_len > 0) { uint32_t *pdata32[3]; pdata32[0] = data32; mydata[0].data32[0] = *data32; IOV_CONSUME(iov, data32, data_i, data_l); pdata32[1] = data32; mydata[0].data32[1] = *data32; IOV_CONSUME(iov, data32, data_i, data_l); pdata32[2] = data32; mydata[1].data32[0] = *data32; IOV_CONSUME(iov, data32, data_i, data_l); mydata[1].data32[1] = *data32; if (crypt_off <= 0) { if (crypt_len > 0) { CVMX_MT_AES_ENC_CBC0(*pdata); CVMX_MT_AES_ENC_CBC1(*data); CVMX_MF_AES_RESULT(*pdata, 0); CVMX_MF_AES_RESULT(*data, 1); crypt_len -= 16; } } else crypt_off -= 16; if (auth_off <= 0) { if (auth_len > 0) { CVM_LOAD_SHA_UNIT(*pdata, next); CVM_LOAD_SHA_UNIT(*data, next); auth_len -= 16; } } else auth_off -= 16; *pdata32[0] = mydata[0].data32[0]; *pdata32[1] = mydata[0].data32[1]; *pdata32[2] = mydata[1].data32[0]; *data32 = mydata[1].data32[1]; IOV_CONSUME(iov, data32, data_i, data_l); } /* finish the hash */ CVMX_PREFETCH0(od->octo_hmouter); #if 0 if (__predict_false(inplen)) { uint64_t tmp = 0; uint8_t *p = (uint8_t *) & tmp; p[inplen] = 0x80; do { inplen--; p[inplen] = ((uint8_t *) data)[inplen]; } while (inplen); CVM_LOAD_SHA_UNIT(tmp, next); } else { CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); } #else CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); #endif /* Finish Inner hash */ while (next != 7) { CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next); } CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next); /* Get the inner hash of HMAC */ CVMX_MF_HSH_IV(tmp1, 0); CVMX_MF_HSH_IV(tmp2, 1); tmp3 = 0; CVMX_MF_HSH_IV(tmp3, 2); /* Initialize hash unit */ CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); CVMX_MT_HSH_IV(od->octo_hmouter[2], 2); CVMX_MT_HSH_DAT(tmp1, 0); CVMX_MT_HSH_DAT(tmp2, 1); tmp3 |= 0x0000000080000000; CVMX_MT_HSH_DAT(tmp3, 2); CVMX_MT_HSH_DATZ(3); CVMX_MT_HSH_DATZ(4); CVMX_MT_HSH_DATZ(5); CVMX_MT_HSH_DATZ(6); CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3)); /* finish the hash */ CVMX_PREFETCH0(od->octo_hmouter); #if 0 if (__predict_false(inplen)) { uint64_t tmp = 0; uint8_t *p = (uint8_t *) & tmp; p[inplen] = 0x80; do { inplen--; p[inplen] = ((uint8_t *) data)[inplen]; } while (inplen); CVM_LOAD_MD5_UNIT(tmp, next); } else { CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); } #else CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); #endif /* save the HMAC */ IOV_INIT(iov, data32, data_i, data_l); while (icv_off > 0) { IOV_CONSUME(iov, data32, data_i, data_l); icv_off -= 4; } CVMX_MF_HSH_IV(tmp1, 0); *data32 = (uint32_t) (tmp1 >> 32); IOV_CONSUME(iov, data32, data_i, data_l); *data32 = (uint32_t) tmp1; IOV_CONSUME(iov, data32, data_i, data_l); CVMX_MF_HSH_IV(tmp1, 1); *data32 = (uint32_t) (tmp1 >> 32); octeon_crypto_disable(s); return 0; } int octo_aes_cbc_sha1_decrypt( struct octo_sess *od, struct iovec *iov, size_t iovcnt, size_t iovlen, int auth_off, int auth_len, int crypt_off, int crypt_len, int icv_off, uint8_t *ivp) { register int next = 0; union { uint32_t data32[2]; uint64_t data64[1]; } mydata[2]; uint64_t *pdata = &mydata[0].data64[0]; uint64_t *data = &mydata[1].data64[0]; uint32_t *data32; uint64_t tmp1, tmp2, tmp3; int data_i, data_l, alen = auth_len; register_t s; dprintf("%s()\n", __func__); if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL || (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) || (crypt_len & 0x7) || (auth_len & 0x7) || (auth_off & 0x3) || (auth_off + auth_len > iovlen))) { dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd " "auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d " "icv_off=%d ivp=%p\n", __func__, od, iov, iovlen, auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp); return -EINVAL; } IOV_INIT(iov, data32, data_i, data_l); CVMX_PREFETCH0(ivp); CVMX_PREFETCH0(od->octo_enckey); s = octeon_crypto_enable(); /* load AES Key */ CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0); CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1); if (od->octo_encklen == 16) { CVMX_MT_AES_KEY(0x0, 2); CVMX_MT_AES_KEY(0x0, 3); } else if (od->octo_encklen == 24) { CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); CVMX_MT_AES_KEY(0x0, 3); } else if (od->octo_encklen == 32) { CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2); CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3); } else { octeon_crypto_disable(s); dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen); return -EINVAL; } CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1); CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0); CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1); /* Load MD5 IV */ CVMX_MT_HSH_IV(od->octo_hminner[0], 0); CVMX_MT_HSH_IV(od->octo_hminner[1], 1); CVMX_MT_HSH_IV(od->octo_hminner[2], 2); while (crypt_off > 0 && auth_off > 0) { IOV_CONSUME(iov, data32, data_i, data_l); crypt_off -= 4; auth_off -= 4; } while (crypt_len > 0 || auth_len > 0) { uint32_t *pdata32[3]; pdata32[0] = data32; mydata[0].data32[0] = *data32; IOV_CONSUME(iov, data32, data_i, data_l); pdata32[1] = data32; mydata[0].data32[1] = *data32; IOV_CONSUME(iov, data32, data_i, data_l); pdata32[2] = data32; mydata[1].data32[0] = *data32; IOV_CONSUME(iov, data32, data_i, data_l); mydata[1].data32[1] = *data32; if (auth_off <= 0) { if (auth_len > 0) { CVM_LOAD_SHA_UNIT(*pdata, next); CVM_LOAD_SHA_UNIT(*data, next); auth_len -= 16; } } else auth_off -= 16; if (crypt_off <= 0) { if (crypt_len > 0) { CVMX_MT_AES_DEC_CBC0(*pdata); CVMX_MT_AES_DEC_CBC1(*data); CVMX_MF_AES_RESULT(*pdata, 0); CVMX_MF_AES_RESULT(*data, 1); crypt_len -= 16; } } else crypt_off -= 16; *pdata32[0] = mydata[0].data32[0]; *pdata32[1] = mydata[0].data32[1]; *pdata32[2] = mydata[1].data32[0]; *data32 = mydata[1].data32[1]; IOV_CONSUME(iov, data32, data_i, data_l); } /* finish the hash */ CVMX_PREFETCH0(od->octo_hmouter); #if 0 if (__predict_false(inplen)) { uint64_t tmp = 0; uint8_t *p = (uint8_t *) & tmp; p[inplen] = 0x80; do { inplen--; p[inplen] = ((uint8_t *) data)[inplen]; } while (inplen); CVM_LOAD_SHA_UNIT(tmp, next); } else { CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); } #else CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next); #endif /* Finish Inner hash */ while (next != 7) { CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next); } CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next); /* Get the inner hash of HMAC */ CVMX_MF_HSH_IV(tmp1, 0); CVMX_MF_HSH_IV(tmp2, 1); tmp3 = 0; CVMX_MF_HSH_IV(tmp3, 2); /* Initialize hash unit */ CVMX_MT_HSH_IV(od->octo_hmouter[0], 0); CVMX_MT_HSH_IV(od->octo_hmouter[1], 1); CVMX_MT_HSH_IV(od->octo_hmouter[2], 2); CVMX_MT_HSH_DAT(tmp1, 0); CVMX_MT_HSH_DAT(tmp2, 1); tmp3 |= 0x0000000080000000; CVMX_MT_HSH_DAT(tmp3, 2); CVMX_MT_HSH_DATZ(3); CVMX_MT_HSH_DATZ(4); CVMX_MT_HSH_DATZ(5); CVMX_MT_HSH_DATZ(6); CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3)); /* finish the hash */ CVMX_PREFETCH0(od->octo_hmouter); #if 0 if (__predict_false(inplen)) { uint64_t tmp = 0; uint8_t *p = (uint8_t *) & tmp; p[inplen] = 0x80; do { inplen--; p[inplen] = ((uint8_t *) data)[inplen]; } while (inplen); CVM_LOAD_MD5_UNIT(tmp, next); } else { CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); } #else CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next); #endif /* save the HMAC */ IOV_INIT(iov, data32, data_i, data_l); while (icv_off > 0) { IOV_CONSUME(iov, data32, data_i, data_l); icv_off -= 4; } CVMX_MF_HSH_IV(tmp1, 0); *data32 = (uint32_t) (tmp1 >> 32); IOV_CONSUME(iov, data32, data_i, data_l); *data32 = (uint32_t) tmp1; IOV_CONSUME(iov, data32, data_i, data_l); CVMX_MF_HSH_IV(tmp1, 1); *data32 = (uint32_t) (tmp1 >> 32); octeon_crypto_disable(s); return 0; } /****************************************************************************/