sys/mips/nlm/dev/sec/nlmsec.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2003-2012 Broadcom Corporation
   5  * All Rights Reserved
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  *
  18  * THIS SOFTWARE IS PROVIDED BY BROADCOM ``AS IS'' AND ANY EXPRESS OR
  19  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  20  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  21  * ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM OR CONTRIBUTORS BE LIABLE
  22  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  23  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  24  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  25  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  26  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
  27  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
  28  * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29  */
  30
  31 #include <sys/cdefs.h>
  32 __FBSDID("$FreeBSD$");
  33
  34 #include <sys/cdefs.h>
  35 #include <sys/param.h>
  36 #include <sys/systm.h>
  37 #include <sys/proc.h>
  38 #include <sys/errno.h>
  39 #include <sys/malloc.h>
  40 #include <sys/kernel.h>
  41 #include <sys/module.h>
  42 #include <sys/mbuf.h>
  43 #include <sys/lock.h>
  44 #include <sys/mutex.h>
  45 #include <sys/sysctl.h>
  46 #include <sys/bus.h>
  47 #include <sys/random.h>
  48 #include <sys/rman.h>
  49 #include <sys/uio.h>
  50 #include <sys/kobj.h>
  51
  52 #include <dev/pci/pcivar.h>
  53
  54 #include <opencrypto/cryptodev.h>
  55
  56 #include "cryptodev_if.h"
  57
  58 #include <vm/vm.h>
  59 #include <vm/pmap.h>
  60
  61 #include <mips/nlm/hal/haldefs.h>
  62 #include <mips/nlm/hal/iomap.h>
  63 #include <mips/nlm/xlp.h>
  64 #include <mips/nlm/hal/sys.h>
  65 #include <mips/nlm/hal/fmn.h>
  66 #include <mips/nlm/hal/nlmsaelib.h>
  67 #include <mips/nlm/dev/sec/nlmseclib.h>
  68 #include <mips/nlm/hal/cop2.h>
  69 #include <mips/nlm/hal/mips-extns.h>
  70 #include <mips/nlm/msgring.h>
  71
  72 unsigned int creditleft;
  73
  74 void xlp_sec_print_data(struct cryptop *crp);
  75
  76 static  int xlp_sec_init(struct xlp_sec_softc *sc);
  77 static  int xlp_sec_newsession(device_t , uint32_t *, struct cryptoini *);
  78 static  int xlp_sec_freesession(device_t , uint64_t);
  79 static  int xlp_sec_process(device_t , struct cryptop *, int);
  80 static  int xlp_copyiv(struct xlp_sec_softc *, struct xlp_sec_command *,
  81     struct cryptodesc *enccrd);
  82 static int xlp_get_nsegs(struct cryptop *, unsigned int *);
  83 static int xlp_alloc_cmd_params(struct xlp_sec_command *, unsigned int);
  84 static void  xlp_free_cmd_params(struct xlp_sec_command *);
  85
  86 static  int xlp_sec_probe(device_t);
  87 static  int xlp_sec_attach(device_t);
  88 static  int xlp_sec_detach(device_t);
  89
  90 static device_method_t xlp_sec_methods[] = {
  91         /* device interface */
  92         DEVMETHOD(device_probe, xlp_sec_probe),
  93         DEVMETHOD(device_attach, xlp_sec_attach),
  94         DEVMETHOD(device_detach, xlp_sec_detach),
  95
  96         /* bus interface */
  97         DEVMETHOD(bus_print_child, bus_generic_print_child),
  98         DEVMETHOD(bus_driver_added, bus_generic_driver_added),
  99
 100         /* crypto device methods */
 101         DEVMETHOD(cryptodev_newsession, xlp_sec_newsession),
 102         DEVMETHOD(cryptodev_freesession,xlp_sec_freesession),
 103         DEVMETHOD(cryptodev_process,    xlp_sec_process),
 104
 105         DEVMETHOD_END
 106 };
 107
 108 static driver_t xlp_sec_driver = {
 109         "nlmsec",
 110         xlp_sec_methods,
 111         sizeof(struct xlp_sec_softc)
 112 };
 113 static devclass_t xlp_sec_devclass;
 114
 115 DRIVER_MODULE(nlmsec, pci, xlp_sec_driver, xlp_sec_devclass, 0, 0);
 116 MODULE_DEPEND(nlmsec, crypto, 1, 1, 1);
 117
 118 void
 119 nlm_xlpsec_msgring_handler(int vc, int size, int code, int src_id,
 120     struct nlm_fmn_msg *msg, void *data);
 121
 122 #ifdef NLM_SEC_DEBUG
 123
 124 #define extract_bits(x, bitshift, bitcnt)                               \
 125     (((unsigned long long)x >> bitshift) & ((1ULL << bitcnt) - 1))
 126
 127 void
 128 print_crypto_params(struct xlp_sec_command *cmd, struct nlm_fmn_msg m)
 129 {
 130         unsigned long long msg0,msg1,msg2,msg3,msg4,msg5,msg6,msg7,msg8;
 131
 132         msg0 = cmd->ctrlp->desc0;
 133         msg1 = cmd->paramp->desc0;
 134         msg2 = cmd->paramp->desc1;
 135         msg3 = cmd->paramp->desc2;
 136         msg4 = cmd->paramp->desc3;
 137         msg5 = cmd->paramp->segment[0][0];
 138         msg6 = cmd->paramp->segment[0][1];
 139         msg7 = m.msg[0];
 140         msg8 = m.msg[1];
 141
 142         printf("msg0 %llx msg1 %llx msg2 %llx msg3 %llx msg4 %llx msg5 %llx"
 143             "msg6 %llx msg7 %llx msg8 %llx\n", msg0, msg1, msg2, msg3, msg4,
 144             msg5, msg6, msg7, msg8);
 145
 146         printf("c0: hmac %d htype %d hmode %d ctype %d cmode %d arc4 %x\n",
 147             (unsigned int)extract_bits(msg0, 61, 1),
 148             (unsigned int)extract_bits(msg0, 52, 8),
 149             (unsigned int)extract_bits(msg0, 43, 8),
 150             (unsigned int)extract_bits(msg0, 34, 8),
 151             (unsigned int)extract_bits(msg0, 25, 8),
 152             (unsigned int)extract_bits(msg0, 0, 23));
 153
 154         printf("p0: tls %d hsrc %d hl3 %d enc %d ivl %d hd %llx\n",
 155             (unsigned int)extract_bits(msg1, 63, 1),
 156             (unsigned int)extract_bits(msg1,62,1),
 157             (unsigned int)extract_bits(msg1,60,1),
 158             (unsigned int)extract_bits(msg1,59,1),
 159             (unsigned int)extract_bits(msg1,41,16), extract_bits(msg1,0,40));
 160
 161         printf("p1: clen %u hl %u\n",  (unsigned int)extract_bits(msg2, 32, 32),
 162             (unsigned int)extract_bits(msg2,0,32));
 163
 164         printf("p2: ivoff %d cbit %d coff %d hbit %d hclb %d hoff %d\n",
 165             (unsigned int)extract_bits(msg3, 45, 17),
 166             (unsigned int)extract_bits(msg3, 42,3),
 167             (unsigned int)extract_bits(msg3, 22,16),
 168             (unsigned int)extract_bits(msg3, 19,3),
 169             (unsigned int)extract_bits(msg3, 18,1),
 170             (unsigned int)extract_bits(msg3, 0, 16));
 171
 172         printf("p3: desfbid %d tlen %d arc4 %x hmacpad %d\n",
 173             (unsigned int)extract_bits(msg4, 48,16),
 174             (unsigned int)extract_bits(msg4,11,16),
 175             (unsigned int)extract_bits(msg4,6,3),
 176             (unsigned int)extract_bits(msg4,5,1));
 177
 178         printf("p4: sflen %d sddr %llx \n",
 179             (unsigned int)extract_bits(msg5, 48, 16),extract_bits(msg5, 0, 40));
 180
 181         printf("p5: dflen %d cl3 %d cclob %d cdest %llx \n",
 182             (unsigned int)extract_bits(msg6, 48, 16),
 183             (unsigned int)extract_bits(msg6, 46, 1),
 184             (unsigned int)extract_bits(msg6, 41, 1), extract_bits(msg6, 0, 40));
 185
 186         printf("fmn0: fbid %d dfrlen %d dfrv %d cklen %d cdescaddr %llx\n",
 187             (unsigned int)extract_bits(msg7, 48, 16),
 188             (unsigned int)extract_bits(msg7,46,2),
 189             (unsigned int)extract_bits(msg7,45,1),
 190             (unsigned int)extract_bits(msg7,40,5),
 191             (extract_bits(msg7,0,34)<< 6));
 192
 193         printf("fmn1: arc4 %d hklen %d pdesclen %d pktdescad %llx\n",
 194             (unsigned int)extract_bits(msg8, 63, 1),
 195             (unsigned int)extract_bits(msg8,56,5),
 196             (unsigned int)extract_bits(msg8,43,12),
 197             (extract_bits(msg8,0,34) << 6));
 198
 199         return;
 200 }
 201
 202 void
 203 xlp_sec_print_data(struct cryptop *crp)
 204 {
 205         int i, key_len;
 206         struct cryptodesc *crp_desc;
 207
 208         printf("session id = 0x%llx, crp_ilen = %d, crp_olen=%d \n",
 209             crp->crp_sid, crp->crp_ilen, crp->crp_olen);
 210
 211         printf("crp_flags = 0x%x\n", crp->crp_flags);
 212
 213         printf("crp buf:\n");
 214         for (i = 0; i < crp->crp_ilen; i++) {
 215                 printf("%c  ", crp->crp_buf[i]);
 216                 if (i % 10 == 0)
 217                         printf("\n");
 218         }
 219
 220         printf("\n");
 221         printf("****************** desc ****************\n");
 222         crp_desc = crp->crp_desc;
 223         printf("crd_skip=%d, crd_len=%d, crd_flags=0x%x, crd_alg=%d\n",
 224             crp_desc->crd_skip, crp_desc->crd_len, crp_desc->crd_flags,
 225             crp_desc->crd_alg);
 226
 227         key_len = crp_desc->crd_klen / 8;
 228         printf("key(%d) :\n", key_len);
 229         for (i = 0; i < key_len; i++)
 230                 printf("%d", crp_desc->crd_key[i]);
 231         printf("\n");
 232
 233         printf(" IV : \n");
 234         for (i = 0; i < EALG_MAX_BLOCK_LEN; i++)
 235                 printf("%d", crp_desc->crd_iv[i]);
 236         printf("\n");
 237
 238         printf("crd_next=%p\n", crp_desc->crd_next);
 239         return;
 240 }
 241
 242 void
 243 print_cmd(struct xlp_sec_command *cmd)
 244 {
 245         printf("session_num             :%d\n",cmd->session_num);
 246         printf("crp                     :0x%x\n",(uint32_t)cmd->crp);
 247         printf("enccrd                  :0x%x\n",(uint32_t)cmd->enccrd);
 248         printf("maccrd                  :0x%x\n",(uint32_t)cmd->maccrd);
 249         printf("ses                     :%d\n",(uint32_t)cmd->ses);
 250         printf("ctrlp                   :0x%x\n",(uint32_t)cmd->ctrlp);
 251         printf("paramp                  :0x%x\n",(uint32_t)cmd->paramp);
 252         printf("hashdest                :0x%x\n",(uint32_t)cmd->hashdest);
 253         printf("hashsrc                 :%d\n",cmd->hashsrc);
 254         printf("hmacpad                 :%d\n",cmd->hmacpad);
 255         printf("hashoff                 :%d\n",cmd->hashoff);
 256         printf("hashlen                 :%d\n",cmd->hashlen);
 257         printf("cipheroff               :%d\n",cmd->cipheroff);
 258         printf("cipherlen               :%d\n",cmd->cipherlen);
 259         printf("ivoff                   :%d\n",cmd->ivoff);
 260         printf("ivlen                   :%d\n",cmd->ivlen);
 261         printf("hashalg                 :%d\n",cmd->hashalg);
 262         printf("hashmode                :%d\n",cmd->hashmode);
 263         printf("cipheralg               :%d\n",cmd->cipheralg);
 264         printf("ciphermode              :%d\n",cmd->ciphermode);
 265         printf("nsegs                   :%d\n",cmd->nsegs);
 266         printf("hash_dst_len            :%d\n",cmd->hash_dst_len);
 267 }
 268 #endif /* NLM_SEC_DEBUG */
 269
 270 static int
 271 xlp_sec_init(struct xlp_sec_softc *sc)
 272 {
 273
 274         /* Register interrupt handler for the SEC CMS messages */
 275         if (register_msgring_handler(sc->sec_vc_start,
 276             sc->sec_vc_end, nlm_xlpsec_msgring_handler, sc) != 0) {
 277                 printf("Couldn't register sec msgring handler\n");
 278                 return (-1);
 279         }
 280
 281         /* Do the CMS credit initialization */
 282         /* Currently it is configured by default to 50 when kernel comes up */
 283
 284         return (0);
 285 }
 286
 287 /* This function is called from an interrupt handler */
 288 void
 289 nlm_xlpsec_msgring_handler(int vc, int size, int code, int src_id,
 290     struct nlm_fmn_msg *msg, void *data)
 291 {
 292         struct xlp_sec_command *cmd = NULL;
 293         struct xlp_sec_softc *sc = NULL;
 294         struct cryptodesc *crd = NULL;
 295         unsigned int ivlen = 0;
 296
 297         KASSERT(code == FMN_SWCODE_CRYPTO,
 298             ("%s: bad code = %d, expected code = %d\n", __FUNCTION__,
 299             code, FMN_SWCODE_CRYPTO));
 300
 301         sc = (struct xlp_sec_softc *)data;
 302         KASSERT(src_id >= sc->sec_vc_start && src_id <= sc->sec_vc_end,
 303             ("%s: bad src_id = %d, expect %d - %d\n", __FUNCTION__,
 304             src_id, sc->sec_vc_start, sc->sec_vc_end));
 305
 306         cmd = (struct xlp_sec_command *)(uintptr_t)msg->msg[0];
 307         KASSERT(cmd != NULL && cmd->crp != NULL,
 308                 ("%s :cmd not received properly\n",__FUNCTION__));
 309
 310         KASSERT(CRYPTO_ERROR(msg->msg[1]) == 0,
 311             ("%s: Message rcv msg0 %llx msg1 %llx err %x \n", __FUNCTION__,
 312             (unsigned long long)msg->msg[0], (unsigned long long)msg->msg[1],
 313             (int)CRYPTO_ERROR(msg->msg[1])));
 314
 315         crd = cmd->enccrd;
 316         /* Copy the last 8 or 16 bytes to the session iv, so that in few
 317          * cases this will be used as IV for the next request
 318          */
 319         if (crd != NULL) {
 320                 if ((crd->crd_alg == CRYPTO_DES_CBC ||
 321                     crd->crd_alg == CRYPTO_3DES_CBC ||
 322                     crd->crd_alg == CRYPTO_AES_CBC) &&
 323                     (crd->crd_flags & CRD_F_ENCRYPT)) {
 324                         ivlen = ((crd->crd_alg == CRYPTO_AES_CBC) ?
 325                             XLP_SEC_AES_IV_LENGTH : XLP_SEC_DES_IV_LENGTH);
 326                         crypto_copydata(cmd->crp->crp_flags, cmd->crp->crp_buf,
 327                             crd->crd_skip + crd->crd_len - ivlen, ivlen,
 328                             sc->sc_sessions[cmd->session_num].ses_iv);
 329                 }
 330         }
 331
 332         /* If there are not enough credits to send, then send request
 333          * will fail with ERESTART and the driver will be blocked until it is
 334          * unblocked here after knowing that there are sufficient credits to
 335          * send the request again.
 336          */
 337         if (sc->sc_needwakeup) {
 338                 atomic_add_int(&creditleft, sc->sec_msgsz);
 339                 if (creditleft >= (NLM_CRYPTO_LEFT_REQS)) {
 340                         crypto_unblock(sc->sc_cid, sc->sc_needwakeup);
 341                         sc->sc_needwakeup &= (~(CRYPTO_SYMQ | CRYPTO_ASYMQ));
 342                 }
 343         }
 344         if(cmd->maccrd) {
 345                 crypto_copyback(cmd->crp->crp_flags,
 346                     cmd->crp->crp_buf, cmd->maccrd->crd_inject,
 347                     cmd->hash_dst_len, cmd->hashdest);
 348         }
 349
 350         /* This indicates completion of the crypto operation */
 351         crypto_done(cmd->crp);
 352
 353         xlp_free_cmd_params(cmd);
 354
 355         return;
 356 }
 357
 358 static int
 359 xlp_sec_probe(device_t dev)
 360 {
 361         struct xlp_sec_softc *sc;
 362
 363         if (pci_get_vendor(dev) == PCI_VENDOR_NETLOGIC &&
 364             pci_get_device(dev) == PCI_DEVICE_ID_NLM_SAE) {
 365                 sc = device_get_softc(dev);
 366                 return (BUS_PROBE_DEFAULT);
 367         }
 368         return (ENXIO);
 369 }
 370
 371 /*
 372  * Attach an interface that successfully probed.
 373  */
 374 static int
 375 xlp_sec_attach(device_t dev)
 376 {
 377         struct xlp_sec_softc *sc = device_get_softc(dev);
 378         uint64_t base;
 379         int qstart, qnum;
 380         int freq, node;
 381
 382         sc->sc_dev = dev;
 383
 384         node = nlm_get_device_node(pci_get_slot(dev));
 385         freq = nlm_set_device_frequency(node, DFS_DEVICE_SAE, 250);
 386         if (bootverbose)
 387                 device_printf(dev, "SAE Freq: %dMHz\n", freq);
 388         if(pci_get_device(dev) == PCI_DEVICE_ID_NLM_SAE) {
 389                 device_set_desc(dev, "XLP Security Accelerator");
 390                 sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE);
 391                 if (sc->sc_cid < 0) {
 392                         printf("xlp_sec - error : could not get the driver"
 393                             " id\n");
 394                         goto error_exit;
 395                 }
 396                 if (crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0) != 0)
 397                         printf("register failed for CRYPTO_DES_CBC\n");
 398
 399                 if (crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0) != 0)
 400                         printf("register failed for CRYPTO_3DES_CBC\n");
 401
 402                 if (crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0) != 0)
 403                         printf("register failed for CRYPTO_AES_CBC\n");
 404
 405                 if (crypto_register(sc->sc_cid, CRYPTO_ARC4, 0, 0) != 0)
 406                         printf("register failed for CRYPTO_ARC4\n");
 407
 408                 if (crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0) != 0)
 409                         printf("register failed for CRYPTO_MD5\n");
 410
 411                 if (crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0) != 0)
 412                         printf("register failed for CRYPTO_SHA1\n");
 413
 414                 if (crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0) != 0)
 415                         printf("register failed for CRYPTO_MD5_HMAC\n");
 416
 417                 if (crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0) != 0)
 418                         printf("register failed for CRYPTO_SHA1_HMAC\n");
 419
 420                 base = nlm_get_sec_pcibase(node);
 421                 qstart = nlm_qidstart(base);
 422                 qnum = nlm_qnum(base);
 423                 sc->sec_vc_start = qstart;
 424                 sc->sec_vc_end = qstart + qnum - 1;
 425         }
 426
 427         if (xlp_sec_init(sc) != 0)
 428                 goto error_exit;
 429         if (bootverbose)
 430                 device_printf(dev, "SEC Initialization complete!\n");
 431         return (0);
 432
 433 error_exit:
 434         return (ENXIO);
 435
 436 }
 437
 438 /*
 439  * Detach an interface that successfully probed.
 440  */
 441 static int
 442 xlp_sec_detach(device_t dev)
 443 {
 444         return (0);
 445 }
 446
 447 /*
 448  * Allocate a new 'session' and return an encoded session id.  'sidp'
 449  * contains our registration id, and should contain an encoded session
 450  * id on successful allocation.
 451  */
 452 static int
 453 xlp_sec_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri)
 454 {
 455         struct cryptoini *c;
 456         struct xlp_sec_softc *sc = device_get_softc(dev);
 457         int mac = 0, cry = 0, sesn;
 458         struct xlp_sec_session *ses = NULL;
 459         struct xlp_sec_command *cmd = NULL;
 460
 461         if (sidp == NULL || cri == NULL || sc == NULL)
 462                 return (EINVAL);
 463
 464         if (sc->sc_sessions == NULL) {
 465                 ses = sc->sc_sessions = malloc(sizeof(struct xlp_sec_session),
 466                     M_DEVBUF, M_NOWAIT);
 467                 if (ses == NULL)
 468                         return (ENOMEM);
 469                 sesn = 0;
 470                 sc->sc_nsessions = 1;
 471         } else {
 472                 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
 473                         if (!sc->sc_sessions[sesn].hs_used) {
 474                                 ses = &sc->sc_sessions[sesn];
 475                                 break;
 476                         }
 477                 }
 478
 479                 if (ses == NULL) {
 480                         sesn = sc->sc_nsessions;
 481                         ses = malloc((sesn + 1)*sizeof(struct xlp_sec_session),
 482                             M_DEVBUF, M_NOWAIT);
 483                         if (ses == NULL)
 484                                 return (ENOMEM);
 485                         bcopy(sc->sc_sessions, ses, sesn * sizeof(*ses));
 486                         bzero(sc->sc_sessions, sesn * sizeof(*ses));
 487                         free(sc->sc_sessions, M_DEVBUF);
 488                         sc->sc_sessions = ses;
 489                         ses = &sc->sc_sessions[sesn];
 490                         sc->sc_nsessions++;
 491                 }
 492         }
 493         bzero(ses, sizeof(*ses));
 494         ses->sessionid = sesn;
 495         cmd = &ses->cmd;
 496         ses->hs_used = 1;
 497
 498         for (c = cri; c != NULL; c = c->cri_next) {
 499                 switch (c->cri_alg) {
 500                 case CRYPTO_MD5:
 501                 case CRYPTO_SHA1:
 502                 case CRYPTO_MD5_HMAC:
 503                 case CRYPTO_SHA1_HMAC:
 504                         if (mac)
 505                                 return (EINVAL);
 506                         mac = 1;
 507                         ses->hs_mlen = c->cri_mlen;
 508                         if (ses->hs_mlen == 0) {
 509                                 switch (c->cri_alg) {
 510                                 case CRYPTO_MD5:
 511                                 case CRYPTO_MD5_HMAC:
 512                                         ses->hs_mlen = 16;
 513                                         break;
 514                                 case CRYPTO_SHA1:
 515                                 case CRYPTO_SHA1_HMAC:
 516                                         ses->hs_mlen = 20;
 517                                         break;
 518                                 }
 519                         }
 520                         break;
 521                 case CRYPTO_DES_CBC:
 522                 case CRYPTO_3DES_CBC:
 523                 case CRYPTO_AES_CBC:
 524                         /* XXX this may read fewer, does it matter? */
 525                         read_random(ses->ses_iv, c->cri_alg ==
 526                             CRYPTO_AES_CBC ? XLP_SEC_AES_IV_LENGTH :
 527                             XLP_SEC_DES_IV_LENGTH);
 528                         /* FALLTHROUGH */
 529                 case CRYPTO_ARC4:
 530                         if (cry)
 531                                 return (EINVAL);
 532                         cry = 1;
 533                         break;
 534                 default:
 535                         return (EINVAL);
 536                 }
 537         }
 538         if (mac == 0 && cry == 0)
 539                 return (EINVAL);
 540
 541         cmd->hash_dst_len = ses->hs_mlen;
 542         *sidp = XLP_SEC_SID(device_get_unit(sc->sc_dev), sesn);
 543         return (0);
 544 }
 545
 546 /*
 547  * Deallocate a session.
 548  * XXX this routine should run a zero'd mac/encrypt key into context ram.
 549  * XXX to blow away any keys already stored there.
 550  */
 551 static int
 552 xlp_sec_freesession(device_t dev, u_int64_t tid)
 553 {
 554         struct xlp_sec_softc *sc = device_get_softc(dev);
 555         int session;
 556         u_int32_t sid = CRYPTO_SESID2LID(tid);
 557
 558         if (sc == NULL)
 559                 return (EINVAL);
 560
 561         session = XLP_SEC_SESSION(sid);
 562         if (session >= sc->sc_nsessions)
 563                 return (EINVAL);
 564
 565         sc->sc_sessions[session].hs_used = 0;
 566         return (0);
 567 }
 568
 569 static int
 570 xlp_copyiv(struct xlp_sec_softc *sc, struct xlp_sec_command *cmd,
 571     struct cryptodesc *enccrd)
 572 {
 573         unsigned int ivlen = 0;
 574         int session;
 575         struct cryptop *crp = NULL;
 576
 577         crp = cmd->crp;
 578         session = cmd->session_num;
 579
 580         if (enccrd->crd_alg != CRYPTO_ARC4) {
 581                 ivlen = ((enccrd->crd_alg == CRYPTO_AES_CBC) ?
 582                     XLP_SEC_AES_IV_LENGTH : XLP_SEC_DES_IV_LENGTH);
 583                 if (enccrd->crd_flags & CRD_F_ENCRYPT) {
 584                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) {
 585                                 bcopy(enccrd->crd_iv, cmd->iv, ivlen);
 586                         } else {
 587                                 bcopy(sc->sc_sessions[session].ses_iv, cmd->iv,
 588                                     ivlen);
 589                         }
 590                         if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
 591                                 crypto_copyback(crp->crp_flags,
 592                                     crp->crp_buf, enccrd->crd_inject,
 593                                     ivlen, cmd->iv);
 594                         }
 595                 } else {
 596                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) {
 597                                 bcopy(enccrd->crd_iv, cmd->iv, ivlen);
 598                         } else {
 599                                 crypto_copydata(crp->crp_flags, crp->crp_buf,
 600                                     enccrd->crd_inject, ivlen, cmd->iv);
 601                         }
 602                 }
 603         }
 604         return (0);
 605 }
 606
 607 static int
 608 xlp_get_nsegs(struct cryptop *crp, unsigned int *nsegs)
 609 {
 610
 611         if (crp->crp_flags & CRYPTO_F_IMBUF) {
 612                 struct mbuf *m = NULL;
 613
 614                 m = (struct mbuf *)crp->crp_buf;
 615                 while (m != NULL) {
 616                         *nsegs += NLM_CRYPTO_NUM_SEGS_REQD(m->m_len);
 617                         m = m->m_next;
 618                 }
 619         } else if (crp->crp_flags & CRYPTO_F_IOV) {
 620                 struct uio *uio = NULL;
 621                 struct iovec *iov = NULL;
 622                 int iol = 0;
 623
 624                 uio = (struct uio *)crp->crp_buf;
 625                 iov = (struct iovec *)uio->uio_iov;
 626                 iol = uio->uio_iovcnt;
 627                 while (iol > 0) {
 628                         *nsegs += NLM_CRYPTO_NUM_SEGS_REQD(iov->iov_len);
 629                         iol--;
 630                         iov++;
 631                 }
 632         } else {
 633                 *nsegs = NLM_CRYPTO_NUM_SEGS_REQD(crp->crp_ilen);
 634         }
 635         return (0);
 636 }
 637
 638 static int
 639 xlp_alloc_cmd_params(struct xlp_sec_command *cmd, unsigned int nsegs)
 640 {
 641         int err = 0;
 642
 643         if(cmd == NULL) {
 644                 err = EINVAL;
 645                 goto error;
 646         }
 647         if ((cmd->ctrlp = malloc(sizeof(struct nlm_crypto_pkt_ctrl), M_DEVBUF,
 648             M_NOWAIT | M_ZERO)) == NULL) {
 649                 err = ENOMEM;
 650                 goto error;
 651         }
 652         if (((uintptr_t)cmd->ctrlp & (XLP_L2L3_CACHELINE_SIZE - 1))) {
 653                 err = EINVAL;
 654                 goto error;
 655         }
 656         /* (nsegs - 1) because one seg is part of the structure already */
 657         if ((cmd->paramp = malloc(sizeof(struct nlm_crypto_pkt_param) +
 658             (16 * (nsegs - 1)), M_DEVBUF, M_NOWAIT | M_ZERO)) == NULL) {
 659                 err = ENOMEM;
 660                 goto error;
 661         }
 662         if (((uintptr_t)cmd->paramp & (XLP_L2L3_CACHELINE_SIZE - 1))) {
 663                 err = EINVAL;
 664                 goto error;
 665         }
 666         if ((cmd->iv = malloc(EALG_MAX_BLOCK_LEN, M_DEVBUF,
 667             M_NOWAIT | M_ZERO)) == NULL) {
 668                 err = ENOMEM;
 669                 goto error;
 670         }
 671         if ((cmd->hashdest = malloc(HASH_MAX_LEN, M_DEVBUF,
 672             M_NOWAIT | M_ZERO)) == NULL) {
 673                 err = ENOMEM;
 674                 goto error;
 675         }
 676 error:
 677         return (err);
 678 }
 679
 680 static void
 681 xlp_free_cmd_params(struct xlp_sec_command *cmd)
 682 {
 683         if (cmd->ctrlp != NULL)
 684                 free(cmd->ctrlp, M_DEVBUF);
 685         if (cmd->paramp != NULL)
 686                 free(cmd->paramp, M_DEVBUF);
 687         if (cmd->iv != NULL)
 688                 free(cmd->iv, M_DEVBUF);
 689         if (cmd->hashdest != NULL)
 690                 free(cmd->hashdest, M_DEVBUF);
 691         if (cmd != NULL)
 692                 free(cmd, M_DEVBUF);
 693         return;
 694 }
 695
 696 static int
 697 xlp_sec_process(device_t dev, struct cryptop *crp, int hint)
 698 {
 699         struct xlp_sec_softc *sc = device_get_softc(dev);
 700         struct xlp_sec_command *cmd = NULL;
 701         int session, err = -1, ret = 0;
 702         struct cryptodesc *crd1, *crd2;
 703         struct xlp_sec_session *ses;
 704         unsigned int nsegs = 0;
 705
 706         if (crp == NULL || crp->crp_callback == NULL) {
 707                 return (EINVAL);
 708         }
 709         session = XLP_SEC_SESSION(crp->crp_sid);
 710         if (sc == NULL || session >= sc->sc_nsessions) {
 711                 err = EINVAL;
 712                 goto errout;
 713         }
 714         ses = &sc->sc_sessions[session];
 715
 716         if ((cmd = malloc(sizeof(struct xlp_sec_command), M_DEVBUF,
 717             M_NOWAIT | M_ZERO)) == NULL) {
 718                 err = ENOMEM;
 719                 goto errout;
 720         }
 721
 722         cmd->crp = crp;
 723         cmd->session_num = session;
 724         cmd->hash_dst_len = ses->hs_mlen;
 725
 726         if ((crd1 = crp->crp_desc) == NULL) {
 727                 err = EINVAL;
 728                 goto errout;
 729         }
 730         crd2 = crd1->crd_next;
 731
 732         if ((ret = xlp_get_nsegs(crp, &nsegs)) != 0) {
 733                 err = EINVAL;
 734                 goto errout;
 735         }
 736         if (((crd1 != NULL) && (crd1->crd_flags & CRD_F_IV_EXPLICIT)) ||
 737             ((crd2 != NULL) && (crd2->crd_flags & CRD_F_IV_EXPLICIT))) {
 738                 /* Since IV is given as separate segment to avoid copy */
 739                 nsegs += 1;
 740         }
 741         cmd->nsegs = nsegs;
 742
 743         if ((err = xlp_alloc_cmd_params(cmd, nsegs)) != 0)
 744                 goto errout;
 745
 746         if ((crd1 != NULL) && (crd2 == NULL)) {
 747                 if (crd1->crd_alg == CRYPTO_DES_CBC ||
 748                     crd1->crd_alg == CRYPTO_3DES_CBC ||
 749                     crd1->crd_alg == CRYPTO_AES_CBC ||
 750                     crd1->crd_alg == CRYPTO_ARC4) {
 751                         cmd->enccrd = crd1;
 752                         cmd->maccrd = NULL;
 753                         if ((ret = nlm_get_cipher_param(cmd)) != 0) {
 754                                 err = EINVAL;
 755                                 goto errout;
 756                         }
 757                         if (crd1->crd_flags & CRD_F_IV_EXPLICIT)
 758                                 cmd->cipheroff = cmd->ivlen;
 759                         else
 760                                 cmd->cipheroff = cmd->enccrd->crd_skip;
 761                         cmd->cipherlen = cmd->enccrd->crd_len;
 762                         if (crd1->crd_flags & CRD_F_IV_PRESENT)
 763                                 cmd->ivoff  = 0;
 764                         else
 765                                 cmd->ivoff  = cmd->enccrd->crd_inject;
 766                         if ((err = xlp_copyiv(sc, cmd, cmd->enccrd)) != 0)
 767                                 goto errout;
 768                         if ((err = nlm_crypto_do_cipher(sc, cmd)) != 0)
 769                                 goto errout;
 770                 } else if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
 771                     crd1->crd_alg == CRYPTO_SHA1_HMAC ||
 772                     crd1->crd_alg == CRYPTO_SHA1 ||
 773                     crd1->crd_alg == CRYPTO_MD5) {
 774                         cmd->enccrd = NULL;
 775                         cmd->maccrd = crd1;
 776                         if ((ret = nlm_get_digest_param(cmd)) != 0) {
 777                                 err = EINVAL;
 778                                 goto errout;
 779                         }
 780                         cmd->hashoff = cmd->maccrd->crd_skip;
 781                         cmd->hashlen = cmd->maccrd->crd_len;
 782                         cmd->hmacpad = 0;
 783                         cmd->hashsrc = 0;
 784                         if ((err = nlm_crypto_do_digest(sc, cmd)) != 0)
 785                                 goto errout;
 786                 } else {
 787                         err = EINVAL;
 788                         goto errout;
 789                 }
 790         } else if( (crd1 != NULL) && (crd2 != NULL) ) {
 791                 if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
 792                     crd1->crd_alg == CRYPTO_SHA1_HMAC ||
 793                     crd1->crd_alg == CRYPTO_MD5 ||
 794                     crd1->crd_alg == CRYPTO_SHA1) &&
 795                     (crd2->crd_alg == CRYPTO_DES_CBC ||
 796                     crd2->crd_alg == CRYPTO_3DES_CBC ||
 797                     crd2->crd_alg == CRYPTO_AES_CBC ||
 798                     crd2->crd_alg == CRYPTO_ARC4)) {
 799                         cmd->maccrd = crd1;
 800                         cmd->enccrd = crd2;
 801                 } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
 802                     crd1->crd_alg == CRYPTO_ARC4 ||
 803                     crd1->crd_alg == CRYPTO_3DES_CBC ||
 804                     crd1->crd_alg == CRYPTO_AES_CBC) &&
 805                     (crd2->crd_alg == CRYPTO_MD5_HMAC ||
 806                     crd2->crd_alg == CRYPTO_SHA1_HMAC ||
 807                     crd2->crd_alg == CRYPTO_MD5 ||
 808                     crd2->crd_alg == CRYPTO_SHA1)) {
 809                         cmd->enccrd = crd1;
 810                         cmd->maccrd = crd2;
 811                 } else {
 812                         err = EINVAL;
 813                         goto errout;
 814                 }
 815                 if ((ret = nlm_get_cipher_param(cmd)) != 0) {
 816                         err = EINVAL;
 817                         goto errout;
 818                 }
 819                 if ((ret = nlm_get_digest_param(cmd)) != 0) {
 820                         err = EINVAL;
 821                         goto errout;
 822                 }
 823                 cmd->ivoff  = cmd->enccrd->crd_inject;
 824                 cmd->hashoff = cmd->maccrd->crd_skip;
 825                 cmd->hashlen = cmd->maccrd->crd_len;
 826                 cmd->hmacpad = 0;
 827                 if (cmd->enccrd->crd_flags & CRD_F_ENCRYPT)
 828                         cmd->hashsrc = 1;
 829                 else
 830                         cmd->hashsrc = 0;
 831                 cmd->cipheroff = cmd->enccrd->crd_skip;
 832                 cmd->cipherlen = cmd->enccrd->crd_len;
 833                 if ((err = xlp_copyiv(sc, cmd, cmd->enccrd)) != 0)
 834                         goto errout;
 835                 if ((err = nlm_crypto_do_cipher_digest(sc, cmd)) != 0)
 836                         goto errout;
 837         } else {
 838                 err = EINVAL;
 839                 goto errout;
 840         }
 841         return (0);
 842 errout:
 843         xlp_free_cmd_params(cmd);
 844         if (err == ERESTART) {
 845                 sc->sc_needwakeup |= CRYPTO_SYMQ;
 846                 creditleft = 0;
 847                 return (err);
 848         }
 849         crp->crp_etype = err;
 850         crypto_done(crp);
 851         return (err);
 852 }