sys/mips/nlm/dev/sec/nlmsec.c

   1 /*-
   2  * Copyright (c) 2003-2012 Broadcom Corporation
   3  * All Rights Reserved
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  *
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in
  13  *    the documentation and/or other materials provided with the
  14  *    distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY BROADCOM ``AS IS'' AND ANY EXPRESS OR
  17  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  18  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19  * ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM OR CONTRIBUTORS BE LIABLE
  20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  21  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  22  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  23  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  24  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
  25  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
  26  * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include <sys/cdefs.h>
  33 #include <sys/param.h>
  34 #include <sys/systm.h>
  35 #include <sys/proc.h>
  36 #include <sys/errno.h>
  37 #include <sys/malloc.h>
  38 #include <sys/kernel.h>
  39 #include <sys/module.h>
  40 #include <sys/mbuf.h>
  41 #include <sys/lock.h>
  42 #include <sys/mutex.h>
  43 #include <sys/sysctl.h>
  44 #include <sys/bus.h>
  45 #include <sys/random.h>
  46 #include <sys/rman.h>
  47 #include <sys/uio.h>
  48 #include <sys/kobj.h>
  49
  50 #include <dev/pci/pcivar.h>
  51
  52 #include <opencrypto/cryptodev.h>
  53
  54 #include "cryptodev_if.h"
  55
  56 #include <vm/vm.h>
  57 #include <vm/pmap.h>
  58
  59 #include <mips/nlm/hal/haldefs.h>
  60 #include <mips/nlm/hal/iomap.h>
  61 #include <mips/nlm/xlp.h>
  62 #include <mips/nlm/hal/sys.h>
  63 #include <mips/nlm/hal/fmn.h>
  64 #include <mips/nlm/hal/nlmsaelib.h>
  65 #include <mips/nlm/dev/sec/nlmseclib.h>
  66 #include <mips/nlm/hal/cop2.h>
  67 #include <mips/nlm/hal/mips-extns.h>
  68 #include <mips/nlm/msgring.h>
  69
  70 unsigned int creditleft;
  71
  72 void xlp_sec_print_data(struct cryptop *crp);
  73
  74 static  int xlp_sec_init(struct xlp_sec_softc *sc);
  75 static  int xlp_sec_newsession(device_t , uint32_t *, struct cryptoini *);
  76 static  int xlp_sec_freesession(device_t , uint64_t);
  77 static  int xlp_sec_process(device_t , struct cryptop *, int);
  78 static  int xlp_copyiv(struct xlp_sec_softc *, struct xlp_sec_command *,
  79     struct cryptodesc *enccrd);
  80 static int xlp_get_nsegs(struct cryptop *, unsigned int *);
  81 static int xlp_alloc_cmd_params(struct xlp_sec_command *, unsigned int);
  82 static void  xlp_free_cmd_params(struct xlp_sec_command *);
  83
  84 static  int xlp_sec_probe(device_t);
  85 static  int xlp_sec_attach(device_t);
  86 static  int xlp_sec_detach(device_t);
  87
  88 static device_method_t xlp_sec_methods[] = {
  89         /* device interface */
  90         DEVMETHOD(device_probe, xlp_sec_probe),
  91         DEVMETHOD(device_attach, xlp_sec_attach),
  92         DEVMETHOD(device_detach, xlp_sec_detach),
  93
  94         /* bus interface */
  95         DEVMETHOD(bus_print_child, bus_generic_print_child),
  96         DEVMETHOD(bus_driver_added, bus_generic_driver_added),
  97
  98         /* crypto device methods */
  99         DEVMETHOD(cryptodev_newsession, xlp_sec_newsession),
 100         DEVMETHOD(cryptodev_freesession,xlp_sec_freesession),
 101         DEVMETHOD(cryptodev_process,    xlp_sec_process),
 102
 103         DEVMETHOD_END
 104 };
 105
 106 static driver_t xlp_sec_driver = {
 107         "nlmsec",
 108         xlp_sec_methods,
 109         sizeof(struct xlp_sec_softc)
 110 };
 111 static devclass_t xlp_sec_devclass;
 112
 113 DRIVER_MODULE(nlmsec, pci, xlp_sec_driver, xlp_sec_devclass, 0, 0);
 114 MODULE_DEPEND(nlmsec, crypto, 1, 1, 1);
 115
 116 void
 117 nlm_xlpsec_msgring_handler(int vc, int size, int code, int src_id,
 118     struct nlm_fmn_msg *msg, void *data);
 119
 120 #ifdef NLM_SEC_DEBUG
 121
 122 #define extract_bits(x, bitshift, bitcnt)                               \
 123     (((unsigned long long)x >> bitshift) & ((1ULL << bitcnt) - 1))
 124
 125 void
 126 print_crypto_params(struct xlp_sec_command *cmd, struct nlm_fmn_msg m)
 127 {
 128         unsigned long long msg0,msg1,msg2,msg3,msg4,msg5,msg6,msg7,msg8;
 129
 130         msg0 = cmd->ctrlp->desc0;
 131         msg1 = cmd->paramp->desc0;
 132         msg2 = cmd->paramp->desc1;
 133         msg3 = cmd->paramp->desc2;
 134         msg4 = cmd->paramp->desc3;
 135         msg5 = cmd->paramp->segment[0][0];
 136         msg6 = cmd->paramp->segment[0][1];
 137         msg7 = m.msg[0];
 138         msg8 = m.msg[1];
 139
 140         printf("msg0 %llx msg1 %llx msg2 %llx msg3 %llx msg4 %llx msg5 %llx"
 141             "msg6 %llx msg7 %llx msg8 %llx\n", msg0, msg1, msg2, msg3, msg4,
 142             msg5, msg6, msg7, msg8);
 143
 144         printf("c0: hmac %d htype %d hmode %d ctype %d cmode %d arc4 %x\n",
 145             (unsigned int)extract_bits(msg0, 61, 1),
 146             (unsigned int)extract_bits(msg0, 52, 8),
 147             (unsigned int)extract_bits(msg0, 43, 8),
 148             (unsigned int)extract_bits(msg0, 34, 8),
 149             (unsigned int)extract_bits(msg0, 25, 8),
 150             (unsigned int)extract_bits(msg0, 0, 23));
 151
 152         printf("p0: tls %d hsrc %d hl3 %d enc %d ivl %d hd %llx\n",
 153             (unsigned int)extract_bits(msg1, 63, 1),
 154             (unsigned int)extract_bits(msg1,62,1),
 155             (unsigned int)extract_bits(msg1,60,1),
 156             (unsigned int)extract_bits(msg1,59,1),
 157             (unsigned int)extract_bits(msg1,41,16), extract_bits(msg1,0,40));
 158
 159         printf("p1: clen %u hl %u\n",  (unsigned int)extract_bits(msg2, 32, 32),
 160             (unsigned int)extract_bits(msg2,0,32));
 161
 162         printf("p2: ivoff %d cbit %d coff %d hbit %d hclb %d hoff %d\n",
 163             (unsigned int)extract_bits(msg3, 45, 17),
 164             (unsigned int)extract_bits(msg3, 42,3),
 165             (unsigned int)extract_bits(msg3, 22,16),
 166             (unsigned int)extract_bits(msg3, 19,3),
 167             (unsigned int)extract_bits(msg3, 18,1),
 168             (unsigned int)extract_bits(msg3, 0, 16));
 169
 170         printf("p3: desfbid %d tlen %d arc4 %x hmacpad %d\n",
 171             (unsigned int)extract_bits(msg4, 48,16),
 172             (unsigned int)extract_bits(msg4,11,16),
 173             (unsigned int)extract_bits(msg4,6,3),
 174             (unsigned int)extract_bits(msg4,5,1));
 175
 176         printf("p4: sflen %d sddr %llx \n",
 177             (unsigned int)extract_bits(msg5, 48, 16),extract_bits(msg5, 0, 40));
 178
 179         printf("p5: dflen %d cl3 %d cclob %d cdest %llx \n",
 180             (unsigned int)extract_bits(msg6, 48, 16),
 181             (unsigned int)extract_bits(msg6, 46, 1),
 182             (unsigned int)extract_bits(msg6, 41, 1), extract_bits(msg6, 0, 40));
 183
 184         printf("fmn0: fbid %d dfrlen %d dfrv %d cklen %d cdescaddr %llx\n",
 185             (unsigned int)extract_bits(msg7, 48, 16),
 186             (unsigned int)extract_bits(msg7,46,2),
 187             (unsigned int)extract_bits(msg7,45,1),
 188             (unsigned int)extract_bits(msg7,40,5),
 189             (extract_bits(msg7,0,34)<< 6));
 190
 191         printf("fmn1: arc4 %d hklen %d pdesclen %d pktdescad %llx\n",
 192             (unsigned int)extract_bits(msg8, 63, 1),
 193             (unsigned int)extract_bits(msg8,56,5),
 194             (unsigned int)extract_bits(msg8,43,12),
 195             (extract_bits(msg8,0,34) << 6));
 196
 197         return;
 198 }
 199
 200 void
 201 xlp_sec_print_data(struct cryptop *crp)
 202 {
 203         int i, key_len;
 204         struct cryptodesc *crp_desc;
 205
 206         printf("session id = 0x%llx, crp_ilen = %d, crp_olen=%d \n",
 207             crp->crp_sid, crp->crp_ilen, crp->crp_olen);
 208
 209         printf("crp_flags = 0x%x\n", crp->crp_flags);
 210
 211         printf("crp buf:\n");
 212         for (i = 0; i < crp->crp_ilen; i++) {
 213                 printf("%c  ", crp->crp_buf[i]);
 214                 if (i % 10 == 0)
 215                         printf("\n");
 216         }
 217
 218         printf("\n");
 219         printf("****************** desc ****************\n");
 220         crp_desc = crp->crp_desc;
 221         printf("crd_skip=%d, crd_len=%d, crd_flags=0x%x, crd_alg=%d\n",
 222             crp_desc->crd_skip, crp_desc->crd_len, crp_desc->crd_flags,
 223             crp_desc->crd_alg);
 224
 225         key_len = crp_desc->crd_klen / 8;
 226         printf("key(%d) :\n", key_len);
 227         for (i = 0; i < key_len; i++)
 228                 printf("%d", crp_desc->crd_key[i]);
 229         printf("\n");
 230
 231         printf(" IV : \n");
 232         for (i = 0; i < EALG_MAX_BLOCK_LEN; i++)
 233                 printf("%d", crp_desc->crd_iv[i]);
 234         printf("\n");
 235
 236         printf("crd_next=%p\n", crp_desc->crd_next);
 237         return;
 238 }
 239
 240 void
 241 print_cmd(struct xlp_sec_command *cmd)
 242 {
 243         printf("session_num             :%d\n",cmd->session_num);
 244         printf("crp                     :0x%x\n",(uint32_t)cmd->crp);
 245         printf("enccrd                  :0x%x\n",(uint32_t)cmd->enccrd);
 246         printf("maccrd                  :0x%x\n",(uint32_t)cmd->maccrd);
 247         printf("ses                     :%d\n",(uint32_t)cmd->ses);
 248         printf("ctrlp                   :0x%x\n",(uint32_t)cmd->ctrlp);
 249         printf("paramp                  :0x%x\n",(uint32_t)cmd->paramp);
 250         printf("hashdest                :0x%x\n",(uint32_t)cmd->hashdest);
 251         printf("hashsrc                 :%d\n",cmd->hashsrc);
 252         printf("hmacpad                 :%d\n",cmd->hmacpad);
 253         printf("hashoff                 :%d\n",cmd->hashoff);
 254         printf("hashlen                 :%d\n",cmd->hashlen);
 255         printf("cipheroff               :%d\n",cmd->cipheroff);
 256         printf("cipherlen               :%d\n",cmd->cipherlen);
 257         printf("ivoff                   :%d\n",cmd->ivoff);
 258         printf("ivlen                   :%d\n",cmd->ivlen);
 259         printf("hashalg                 :%d\n",cmd->hashalg);
 260         printf("hashmode                :%d\n",cmd->hashmode);
 261         printf("cipheralg               :%d\n",cmd->cipheralg);
 262         printf("ciphermode              :%d\n",cmd->ciphermode);
 263         printf("nsegs                   :%d\n",cmd->nsegs);
 264         printf("hash_dst_len            :%d\n",cmd->hash_dst_len);
 265 }
 266 #endif /* NLM_SEC_DEBUG */
 267
 268 static int
 269 xlp_sec_init(struct xlp_sec_softc *sc)
 270 {
 271
 272         /* Register interrupt handler for the SEC CMS messages */
 273         if (register_msgring_handler(sc->sec_vc_start,
 274             sc->sec_vc_end, nlm_xlpsec_msgring_handler, sc) != 0) {
 275                 printf("Couldn't register sec msgring handler\n");
 276                 return (-1);
 277         }
 278
 279         /* Do the CMS credit initialization */
 280         /* Currently it is configured by default to 50 when kernel comes up */
 281
 282         return (0);
 283 }
 284
 285 /* This function is called from an interrupt handler */
 286 void
 287 nlm_xlpsec_msgring_handler(int vc, int size, int code, int src_id,
 288     struct nlm_fmn_msg *msg, void *data)
 289 {
 290         struct xlp_sec_command *cmd = NULL;
 291         struct xlp_sec_softc *sc = NULL;
 292         struct cryptodesc *crd = NULL;
 293         unsigned int ivlen = 0;
 294
 295         KASSERT(code == FMN_SWCODE_CRYPTO,
 296             ("%s: bad code = %d, expected code = %d\n", __FUNCTION__,
 297             code, FMN_SWCODE_CRYPTO));
 298
 299         sc = (struct xlp_sec_softc *)data;
 300         KASSERT(src_id >= sc->sec_vc_start && src_id <= sc->sec_vc_end,
 301             ("%s: bad src_id = %d, expect %d - %d\n", __FUNCTION__,
 302             src_id, sc->sec_vc_start, sc->sec_vc_end));
 303
 304         cmd = (struct xlp_sec_command *)(uintptr_t)msg->msg[0];
 305         KASSERT(cmd != NULL && cmd->crp != NULL,
 306                 ("%s :cmd not received properly\n",__FUNCTION__));
 307
 308         KASSERT(CRYPTO_ERROR(msg->msg[1]) == 0,
 309             ("%s: Message rcv msg0 %llx msg1 %llx err %x \n", __FUNCTION__,
 310             (unsigned long long)msg->msg[0], (unsigned long long)msg->msg[1],
 311             (int)CRYPTO_ERROR(msg->msg[1])));
 312
 313         crd = cmd->enccrd;
 314         /* Copy the last 8 or 16 bytes to the session iv, so that in few
 315          * cases this will be used as IV for the next request
 316          */
 317         if (crd != NULL) {
 318                 if ((crd->crd_alg == CRYPTO_DES_CBC ||
 319                     crd->crd_alg == CRYPTO_3DES_CBC ||
 320                     crd->crd_alg == CRYPTO_AES_CBC) &&
 321                     (crd->crd_flags & CRD_F_ENCRYPT)) {
 322                         ivlen = ((crd->crd_alg == CRYPTO_AES_CBC) ?
 323                             XLP_SEC_AES_IV_LENGTH : XLP_SEC_DES_IV_LENGTH);
 324                         crypto_copydata(cmd->crp->crp_flags, cmd->crp->crp_buf,
 325                             crd->crd_skip + crd->crd_len - ivlen, ivlen,
 326                             sc->sc_sessions[cmd->session_num].ses_iv);
 327                 }
 328         }
 329
 330         /* If there are not enough credits to send, then send request
 331          * will fail with ERESTART and the driver will be blocked until it is
 332          * unblocked here after knowing that there are sufficient credits to
 333          * send the request again.
 334          */
 335         if (sc->sc_needwakeup) {
 336                 atomic_add_int(&creditleft, sc->sec_msgsz);
 337                 if (creditleft >= (NLM_CRYPTO_LEFT_REQS)) {
 338                         crypto_unblock(sc->sc_cid, sc->sc_needwakeup);
 339                         sc->sc_needwakeup &= (~(CRYPTO_SYMQ | CRYPTO_ASYMQ));
 340                 }
 341         }
 342         if(cmd->maccrd) {
 343                 crypto_copyback(cmd->crp->crp_flags,
 344                     cmd->crp->crp_buf, cmd->maccrd->crd_inject,
 345                     cmd->hash_dst_len, cmd->hashdest);
 346         }
 347
 348         /* This indicates completion of the crypto operation */
 349         crypto_done(cmd->crp);
 350
 351         xlp_free_cmd_params(cmd);
 352
 353         return;
 354 }
 355
 356 static int
 357 xlp_sec_probe(device_t dev)
 358 {
 359         struct xlp_sec_softc *sc;
 360
 361         if (pci_get_vendor(dev) == PCI_VENDOR_NETLOGIC &&
 362             pci_get_device(dev) == PCI_DEVICE_ID_NLM_SAE) {
 363                 sc = device_get_softc(dev);
 364                 return (BUS_PROBE_DEFAULT);
 365         }
 366         return (ENXIO);
 367 }
 368
 369 /*
 370  * Attach an interface that successfully probed.
 371  */
 372 static int
 373 xlp_sec_attach(device_t dev)
 374 {
 375         struct xlp_sec_softc *sc = device_get_softc(dev);
 376         uint64_t base;
 377         int qstart, qnum;
 378         int freq, node;
 379
 380         sc->sc_dev = dev;
 381
 382         node = nlm_get_device_node(pci_get_slot(dev));
 383         freq = nlm_set_device_frequency(node, DFS_DEVICE_SAE, 250);
 384         if (bootverbose)
 385                 device_printf(dev, "SAE Freq: %dMHz\n", freq);
 386         if(pci_get_device(dev) == PCI_DEVICE_ID_NLM_SAE) {
 387                 device_set_desc(dev, "XLP Security Accelerator");
 388                 sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE);
 389                 if (sc->sc_cid < 0) {
 390                         printf("xlp_sec - error : could not get the driver"
 391                             " id\n");
 392                         goto error_exit;
 393                 }
 394                 if (crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0) != 0)
 395                         printf("register failed for CRYPTO_DES_CBC\n");
 396
 397                 if (crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0) != 0)
 398                         printf("register failed for CRYPTO_3DES_CBC\n");
 399
 400                 if (crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0) != 0)
 401                         printf("register failed for CRYPTO_AES_CBC\n");
 402
 403                 if (crypto_register(sc->sc_cid, CRYPTO_ARC4, 0, 0) != 0)
 404                         printf("register failed for CRYPTO_ARC4\n");
 405
 406                 if (crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0) != 0)
 407                         printf("register failed for CRYPTO_MD5\n");
 408
 409                 if (crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0) != 0)
 410                         printf("register failed for CRYPTO_SHA1\n");
 411
 412                 if (crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0) != 0)
 413                         printf("register failed for CRYPTO_MD5_HMAC\n");
 414
 415                 if (crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0) != 0)
 416                         printf("register failed for CRYPTO_SHA1_HMAC\n");
 417
 418                 base = nlm_get_sec_pcibase(node);
 419                 qstart = nlm_qidstart(base);
 420                 qnum = nlm_qnum(base);
 421                 sc->sec_vc_start = qstart;
 422                 sc->sec_vc_end = qstart + qnum - 1;
 423         }
 424
 425         if (xlp_sec_init(sc) != 0)
 426                 goto error_exit;
 427         if (bootverbose)
 428                 device_printf(dev, "SEC Initialization complete!\n");
 429         return (0);
 430
 431 error_exit:
 432         return (ENXIO);
 433
 434 }
 435
 436 /*
 437  * Detach an interface that successfully probed.
 438  */
 439 static int
 440 xlp_sec_detach(device_t dev)
 441 {
 442         return (0);
 443 }
 444
 445 /*
 446  * Allocate a new 'session' and return an encoded session id.  'sidp'
 447  * contains our registration id, and should contain an encoded session
 448  * id on successful allocation.
 449  */
 450 static int
 451 xlp_sec_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri)
 452 {
 453         struct cryptoini *c;
 454         struct xlp_sec_softc *sc = device_get_softc(dev);
 455         int mac = 0, cry = 0, sesn;
 456         struct xlp_sec_session *ses = NULL;
 457         struct xlp_sec_command *cmd = NULL;
 458
 459         if (sidp == NULL || cri == NULL || sc == NULL)
 460                 return (EINVAL);
 461
 462         if (sc->sc_sessions == NULL) {
 463                 ses = sc->sc_sessions = malloc(sizeof(struct xlp_sec_session),
 464                     M_DEVBUF, M_NOWAIT);
 465                 if (ses == NULL)
 466                         return (ENOMEM);
 467                 sesn = 0;
 468                 sc->sc_nsessions = 1;
 469         } else {
 470                 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
 471                         if (!sc->sc_sessions[sesn].hs_used) {
 472                                 ses = &sc->sc_sessions[sesn];
 473                                 break;
 474                         }
 475                 }
 476
 477                 if (ses == NULL) {
 478                         sesn = sc->sc_nsessions;
 479                         ses = malloc((sesn + 1)*sizeof(struct xlp_sec_session),
 480                             M_DEVBUF, M_NOWAIT);
 481                         if (ses == NULL)
 482                                 return (ENOMEM);
 483                         bcopy(sc->sc_sessions, ses, sesn * sizeof(*ses));
 484                         bzero(sc->sc_sessions, sesn * sizeof(*ses));
 485                         free(sc->sc_sessions, M_DEVBUF);
 486                         sc->sc_sessions = ses;
 487                         ses = &sc->sc_sessions[sesn];
 488                         sc->sc_nsessions++;
 489                 }
 490         }
 491         bzero(ses, sizeof(*ses));
 492         ses->sessionid = sesn;
 493         cmd = &ses->cmd;
 494         ses->hs_used = 1;
 495
 496         for (c = cri; c != NULL; c = c->cri_next) {
 497                 switch (c->cri_alg) {
 498                 case CRYPTO_MD5:
 499                 case CRYPTO_SHA1:
 500                 case CRYPTO_MD5_HMAC:
 501                 case CRYPTO_SHA1_HMAC:
 502                         if (mac)
 503                                 return (EINVAL);
 504                         mac = 1;
 505                         ses->hs_mlen = c->cri_mlen;
 506                         if (ses->hs_mlen == 0) {
 507                                 switch (c->cri_alg) {
 508                                 case CRYPTO_MD5:
 509                                 case CRYPTO_MD5_HMAC:
 510                                         ses->hs_mlen = 16;
 511                                         break;
 512                                 case CRYPTO_SHA1:
 513                                 case CRYPTO_SHA1_HMAC:
 514                                         ses->hs_mlen = 20;
 515                                         break;
 516                                 }
 517                         }
 518                         break;
 519                 case CRYPTO_DES_CBC:
 520                 case CRYPTO_3DES_CBC:
 521                 case CRYPTO_AES_CBC:
 522                         /* XXX this may read fewer, does it matter? */
 523                         read_random(ses->ses_iv, c->cri_alg ==
 524                             CRYPTO_AES_CBC ? XLP_SEC_AES_IV_LENGTH :
 525                             XLP_SEC_DES_IV_LENGTH);
 526                         /* FALLTHROUGH */
 527                 case CRYPTO_ARC4:
 528                         if (cry)
 529                                 return (EINVAL);
 530                         cry = 1;
 531                         break;
 532                 default:
 533                         return (EINVAL);
 534                 }
 535         }
 536         if (mac == 0 && cry == 0)
 537                 return (EINVAL);
 538
 539         cmd->hash_dst_len = ses->hs_mlen;
 540         *sidp = XLP_SEC_SID(device_get_unit(sc->sc_dev), sesn);
 541         return (0);
 542 }
 543
 544 /*
 545  * Deallocate a session.
 546  * XXX this routine should run a zero'd mac/encrypt key into context ram.
 547  * XXX to blow away any keys already stored there.
 548  */
 549 static int
 550 xlp_sec_freesession(device_t dev, u_int64_t tid)
 551 {
 552         struct xlp_sec_softc *sc = device_get_softc(dev);
 553         int session;
 554         u_int32_t sid = CRYPTO_SESID2LID(tid);
 555
 556         if (sc == NULL)
 557                 return (EINVAL);
 558
 559         session = XLP_SEC_SESSION(sid);
 560         if (session >= sc->sc_nsessions)
 561                 return (EINVAL);
 562
 563         sc->sc_sessions[session].hs_used = 0;
 564         return (0);
 565 }
 566
 567 static int
 568 xlp_copyiv(struct xlp_sec_softc *sc, struct xlp_sec_command *cmd,
 569     struct cryptodesc *enccrd)
 570 {
 571         unsigned int ivlen = 0;
 572         int session;
 573         struct cryptop *crp = NULL;
 574
 575         crp = cmd->crp;
 576         session = cmd->session_num;
 577
 578         if (enccrd->crd_alg != CRYPTO_ARC4) {
 579                 ivlen = ((enccrd->crd_alg == CRYPTO_AES_CBC) ?
 580                     XLP_SEC_AES_IV_LENGTH : XLP_SEC_DES_IV_LENGTH);
 581                 if (enccrd->crd_flags & CRD_F_ENCRYPT) {
 582                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) {
 583                                 bcopy(enccrd->crd_iv, cmd->iv, ivlen);
 584                         } else {
 585                                 bcopy(sc->sc_sessions[session].ses_iv, cmd->iv,
 586                                     ivlen);
 587                         }
 588                         if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
 589                                 crypto_copyback(crp->crp_flags,
 590                                     crp->crp_buf, enccrd->crd_inject,
 591                                     ivlen, cmd->iv);
 592                         }
 593                 } else {
 594                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) {
 595                                 bcopy(enccrd->crd_iv, cmd->iv, ivlen);
 596                         } else {
 597                                 crypto_copydata(crp->crp_flags, crp->crp_buf,
 598                                     enccrd->crd_inject, ivlen, cmd->iv);
 599                         }
 600                 }
 601         }
 602         return (0);
 603 }
 604
 605 static int
 606 xlp_get_nsegs(struct cryptop *crp, unsigned int *nsegs)
 607 {
 608
 609         if (crp->crp_flags & CRYPTO_F_IMBUF) {
 610                 struct mbuf *m = NULL;
 611
 612                 m = (struct mbuf *)crp->crp_buf;
 613                 while (m != NULL) {
 614                         *nsegs += NLM_CRYPTO_NUM_SEGS_REQD(m->m_len);
 615                         m = m->m_next;
 616                 }
 617         } else if (crp->crp_flags & CRYPTO_F_IOV) {
 618                 struct uio *uio = NULL;
 619                 struct iovec *iov = NULL;
 620                 int iol = 0;
 621
 622                 uio = (struct uio *)crp->crp_buf;
 623                 iov = (struct iovec *)uio->uio_iov;
 624                 iol = uio->uio_iovcnt;
 625                 while (iol > 0) {
 626                         *nsegs += NLM_CRYPTO_NUM_SEGS_REQD(iov->iov_len);
 627                         iol--;
 628                         iov++;
 629                 }
 630         } else {
 631                 *nsegs = NLM_CRYPTO_NUM_SEGS_REQD(crp->crp_ilen);
 632         }
 633         return (0);
 634 }
 635
 636 static int
 637 xlp_alloc_cmd_params(struct xlp_sec_command *cmd, unsigned int nsegs)
 638 {
 639         int err = 0;
 640
 641         if(cmd == NULL) {
 642                 err = EINVAL;
 643                 goto error;
 644         }
 645         if ((cmd->ctrlp = malloc(sizeof(struct nlm_crypto_pkt_ctrl), M_DEVBUF,
 646             M_NOWAIT | M_ZERO)) == NULL) {
 647                 err = ENOMEM;
 648                 goto error;
 649         }
 650         if (((uintptr_t)cmd->ctrlp & (XLP_L2L3_CACHELINE_SIZE - 1))) {
 651                 err = EINVAL;
 652                 goto error;
 653         }
 654         /* (nsegs - 1) because one seg is part of the structure already */
 655         if ((cmd->paramp = malloc(sizeof(struct nlm_crypto_pkt_param) +
 656             (16 * (nsegs - 1)), M_DEVBUF, M_NOWAIT | M_ZERO)) == NULL) {
 657                 err = ENOMEM;
 658                 goto error;
 659         }
 660         if (((uintptr_t)cmd->paramp & (XLP_L2L3_CACHELINE_SIZE - 1))) {
 661                 err = EINVAL;
 662                 goto error;
 663         }
 664         if ((cmd->iv = malloc(EALG_MAX_BLOCK_LEN, M_DEVBUF,
 665             M_NOWAIT | M_ZERO)) == NULL) {
 666                 err = ENOMEM;
 667                 goto error;
 668         }
 669         if ((cmd->hashdest = malloc(HASH_MAX_LEN, M_DEVBUF,
 670             M_NOWAIT | M_ZERO)) == NULL) {
 671                 err = ENOMEM;
 672                 goto error;
 673         }
 674 error:
 675         return (err);
 676 }
 677
 678 static void
 679 xlp_free_cmd_params(struct xlp_sec_command *cmd)
 680 {
 681         if (cmd->ctrlp != NULL)
 682                 free(cmd->ctrlp, M_DEVBUF);
 683         if (cmd->paramp != NULL)
 684                 free(cmd->paramp, M_DEVBUF);
 685         if (cmd->iv != NULL)
 686                 free(cmd->iv, M_DEVBUF);
 687         if (cmd->hashdest != NULL)
 688                 free(cmd->hashdest, M_DEVBUF);
 689         if (cmd != NULL)
 690                 free(cmd, M_DEVBUF);
 691         return;
 692 }
 693
 694 static int
 695 xlp_sec_process(device_t dev, struct cryptop *crp, int hint)
 696 {
 697         struct xlp_sec_softc *sc = device_get_softc(dev);
 698         struct xlp_sec_command *cmd = NULL;
 699         int session, err = -1, ret = 0;
 700         struct cryptodesc *crd1, *crd2;
 701         struct xlp_sec_session *ses;
 702         unsigned int nsegs = 0;
 703
 704         if (crp == NULL || crp->crp_callback == NULL) {
 705                 return (EINVAL);
 706         }
 707         session = XLP_SEC_SESSION(crp->crp_sid);
 708         if (sc == NULL || session >= sc->sc_nsessions) {
 709                 err = EINVAL;
 710                 goto errout;
 711         }
 712         ses = &sc->sc_sessions[session];
 713
 714         if ((cmd = malloc(sizeof(struct xlp_sec_command), M_DEVBUF,
 715             M_NOWAIT | M_ZERO)) == NULL) {
 716                 err = ENOMEM;
 717                 goto errout;
 718         }
 719
 720         cmd->crp = crp;
 721         cmd->session_num = session;
 722         cmd->hash_dst_len = ses->hs_mlen;
 723
 724         if ((crd1 = crp->crp_desc) == NULL) {
 725                 err = EINVAL;
 726                 goto errout;
 727         }
 728         crd2 = crd1->crd_next;
 729
 730         if ((ret = xlp_get_nsegs(crp, &nsegs)) != 0) {
 731                 err = EINVAL;
 732                 goto errout;
 733         }
 734         if (((crd1 != NULL) && (crd1->crd_flags & CRD_F_IV_EXPLICIT)) ||
 735             ((crd2 != NULL) && (crd2->crd_flags & CRD_F_IV_EXPLICIT))) {
 736                 /* Since IV is given as separate segment to avoid copy */
 737                 nsegs += 1;
 738         }
 739         cmd->nsegs = nsegs;
 740
 741         if ((err = xlp_alloc_cmd_params(cmd, nsegs)) != 0)
 742                 goto errout;
 743
 744         if ((crd1 != NULL) && (crd2 == NULL)) {
 745                 if (crd1->crd_alg == CRYPTO_DES_CBC ||
 746                     crd1->crd_alg == CRYPTO_3DES_CBC ||
 747                     crd1->crd_alg == CRYPTO_AES_CBC ||
 748                     crd1->crd_alg == CRYPTO_ARC4) {
 749                         cmd->enccrd = crd1;
 750                         cmd->maccrd = NULL;
 751                         if ((ret = nlm_get_cipher_param(cmd)) != 0) {
 752                                 err = EINVAL;
 753                                 goto errout;
 754                         }
 755                         if (crd1->crd_flags & CRD_F_IV_EXPLICIT)
 756                                 cmd->cipheroff = cmd->ivlen;
 757                         else
 758                                 cmd->cipheroff = cmd->enccrd->crd_skip;
 759                         cmd->cipherlen = cmd->enccrd->crd_len;
 760                         if (crd1->crd_flags & CRD_F_IV_PRESENT)
 761                                 cmd->ivoff  = 0;
 762                         else
 763                                 cmd->ivoff  = cmd->enccrd->crd_inject;
 764                         if ((err = xlp_copyiv(sc, cmd, cmd->enccrd)) != 0)
 765                                 goto errout;
 766                         if ((err = nlm_crypto_do_cipher(sc, cmd)) != 0)
 767                                 goto errout;
 768                 } else if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
 769                     crd1->crd_alg == CRYPTO_SHA1_HMAC ||
 770                     crd1->crd_alg == CRYPTO_SHA1 ||
 771                     crd1->crd_alg == CRYPTO_MD5) {
 772                         cmd->enccrd = NULL;
 773                         cmd->maccrd = crd1;
 774                         if ((ret = nlm_get_digest_param(cmd)) != 0) {
 775                                 err = EINVAL;
 776                                 goto errout;
 777                         }
 778                         cmd->hashoff = cmd->maccrd->crd_skip;
 779                         cmd->hashlen = cmd->maccrd->crd_len;
 780                         cmd->hmacpad = 0;
 781                         cmd->hashsrc = 0;
 782                         if ((err = nlm_crypto_do_digest(sc, cmd)) != 0)
 783                                 goto errout;
 784                 } else {
 785                         err = EINVAL;
 786                         goto errout;
 787                 }
 788         } else if( (crd1 != NULL) && (crd2 != NULL) ) {
 789                 if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
 790                     crd1->crd_alg == CRYPTO_SHA1_HMAC ||
 791                     crd1->crd_alg == CRYPTO_MD5 ||
 792                     crd1->crd_alg == CRYPTO_SHA1) &&
 793                     (crd2->crd_alg == CRYPTO_DES_CBC ||
 794                     crd2->crd_alg == CRYPTO_3DES_CBC ||
 795                     crd2->crd_alg == CRYPTO_AES_CBC ||
 796                     crd2->crd_alg == CRYPTO_ARC4)) {
 797                         cmd->maccrd = crd1;
 798                         cmd->enccrd = crd2;
 799                 } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
 800                     crd1->crd_alg == CRYPTO_ARC4 ||
 801                     crd1->crd_alg == CRYPTO_3DES_CBC ||
 802                     crd1->crd_alg == CRYPTO_AES_CBC) &&
 803                     (crd2->crd_alg == CRYPTO_MD5_HMAC ||
 804                     crd2->crd_alg == CRYPTO_SHA1_HMAC ||
 805                     crd2->crd_alg == CRYPTO_MD5 ||
 806                     crd2->crd_alg == CRYPTO_SHA1)) {
 807                         cmd->enccrd = crd1;
 808                         cmd->maccrd = crd2;
 809                 } else {
 810                         err = EINVAL;
 811                         goto errout;
 812                 }
 813                 if ((ret = nlm_get_cipher_param(cmd)) != 0) {
 814                         err = EINVAL;
 815                         goto errout;
 816                 }
 817                 if ((ret = nlm_get_digest_param(cmd)) != 0) {
 818                         err = EINVAL;
 819                         goto errout;
 820                 }
 821                 cmd->ivoff  = cmd->enccrd->crd_inject;
 822                 cmd->hashoff = cmd->maccrd->crd_skip;
 823                 cmd->hashlen = cmd->maccrd->crd_len;
 824                 cmd->hmacpad = 0;
 825                 if (cmd->enccrd->crd_flags & CRD_F_ENCRYPT)
 826                         cmd->hashsrc = 1;
 827                 else
 828                         cmd->hashsrc = 0;
 829                 cmd->cipheroff = cmd->enccrd->crd_skip;
 830                 cmd->cipherlen = cmd->enccrd->crd_len;
 831                 if ((err = xlp_copyiv(sc, cmd, cmd->enccrd)) != 0)
 832                         goto errout;
 833                 if ((err = nlm_crypto_do_cipher_digest(sc, cmd)) != 0)
 834                         goto errout;
 835         } else {
 836                 err = EINVAL;
 837                 goto errout;
 838         }
 839         return (0);
 840 errout:
 841         xlp_free_cmd_params(cmd);
 842         if (err == ERESTART) {
 843                 sc->sc_needwakeup |= CRYPTO_SYMQ;
 844                 creditleft = 0;
 845                 return (err);
 846         }
 847         crp->crp_etype = err;
 848         crypto_done(crp);
 849         return (err);
 850 }