sys/dev/ubsec/ubsec.c

   1 /*      $OpenBSD: ubsec.c,v 1.115 2002/09/24 18:33:26 jason Exp $       */
   2
   3 /*-
   4  * Copyright (c) 2000 Jason L. Wright (jason@thought.net)
   5  * Copyright (c) 2000 Theo de Raadt (deraadt@openbsd.org)
   6  * Copyright (c) 2001 Patrik Lindergren (patrik@ipunplugged.com)
   7  *
   8  * All rights reserved.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  * 3. All advertising materials mentioning features or use of this software
  19  *    must display the following acknowledgement:
  20  *      This product includes software developed by Jason L. Wright
  21  * 4. The name of the author may not be used to endorse or promote products
  22  *    derived from this software without specific prior written permission.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  25  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  26  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  27  * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
  28  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  29  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  30  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  31  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  32  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  33  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  34  * POSSIBILITY OF SUCH DAMAGE.
  35  *
  36  * Effort sponsored in part by the Defense Advanced Research Projects
  37  * Agency (DARPA) and Air Force Research Laboratory, Air Force
  38  * Materiel Command, USAF, under agreement number F30602-01-2-0537.
  39  */
  40
  41 #include <sys/cdefs.h>
  42 __FBSDID("$FreeBSD$");
  43
  44 /*
  45  * uBsec 5[56]01, 58xx hardware crypto accelerator
  46  */
  47
  48 #include "opt_ubsec.h"
  49
  50 #include <sys/param.h>
  51 #include <sys/systm.h>
  52 #include <sys/proc.h>
  53 #include <sys/errno.h>
  54 #include <sys/malloc.h>
  55 #include <sys/kernel.h>
  56 #include <sys/module.h>
  57 #include <sys/mbuf.h>
  58 #include <sys/lock.h>
  59 #include <sys/mutex.h>
  60 #include <sys/sysctl.h>
  61 #include <sys/endian.h>
  62
  63 #include <vm/vm.h>
  64 #include <vm/pmap.h>
  65
  66 #include <machine/bus.h>
  67 #include <machine/resource.h>
  68 #include <sys/bus.h>
  69 #include <sys/rman.h>
  70
  71 #include <crypto/sha1.h>
  72 #include <opencrypto/cryptodev.h>
  73 #include <opencrypto/cryptosoft.h>
  74 #include <sys/md5.h>
  75 #include <sys/random.h>
  76
  77 #include <dev/pci/pcivar.h>
  78 #include <dev/pci/pcireg.h>
  79
  80 /* grr, #defines for gratuitous incompatibility in queue.h */
  81 #define SIMPLEQ_HEAD            STAILQ_HEAD
  82 #define SIMPLEQ_ENTRY           STAILQ_ENTRY
  83 #define SIMPLEQ_INIT            STAILQ_INIT
  84 #define SIMPLEQ_INSERT_TAIL     STAILQ_INSERT_TAIL
  85 #define SIMPLEQ_EMPTY           STAILQ_EMPTY
  86 #define SIMPLEQ_FIRST           STAILQ_FIRST
  87 #define SIMPLEQ_REMOVE_HEAD     STAILQ_REMOVE_HEAD_UNTIL
  88 #define SIMPLEQ_FOREACH         STAILQ_FOREACH
  89 /* ditto for endian.h */
  90 #define letoh16(x)              le16toh(x)
  91 #define letoh32(x)              le32toh(x)
  92
  93 #ifdef UBSEC_RNDTEST
  94 #include <dev/rndtest/rndtest.h>
  95 #endif
  96 #include <dev/ubsec/ubsecreg.h>
  97 #include <dev/ubsec/ubsecvar.h>
  98
  99 /*
 100  * Prototypes and count for the pci_device structure
 101  */
 102 static  int ubsec_probe(device_t);
 103 static  int ubsec_attach(device_t);
 104 static  int ubsec_detach(device_t);
 105 static  int ubsec_suspend(device_t);
 106 static  int ubsec_resume(device_t);
 107 static  void ubsec_shutdown(device_t);
 108
 109 static device_method_t ubsec_methods[] = {
 110         /* Device interface */
 111         DEVMETHOD(device_probe,         ubsec_probe),
 112         DEVMETHOD(device_attach,        ubsec_attach),
 113         DEVMETHOD(device_detach,        ubsec_detach),
 114         DEVMETHOD(device_suspend,       ubsec_suspend),
 115         DEVMETHOD(device_resume,        ubsec_resume),
 116         DEVMETHOD(device_shutdown,      ubsec_shutdown),
 117
 118         /* bus interface */
 119         DEVMETHOD(bus_print_child,      bus_generic_print_child),
 120         DEVMETHOD(bus_driver_added,     bus_generic_driver_added),
 121
 122         { 0, 0 }
 123 };
 124 static driver_t ubsec_driver = {
 125         "ubsec",
 126         ubsec_methods,
 127         sizeof (struct ubsec_softc)
 128 };
 129 static devclass_t ubsec_devclass;
 130
 131 DRIVER_MODULE(ubsec, pci, ubsec_driver, ubsec_devclass, 0, 0);
 132 MODULE_DEPEND(ubsec, crypto, 1, 1, 1);
 133 #ifdef UBSEC_RNDTEST
 134 MODULE_DEPEND(ubsec, rndtest, 1, 1, 1);
 135 #endif
 136
 137 static  void ubsec_intr(void *);
 138 static  int ubsec_newsession(void *, u_int32_t *, struct cryptoini *);
 139 static  int ubsec_freesession(void *, u_int64_t);
 140 static  int ubsec_process(void *, struct cryptop *, int);
 141 static  void ubsec_callback(struct ubsec_softc *, struct ubsec_q *);
 142 static  void ubsec_feed(struct ubsec_softc *);
 143 static  void ubsec_mcopy(struct mbuf *, struct mbuf *, int, int);
 144 static  void ubsec_callback2(struct ubsec_softc *, struct ubsec_q2 *);
 145 static  int ubsec_feed2(struct ubsec_softc *);
 146 static  void ubsec_rng(void *);
 147 static  int ubsec_dma_malloc(struct ubsec_softc *, bus_size_t,
 148                              struct ubsec_dma_alloc *, int);
 149 #define ubsec_dma_sync(_dma, _flags) \
 150         bus_dmamap_sync((_dma)->dma_tag, (_dma)->dma_map, (_flags))
 151 static  void ubsec_dma_free(struct ubsec_softc *, struct ubsec_dma_alloc *);
 152 static  int ubsec_dmamap_aligned(struct ubsec_operand *op);
 153
 154 static  void ubsec_reset_board(struct ubsec_softc *sc);
 155 static  void ubsec_init_board(struct ubsec_softc *sc);
 156 static  void ubsec_init_pciregs(device_t dev);
 157 static  void ubsec_totalreset(struct ubsec_softc *sc);
 158
 159 static  int ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q);
 160
 161 static  int ubsec_kprocess(void*, struct cryptkop *, int);
 162 static  int ubsec_kprocess_modexp_hw(struct ubsec_softc *, struct cryptkop *, int);
 163 static  int ubsec_kprocess_modexp_sw(struct ubsec_softc *, struct cryptkop *, int);
 164 static  int ubsec_kprocess_rsapriv(struct ubsec_softc *, struct cryptkop *, int);
 165 static  void ubsec_kfree(struct ubsec_softc *, struct ubsec_q2 *);
 166 static  int ubsec_ksigbits(struct crparam *);
 167 static  void ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
 168 static  void ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
 169
 170 SYSCTL_NODE(_hw, OID_AUTO, ubsec, CTLFLAG_RD, 0, "Broadcom driver parameters");
 171
 172 #ifdef UBSEC_DEBUG
 173 static  void ubsec_dump_pb(volatile struct ubsec_pktbuf *);
 174 static  void ubsec_dump_mcr(struct ubsec_mcr *);
 175 static  void ubsec_dump_ctx2(struct ubsec_ctx_keyop *);
 176
 177 static  int ubsec_debug = 0;
 178 SYSCTL_INT(_hw_ubsec, OID_AUTO, debug, CTLFLAG_RW, &ubsec_debug,
 179             0, "control debugging msgs");
 180 #endif
 181
 182 #define READ_REG(sc,r) \
 183         bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r))
 184
 185 #define WRITE_REG(sc,reg,val) \
 186         bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val)
 187
 188 #define SWAP32(x) (x) = htole32(ntohl((x)))
 189 #define HTOLE32(x) (x) = htole32(x)
 190
 191 struct ubsec_stats ubsecstats;
 192 SYSCTL_STRUCT(_hw_ubsec, OID_AUTO, stats, CTLFLAG_RD, &ubsecstats,
 193             ubsec_stats, "driver statistics");
 194
 195 static int
 196 ubsec_probe(device_t dev)
 197 {
 198         if (pci_get_vendor(dev) == PCI_VENDOR_SUN &&
 199             (pci_get_device(dev) == PCI_PRODUCT_SUN_5821 ||
 200              pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K))
 201                 return (BUS_PROBE_DEFAULT);
 202         if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL &&
 203             (pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5501 ||
 204              pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601))
 205                 return (BUS_PROBE_DEFAULT);
 206         if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 207             (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5801 ||
 208              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 ||
 209              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805 ||
 210              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820 ||
 211              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 ||
 212              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 ||
 213              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823
 214              ))
 215                 return (BUS_PROBE_DEFAULT);
 216         return (ENXIO);
 217 }
 218
 219 static const char*
 220 ubsec_partname(struct ubsec_softc *sc)
 221 {
 222         /* XXX sprintf numbers when not decoded */
 223         switch (pci_get_vendor(sc->sc_dev)) {
 224         case PCI_VENDOR_BROADCOM:
 225                 switch (pci_get_device(sc->sc_dev)) {
 226                 case PCI_PRODUCT_BROADCOM_5801: return "Broadcom 5801";
 227                 case PCI_PRODUCT_BROADCOM_5802: return "Broadcom 5802";
 228                 case PCI_PRODUCT_BROADCOM_5805: return "Broadcom 5805";
 229                 case PCI_PRODUCT_BROADCOM_5820: return "Broadcom 5820";
 230                 case PCI_PRODUCT_BROADCOM_5821: return "Broadcom 5821";
 231                 case PCI_PRODUCT_BROADCOM_5822: return "Broadcom 5822";
 232                 case PCI_PRODUCT_BROADCOM_5823: return "Broadcom 5823";
 233                 }
 234                 return "Broadcom unknown-part";
 235         case PCI_VENDOR_BLUESTEEL:
 236                 switch (pci_get_device(sc->sc_dev)) {
 237                 case PCI_PRODUCT_BLUESTEEL_5601: return "Bluesteel 5601";
 238                 }
 239                 return "Bluesteel unknown-part";
 240         case PCI_VENDOR_SUN:
 241                 switch (pci_get_device(sc->sc_dev)) {
 242                 case PCI_PRODUCT_SUN_5821: return "Sun Crypto 5821";
 243                 case PCI_PRODUCT_SUN_SCA1K: return "Sun Crypto 1K";
 244                 }
 245                 return "Sun unknown-part";
 246         }
 247         return "Unknown-vendor unknown-part";
 248 }
 249
 250 static void
 251 default_harvest(struct rndtest_state *rsp, void *buf, u_int count)
 252 {
 253         random_harvest(buf, count, count*NBBY, 0, RANDOM_PURE);
 254 }
 255
 256 static int
 257 ubsec_attach(device_t dev)
 258 {
 259         struct ubsec_softc *sc = device_get_softc(dev);
 260         struct ubsec_dma *dmap;
 261         u_int32_t cmd, i;
 262         int rid;
 263
 264         bzero(sc, sizeof (*sc));
 265         sc->sc_dev = dev;
 266
 267         SIMPLEQ_INIT(&sc->sc_queue);
 268         SIMPLEQ_INIT(&sc->sc_qchip);
 269         SIMPLEQ_INIT(&sc->sc_queue2);
 270         SIMPLEQ_INIT(&sc->sc_qchip2);
 271         SIMPLEQ_INIT(&sc->sc_q2free);
 272
 273         /* XXX handle power management */
 274
 275         sc->sc_statmask = BS_STAT_MCR1_DONE | BS_STAT_DMAERR;
 276
 277         if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL &&
 278             pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601)
 279                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG;
 280
 281         if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 282             (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 ||
 283              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805))
 284                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG;
 285
 286         if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 287             pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820)
 288                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG |
 289                     UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY;
 290
 291         if ((pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 292              (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 ||
 293               pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 ||
 294               pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823)) ||
 295             (pci_get_vendor(dev) == PCI_VENDOR_SUN &&
 296              (pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K ||
 297               pci_get_device(dev) == PCI_PRODUCT_SUN_5821))) {
 298                 /* NB: the 5821/5822 defines some additional status bits */
 299                 sc->sc_statmask |= BS_STAT_MCR1_ALLEMPTY |
 300                     BS_STAT_MCR2_ALLEMPTY;
 301                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG |
 302                     UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY;
 303         }
 304
 305         cmd = pci_read_config(dev, PCIR_COMMAND, 4);
 306         cmd |= PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN;
 307         pci_write_config(dev, PCIR_COMMAND, cmd, 4);
 308         cmd = pci_read_config(dev, PCIR_COMMAND, 4);
 309
 310         if (!(cmd & PCIM_CMD_MEMEN)) {
 311                 device_printf(dev, "failed to enable memory mapping\n");
 312                 goto bad;
 313         }
 314
 315         if (!(cmd & PCIM_CMD_BUSMASTEREN)) {
 316                 device_printf(dev, "failed to enable bus mastering\n");
 317                 goto bad;
 318         }
 319
 320         /*
 321          * Setup memory-mapping of PCI registers.
 322          */
 323         rid = BS_BAR;
 324         sc->sc_sr = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 325                                            RF_ACTIVE);
 326         if (sc->sc_sr == NULL) {
 327                 device_printf(dev, "cannot map register space\n");
 328                 goto bad;
 329         }
 330         sc->sc_st = rman_get_bustag(sc->sc_sr);
 331         sc->sc_sh = rman_get_bushandle(sc->sc_sr);
 332
 333         /*
 334          * Arrange interrupt line.
 335          */
 336         rid = 0;
 337         sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 338                                             RF_SHAREABLE|RF_ACTIVE);
 339         if (sc->sc_irq == NULL) {
 340                 device_printf(dev, "could not map interrupt\n");
 341                 goto bad1;
 342         }
 343         /*
 344          * NB: Network code assumes we are blocked with splimp()
 345          *     so make sure the IRQ is mapped appropriately.
 346          */
 347         if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE,
 348                            ubsec_intr, sc, &sc->sc_ih)) {
 349                 device_printf(dev, "could not establish interrupt\n");
 350                 goto bad2;
 351         }
 352
 353         sc->sc_cid = crypto_get_driverid(0);
 354         if (sc->sc_cid < 0) {
 355                 device_printf(dev, "could not get crypto driver id\n");
 356                 goto bad3;
 357         }
 358
 359         /*
 360          * Setup DMA descriptor area.
 361          */
 362         if (bus_dma_tag_create(NULL,                    /* parent */
 363                                1, 0,                    /* alignment, bounds */
 364                                BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
 365                                BUS_SPACE_MAXADDR,       /* highaddr */
 366                                NULL, NULL,              /* filter, filterarg */
 367                                0x3ffff,                 /* maxsize */
 368                                UBS_MAX_SCATTER,         /* nsegments */
 369                                0xffff,                  /* maxsegsize */
 370                                BUS_DMA_ALLOCNOW,        /* flags */
 371                                NULL, NULL,              /* lockfunc, lockarg */
 372                                &sc->sc_dmat)) {
 373                 device_printf(dev, "cannot allocate DMA tag\n");
 374                 goto bad4;
 375         }
 376         SIMPLEQ_INIT(&sc->sc_freequeue);
 377         dmap = sc->sc_dmaa;
 378         for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) {
 379                 struct ubsec_q *q;
 380
 381                 q = (struct ubsec_q *)malloc(sizeof(struct ubsec_q),
 382                     M_DEVBUF, M_NOWAIT);
 383                 if (q == NULL) {
 384                         device_printf(dev, "cannot allocate queue buffers\n");
 385                         break;
 386                 }
 387
 388                 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk),
 389                     &dmap->d_alloc, 0)) {
 390                         device_printf(dev, "cannot allocate dma buffers\n");
 391                         free(q, M_DEVBUF);
 392                         break;
 393                 }
 394                 dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr;
 395
 396                 q->q_dma = dmap;
 397                 sc->sc_queuea[i] = q;
 398
 399                 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
 400         }
 401         mtx_init(&sc->sc_mcr1lock, device_get_nameunit(dev),
 402                 "mcr1 operations", MTX_DEF);
 403         mtx_init(&sc->sc_freeqlock, device_get_nameunit(dev),
 404                 "mcr1 free q", MTX_DEF);
 405
 406         device_printf(sc->sc_dev, "%s\n", ubsec_partname(sc));
 407
 408         crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0,
 409             ubsec_newsession, ubsec_freesession, ubsec_process, sc);
 410         crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0,
 411              ubsec_newsession, ubsec_freesession, ubsec_process, sc);
 412         crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0,
 413              ubsec_newsession, ubsec_freesession, ubsec_process, sc);
 414         crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0,
 415              ubsec_newsession, ubsec_freesession, ubsec_process, sc);
 416
 417         /*
 418          * Reset Broadcom chip
 419          */
 420         ubsec_reset_board(sc);
 421
 422         /*
 423          * Init Broadcom specific PCI settings
 424          */
 425         ubsec_init_pciregs(dev);
 426
 427         /*
 428          * Init Broadcom chip
 429          */
 430         ubsec_init_board(sc);
 431
 432 #ifndef UBSEC_NO_RNG
 433         if (sc->sc_flags & UBS_FLAGS_RNG) {
 434                 sc->sc_statmask |= BS_STAT_MCR2_DONE;
 435 #ifdef UBSEC_RNDTEST
 436                 sc->sc_rndtest = rndtest_attach(dev);
 437                 if (sc->sc_rndtest)
 438                         sc->sc_harvest = rndtest_harvest;
 439                 else
 440                         sc->sc_harvest = default_harvest;
 441 #else
 442                 sc->sc_harvest = default_harvest;
 443 #endif
 444
 445                 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
 446                     &sc->sc_rng.rng_q.q_mcr, 0))
 447                         goto skip_rng;
 448
 449                 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass),
 450                     &sc->sc_rng.rng_q.q_ctx, 0)) {
 451                         ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
 452                         goto skip_rng;
 453                 }
 454
 455                 if (ubsec_dma_malloc(sc, sizeof(u_int32_t) *
 456                     UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, 0)) {
 457                         ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx);
 458                         ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
 459                         goto skip_rng;
 460                 }
 461
 462                 if (hz >= 100)
 463                         sc->sc_rnghz = hz / 100;
 464                 else
 465                         sc->sc_rnghz = 1;
 466                 callout_init(&sc->sc_rngto, CALLOUT_MPSAFE);
 467                 callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
 468 skip_rng:
 469         ;
 470         }
 471 #endif /* UBSEC_NO_RNG */
 472         mtx_init(&sc->sc_mcr2lock, device_get_nameunit(dev),
 473                 "mcr2 operations", MTX_DEF);
 474
 475         if (sc->sc_flags & UBS_FLAGS_KEY) {
 476                 sc->sc_statmask |= BS_STAT_MCR2_DONE;
 477
 478                 crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0,
 479                         ubsec_kprocess, sc);
 480 #if 0
 481                 crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0,
 482                         ubsec_kprocess, sc);
 483 #endif
 484         }
 485         return (0);
 486 bad4:
 487         crypto_unregister_all(sc->sc_cid);
 488 bad3:
 489         bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
 490 bad2:
 491         bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
 492 bad1:
 493         bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
 494 bad:
 495         return (ENXIO);
 496 }
 497
 498 /*
 499  * Detach a device that successfully probed.
 500  */
 501 static int
 502 ubsec_detach(device_t dev)
 503 {
 504         struct ubsec_softc *sc = device_get_softc(dev);
 505
 506         /* XXX wait/abort active ops */
 507
 508         /* disable interrupts */
 509         WRITE_REG(sc, BS_CTRL, READ_REG(sc, BS_CTRL) &~
 510                 (BS_CTRL_MCR2INT | BS_CTRL_MCR1INT | BS_CTRL_DMAERR));
 511
 512         callout_stop(&sc->sc_rngto);
 513
 514         crypto_unregister_all(sc->sc_cid);
 515
 516 #ifdef UBSEC_RNDTEST
 517         if (sc->sc_rndtest)
 518                 rndtest_detach(sc->sc_rndtest);
 519 #endif
 520
 521         while (!SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
 522                 struct ubsec_q *q;
 523
 524                 q = SIMPLEQ_FIRST(&sc->sc_freequeue);
 525                 SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q, q_next);
 526                 ubsec_dma_free(sc, &q->q_dma->d_alloc);
 527                 free(q, M_DEVBUF);
 528         }
 529         mtx_destroy(&sc->sc_mcr1lock);
 530 #ifndef UBSEC_NO_RNG
 531         if (sc->sc_flags & UBS_FLAGS_RNG) {
 532                 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
 533                 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx);
 534                 ubsec_dma_free(sc, &sc->sc_rng.rng_buf);
 535         }
 536 #endif /* UBSEC_NO_RNG */
 537         mtx_destroy(&sc->sc_mcr2lock);
 538
 539         bus_generic_detach(dev);
 540         bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
 541         bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
 542
 543         bus_dma_tag_destroy(sc->sc_dmat);
 544         bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
 545
 546         return (0);
 547 }
 548
 549 /*
 550  * Stop all chip i/o so that the kernel's probe routines don't
 551  * get confused by errant DMAs when rebooting.
 552  */
 553 static void
 554 ubsec_shutdown(device_t dev)
 555 {
 556 #ifdef notyet
 557         ubsec_stop(device_get_softc(dev));
 558 #endif
 559 }
 560
 561 /*
 562  * Device suspend routine.
 563  */
 564 static int
 565 ubsec_suspend(device_t dev)
 566 {
 567         struct ubsec_softc *sc = device_get_softc(dev);
 568
 569 #ifdef notyet
 570         /* XXX stop the device and save PCI settings */
 571 #endif
 572         sc->sc_suspended = 1;
 573
 574         return (0);
 575 }
 576
 577 static int
 578 ubsec_resume(device_t dev)
 579 {
 580         struct ubsec_softc *sc = device_get_softc(dev);
 581
 582 #ifdef notyet
 583         /* XXX retore PCI settings and start the device */
 584 #endif
 585         sc->sc_suspended = 0;
 586         return (0);
 587 }
 588
 589 /*
 590  * UBSEC Interrupt routine
 591  */
 592 static void
 593 ubsec_intr(void *arg)
 594 {
 595         struct ubsec_softc *sc = arg;
 596         volatile u_int32_t stat;
 597         struct ubsec_q *q;
 598         struct ubsec_dma *dmap;
 599         int npkts = 0, i;
 600
 601         stat = READ_REG(sc, BS_STAT);
 602         stat &= sc->sc_statmask;
 603         if (stat == 0)
 604                 return;
 605
 606         WRITE_REG(sc, BS_STAT, stat);           /* IACK */
 607
 608         /*
 609          * Check to see if we have any packets waiting for us
 610          */
 611         if ((stat & BS_STAT_MCR1_DONE)) {
 612                 mtx_lock(&sc->sc_mcr1lock);
 613                 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
 614                         q = SIMPLEQ_FIRST(&sc->sc_qchip);
 615                         dmap = q->q_dma;
 616
 617                         if ((dmap->d_dma->d_mcr.mcr_flags & htole16(UBS_MCR_DONE)) == 0)
 618                                 break;
 619
 620                         SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q, q_next);
 621
 622                         npkts = q->q_nstacked_mcrs;
 623                         sc->sc_nqchip -= 1+npkts;
 624                         /*
 625                          * search for further sc_qchip ubsec_q's that share
 626                          * the same MCR, and complete them too, they must be
 627                          * at the top.
 628                          */
 629                         for (i = 0; i < npkts; i++) {
 630                                 if(q->q_stacked_mcr[i]) {
 631                                         ubsec_callback(sc, q->q_stacked_mcr[i]);
 632                                 } else {
 633                                         break;
 634                                 }
 635                         }
 636                         ubsec_callback(sc, q);
 637                 }
 638                 /*
 639                  * Don't send any more packet to chip if there has been
 640                  * a DMAERR.
 641                  */
 642                 if (!(stat & BS_STAT_DMAERR))
 643                         ubsec_feed(sc);
 644                 mtx_unlock(&sc->sc_mcr1lock);
 645         }
 646
 647         /*
 648          * Check to see if we have any key setups/rng's waiting for us
 649          */
 650         if ((sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) &&
 651             (stat & BS_STAT_MCR2_DONE)) {
 652                 struct ubsec_q2 *q2;
 653                 struct ubsec_mcr *mcr;
 654
 655                 mtx_lock(&sc->sc_mcr2lock);
 656                 while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) {
 657                         q2 = SIMPLEQ_FIRST(&sc->sc_qchip2);
 658
 659                         ubsec_dma_sync(&q2->q_mcr,
 660                             BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
 661
 662                         mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr;
 663                         if ((mcr->mcr_flags & htole16(UBS_MCR_DONE)) == 0) {
 664                                 ubsec_dma_sync(&q2->q_mcr,
 665                                     BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
 666                                 break;
 667                         }
 668                         SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, q2, q_next);
 669                         ubsec_callback2(sc, q2);
 670                         /*
 671                          * Don't send any more packet to chip if there has been
 672                          * a DMAERR.
 673                          */
 674                         if (!(stat & BS_STAT_DMAERR))
 675                                 ubsec_feed2(sc);
 676                 }
 677                 mtx_unlock(&sc->sc_mcr2lock);
 678         }
 679
 680         /*
 681          * Check to see if we got any DMA Error
 682          */
 683         if (stat & BS_STAT_DMAERR) {
 684 #ifdef UBSEC_DEBUG
 685                 if (ubsec_debug) {
 686                         volatile u_int32_t a = READ_REG(sc, BS_ERR);
 687
 688                         printf("dmaerr %s@%08x\n",
 689                             (a & BS_ERR_READ) ? "read" : "write",
 690                             a & BS_ERR_ADDR);
 691                 }
 692 #endif /* UBSEC_DEBUG */
 693                 ubsecstats.hst_dmaerr++;
 694                 mtx_lock(&sc->sc_mcr1lock);
 695                 ubsec_totalreset(sc);
 696                 ubsec_feed(sc);
 697                 mtx_unlock(&sc->sc_mcr1lock);
 698         }
 699
 700         if (sc->sc_needwakeup) {                /* XXX check high watermark */
 701                 int wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ);
 702 #ifdef UBSEC_DEBUG
 703                 if (ubsec_debug)
 704                         device_printf(sc->sc_dev, "wakeup crypto (%x)\n",
 705                                 sc->sc_needwakeup);
 706 #endif /* UBSEC_DEBUG */
 707                 sc->sc_needwakeup &= ~wakeup;
 708                 crypto_unblock(sc->sc_cid, wakeup);
 709         }
 710 }
 711
 712 /*
 713  * ubsec_feed() - aggregate and post requests to chip
 714  */
 715 static void
 716 ubsec_feed(struct ubsec_softc *sc)
 717 {
 718         struct ubsec_q *q, *q2;
 719         int npkts, i;
 720         void *v;
 721         u_int32_t stat;
 722
 723         /*
 724          * Decide how many ops to combine in a single MCR.  We cannot
 725          * aggregate more than UBS_MAX_AGGR because this is the number
 726          * of slots defined in the data structure.  Note that
 727          * aggregation only happens if ops are marked batch'able.
 728          * Aggregating ops reduces the number of interrupts to the host
 729          * but also (potentially) increases the latency for processing
 730          * completed ops as we only get an interrupt when all aggregated
 731          * ops have completed.
 732          */
 733         if (sc->sc_nqueue == 0)
 734                 return;
 735         if (sc->sc_nqueue > 1) {
 736                 npkts = 0;
 737                 SIMPLEQ_FOREACH(q, &sc->sc_queue, q_next) {
 738                         npkts++;
 739                         if ((q->q_crp->crp_flags & CRYPTO_F_BATCH) == 0)
 740                                 break;
 741                 }
 742         } else
 743                 npkts = 1;
 744         /*
 745          * Check device status before going any further.
 746          */
 747         if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) {
 748                 if (stat & BS_STAT_DMAERR) {
 749                         ubsec_totalreset(sc);
 750                         ubsecstats.hst_dmaerr++;
 751                 } else
 752                         ubsecstats.hst_mcr1full++;
 753                 return;
 754         }
 755         if (sc->sc_nqueue > ubsecstats.hst_maxqueue)
 756                 ubsecstats.hst_maxqueue = sc->sc_nqueue;
 757         if (npkts > UBS_MAX_AGGR)
 758                 npkts = UBS_MAX_AGGR;
 759         if (npkts < 2)                          /* special case 1 op */
 760                 goto feed1;
 761
 762         ubsecstats.hst_totbatch += npkts-1;
 763 #ifdef UBSEC_DEBUG
 764         if (ubsec_debug)
 765                 printf("merging %d records\n", npkts);
 766 #endif /* UBSEC_DEBUG */
 767
 768         q = SIMPLEQ_FIRST(&sc->sc_queue);
 769         SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q, q_next);
 770         --sc->sc_nqueue;
 771
 772         bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE);
 773         if (q->q_dst_map != NULL)
 774                 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD);
 775
 776         q->q_nstacked_mcrs = npkts - 1;         /* Number of packets stacked */
 777
 778         for (i = 0; i < q->q_nstacked_mcrs; i++) {
 779                 q2 = SIMPLEQ_FIRST(&sc->sc_queue);
 780                 bus_dmamap_sync(sc->sc_dmat, q2->q_src_map,
 781                     BUS_DMASYNC_PREWRITE);
 782                 if (q2->q_dst_map != NULL)
 783                         bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map,
 784                             BUS_DMASYNC_PREREAD);
 785                 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q2, q_next);
 786                 --sc->sc_nqueue;
 787
 788                 v = (void*)(((char *)&q2->q_dma->d_dma->d_mcr) + sizeof(struct ubsec_mcr) -
 789                     sizeof(struct ubsec_mcr_add));
 790                 bcopy(v, &q->q_dma->d_dma->d_mcradd[i], sizeof(struct ubsec_mcr_add));
 791                 q->q_stacked_mcr[i] = q2;
 792         }
 793         q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts);
 794         SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
 795         sc->sc_nqchip += npkts;
 796         if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
 797                 ubsecstats.hst_maxqchip = sc->sc_nqchip;
 798         ubsec_dma_sync(&q->q_dma->d_alloc,
 799             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 800         WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
 801             offsetof(struct ubsec_dmachunk, d_mcr));
 802         return;
 803 feed1:
 804         q = SIMPLEQ_FIRST(&sc->sc_queue);
 805
 806         bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE);
 807         if (q->q_dst_map != NULL)
 808                 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD);
 809         ubsec_dma_sync(&q->q_dma->d_alloc,
 810             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 811
 812         WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
 813             offsetof(struct ubsec_dmachunk, d_mcr));
 814 #ifdef UBSEC_DEBUG
 815         if (ubsec_debug)
 816                 printf("feed1: q->chip %p %08x stat %08x\n",
 817                       q, (u_int32_t)vtophys(&q->q_dma->d_dma->d_mcr),
 818                       stat);
 819 #endif /* UBSEC_DEBUG */
 820         SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q, q_next);
 821         --sc->sc_nqueue;
 822         SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
 823         sc->sc_nqchip++;
 824         if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
 825                 ubsecstats.hst_maxqchip = sc->sc_nqchip;
 826         return;
 827 }
 828
 829 /*
 830  * Allocate a new 'session' and return an encoded session id.  'sidp'
 831  * contains our registration id, and should contain an encoded session
 832  * id on successful allocation.
 833  */
 834 static int
 835 ubsec_newsession(void *arg, u_int32_t *sidp, struct cryptoini *cri)
 836 {
 837         struct cryptoini *c, *encini = NULL, *macini = NULL;
 838         struct ubsec_softc *sc = arg;
 839         struct ubsec_session *ses = NULL;
 840         MD5_CTX md5ctx;
 841         SHA1_CTX sha1ctx;
 842         int i, sesn;
 843
 844         if (sidp == NULL || cri == NULL || sc == NULL)
 845                 return (EINVAL);
 846
 847         for (c = cri; c != NULL; c = c->cri_next) {
 848                 if (c->cri_alg == CRYPTO_MD5_HMAC ||
 849                     c->cri_alg == CRYPTO_SHA1_HMAC) {
 850                         if (macini)
 851                                 return (EINVAL);
 852                         macini = c;
 853                 } else if (c->cri_alg == CRYPTO_DES_CBC ||
 854                     c->cri_alg == CRYPTO_3DES_CBC) {
 855                         if (encini)
 856                                 return (EINVAL);
 857                         encini = c;
 858                 } else
 859                         return (EINVAL);
 860         }
 861         if (encini == NULL && macini == NULL)
 862                 return (EINVAL);
 863
 864         if (sc->sc_sessions == NULL) {
 865                 ses = sc->sc_sessions = (struct ubsec_session *)malloc(
 866                     sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT);
 867                 if (ses == NULL)
 868                         return (ENOMEM);
 869                 sesn = 0;
 870                 sc->sc_nsessions = 1;
 871         } else {
 872                 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
 873                         if (sc->sc_sessions[sesn].ses_used == 0) {
 874                                 ses = &sc->sc_sessions[sesn];
 875                                 break;
 876                         }
 877                 }
 878
 879                 if (ses == NULL) {
 880                         sesn = sc->sc_nsessions;
 881                         ses = (struct ubsec_session *)malloc((sesn + 1) *
 882                             sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT);
 883                         if (ses == NULL)
 884                                 return (ENOMEM);
 885                         bcopy(sc->sc_sessions, ses, sesn *
 886                             sizeof(struct ubsec_session));
 887                         bzero(sc->sc_sessions, sesn *
 888                             sizeof(struct ubsec_session));
 889                         free(sc->sc_sessions, M_DEVBUF);
 890                         sc->sc_sessions = ses;
 891                         ses = &sc->sc_sessions[sesn];
 892                         sc->sc_nsessions++;
 893                 }
 894         }
 895         bzero(ses, sizeof(struct ubsec_session));
 896         ses->ses_used = 1;
 897
 898         if (encini) {
 899                 /* get an IV, network byte order */
 900                 /* XXX may read fewer than requested */
 901                 read_random(ses->ses_iv, sizeof(ses->ses_iv));
 902
 903                 /* Go ahead and compute key in ubsec's byte order */
 904                 if (encini->cri_alg == CRYPTO_DES_CBC) {
 905                         bcopy(encini->cri_key, &ses->ses_deskey[0], 8);
 906                         bcopy(encini->cri_key, &ses->ses_deskey[2], 8);
 907                         bcopy(encini->cri_key, &ses->ses_deskey[4], 8);
 908                 } else
 909                         bcopy(encini->cri_key, ses->ses_deskey, 24);
 910
 911                 SWAP32(ses->ses_deskey[0]);
 912                 SWAP32(ses->ses_deskey[1]);
 913                 SWAP32(ses->ses_deskey[2]);
 914                 SWAP32(ses->ses_deskey[3]);
 915                 SWAP32(ses->ses_deskey[4]);
 916                 SWAP32(ses->ses_deskey[5]);
 917         }
 918
 919         if (macini) {
 920                 ses->ses_mlen = macini->cri_mlen;
 921                 if (ses->ses_mlen == 0) {
 922                         if (macini->cri_alg == CRYPTO_MD5_HMAC)
 923                                 ses->ses_mlen = MD5_DIGEST_LENGTH;
 924                         else
 925                                 ses->ses_mlen = SHA1_RESULTLEN;
 926                 }
 927
 928                 for (i = 0; i < macini->cri_klen / 8; i++)
 929                         macini->cri_key[i] ^= HMAC_IPAD_VAL;
 930
 931                 if (macini->cri_alg == CRYPTO_MD5_HMAC) {
 932                         MD5Init(&md5ctx);
 933                         MD5Update(&md5ctx, macini->cri_key,
 934                             macini->cri_klen / 8);
 935                         MD5Update(&md5ctx, hmac_ipad_buffer,
 936                             HMAC_BLOCK_LEN - (macini->cri_klen / 8));
 937                         bcopy(md5ctx.state, ses->ses_hminner,
 938                             sizeof(md5ctx.state));
 939                 } else {
 940                         SHA1Init(&sha1ctx);
 941                         SHA1Update(&sha1ctx, macini->cri_key,
 942                             macini->cri_klen / 8);
 943                         SHA1Update(&sha1ctx, hmac_ipad_buffer,
 944                             HMAC_BLOCK_LEN - (macini->cri_klen / 8));
 945                         bcopy(sha1ctx.h.b32, ses->ses_hminner,
 946                             sizeof(sha1ctx.h.b32));
 947                 }
 948
 949                 for (i = 0; i < macini->cri_klen / 8; i++)
 950                         macini->cri_key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
 951
 952                 if (macini->cri_alg == CRYPTO_MD5_HMAC) {
 953                         MD5Init(&md5ctx);
 954                         MD5Update(&md5ctx, macini->cri_key,
 955                             macini->cri_klen / 8);
 956                         MD5Update(&md5ctx, hmac_opad_buffer,
 957                             HMAC_BLOCK_LEN - (macini->cri_klen / 8));
 958                         bcopy(md5ctx.state, ses->ses_hmouter,
 959                             sizeof(md5ctx.state));
 960                 } else {
 961                         SHA1Init(&sha1ctx);
 962                         SHA1Update(&sha1ctx, macini->cri_key,
 963                             macini->cri_klen / 8);
 964                         SHA1Update(&sha1ctx, hmac_opad_buffer,
 965                             HMAC_BLOCK_LEN - (macini->cri_klen / 8));
 966                         bcopy(sha1ctx.h.b32, ses->ses_hmouter,
 967                             sizeof(sha1ctx.h.b32));
 968                 }
 969
 970                 for (i = 0; i < macini->cri_klen / 8; i++)
 971                         macini->cri_key[i] ^= HMAC_OPAD_VAL;
 972         }
 973
 974         *sidp = UBSEC_SID(device_get_unit(sc->sc_dev), sesn);
 975         return (0);
 976 }
 977
 978 /*
 979  * Deallocate a session.
 980  */
 981 static int
 982 ubsec_freesession(void *arg, u_int64_t tid)
 983 {
 984         struct ubsec_softc *sc = arg;
 985         int session, ret;
 986         u_int32_t sid = CRYPTO_SESID2LID(tid);
 987
 988         if (sc == NULL)
 989                 return (EINVAL);
 990
 991         session = UBSEC_SESSION(sid);
 992         if (session < sc->sc_nsessions) {
 993                 bzero(&sc->sc_sessions[session],
 994                         sizeof(sc->sc_sessions[session]));
 995                 ret = 0;
 996         } else
 997                 ret = EINVAL;
 998
 999         return (ret);
1000 }
1001
1002 static void
1003 ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error)
1004 {
1005         struct ubsec_operand *op = arg;
1006
1007         KASSERT(nsegs <= UBS_MAX_SCATTER,
1008                 ("Too many DMA segments returned when mapping operand"));
1009 #ifdef UBSEC_DEBUG
1010         if (ubsec_debug)
1011                 printf("ubsec_op_cb: mapsize %u nsegs %d\n",
1012                         (u_int) mapsize, nsegs);
1013 #endif
1014         op->mapsize = mapsize;
1015         op->nsegs = nsegs;
1016         bcopy(seg, op->segs, nsegs * sizeof (seg[0]));
1017 }
1018
1019 static int
1020 ubsec_process(void *arg, struct cryptop *crp, int hint)
1021 {
1022         struct ubsec_q *q = NULL;
1023         int err = 0, i, j, nicealign;
1024         struct ubsec_softc *sc = arg;
1025         struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
1026         int encoffset = 0, macoffset = 0, cpskip, cpoffset;
1027         int sskip, dskip, stheend, dtheend;
1028         int16_t coffset;
1029         struct ubsec_session *ses;
1030         struct ubsec_pktctx ctx;
1031         struct ubsec_dma *dmap = NULL;
1032
1033         if (crp == NULL || crp->crp_callback == NULL || sc == NULL) {
1034                 ubsecstats.hst_invalid++;
1035                 return (EINVAL);
1036         }
1037         if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) {
1038                 ubsecstats.hst_badsession++;
1039                 return (EINVAL);
1040         }
1041
1042         mtx_lock(&sc->sc_freeqlock);
1043         if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
1044                 ubsecstats.hst_queuefull++;
1045                 sc->sc_needwakeup |= CRYPTO_SYMQ;
1046                 mtx_unlock(&sc->sc_freeqlock);
1047                 return (ERESTART);
1048         }
1049         q = SIMPLEQ_FIRST(&sc->sc_freequeue);
1050         SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q, q_next);
1051         mtx_unlock(&sc->sc_freeqlock);
1052
1053         dmap = q->q_dma; /* Save dma pointer */
1054         bzero(q, sizeof(struct ubsec_q));
1055         bzero(&ctx, sizeof(ctx));
1056
1057         q->q_sesn = UBSEC_SESSION(crp->crp_sid);
1058         q->q_dma = dmap;
1059         ses = &sc->sc_sessions[q->q_sesn];
1060
1061         if (crp->crp_flags & CRYPTO_F_IMBUF) {
1062                 q->q_src_m = (struct mbuf *)crp->crp_buf;
1063                 q->q_dst_m = (struct mbuf *)crp->crp_buf;
1064         } else if (crp->crp_flags & CRYPTO_F_IOV) {
1065                 q->q_src_io = (struct uio *)crp->crp_buf;
1066                 q->q_dst_io = (struct uio *)crp->crp_buf;
1067         } else {
1068                 ubsecstats.hst_badflags++;
1069                 err = EINVAL;
1070                 goto errout;    /* XXX we don't handle contiguous blocks! */
1071         }
1072
1073         bzero(&dmap->d_dma->d_mcr, sizeof(struct ubsec_mcr));
1074
1075         dmap->d_dma->d_mcr.mcr_pkts = htole16(1);
1076         dmap->d_dma->d_mcr.mcr_flags = 0;
1077         q->q_crp = crp;
1078
1079         crd1 = crp->crp_desc;
1080         if (crd1 == NULL) {
1081                 ubsecstats.hst_nodesc++;
1082                 err = EINVAL;
1083                 goto errout;
1084         }
1085         crd2 = crd1->crd_next;
1086
1087         if ((crd1->crd_flags & CRD_F_KEY_EXPLICIT) ||
1088             (crd2 != NULL && (crd2->crd_flags & CRD_F_KEY_EXPLICIT))) {
1089                 ubsecstats.hst_badflags++;
1090                 err = EINVAL;
1091                 goto errout;
1092         }
1093
1094         if (crd2 == NULL) {
1095                 if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
1096                     crd1->crd_alg == CRYPTO_SHA1_HMAC) {
1097                         maccrd = crd1;
1098                         enccrd = NULL;
1099                 } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
1100                     crd1->crd_alg == CRYPTO_3DES_CBC) {
1101                         maccrd = NULL;
1102                         enccrd = crd1;
1103                 } else {
1104                         ubsecstats.hst_badalg++;
1105                         err = EINVAL;
1106                         goto errout;
1107                 }
1108         } else {
1109                 if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
1110                     crd1->crd_alg == CRYPTO_SHA1_HMAC) &&
1111                     (crd2->crd_alg == CRYPTO_DES_CBC ||
1112                         crd2->crd_alg == CRYPTO_3DES_CBC) &&
1113                     ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
1114                         maccrd = crd1;
1115                         enccrd = crd2;
1116                 } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
1117                     crd1->crd_alg == CRYPTO_3DES_CBC) &&
1118                     (crd2->crd_alg == CRYPTO_MD5_HMAC ||
1119                         crd2->crd_alg == CRYPTO_SHA1_HMAC) &&
1120                     (crd1->crd_flags & CRD_F_ENCRYPT)) {
1121                         enccrd = crd1;
1122                         maccrd = crd2;
1123                 } else {
1124                         /*
1125                          * We cannot order the ubsec as requested
1126                          */
1127                         ubsecstats.hst_badalg++;
1128                         err = EINVAL;
1129                         goto errout;
1130                 }
1131         }
1132
1133         if (enccrd) {
1134                 encoffset = enccrd->crd_skip;
1135                 ctx.pc_flags |= htole16(UBS_PKTCTX_ENC_3DES);
1136
1137                 if (enccrd->crd_flags & CRD_F_ENCRYPT) {
1138                         q->q_flags |= UBSEC_QFLAGS_COPYOUTIV;
1139
1140                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
1141                                 bcopy(enccrd->crd_iv, ctx.pc_iv, 8);
1142                         else {
1143                                 ctx.pc_iv[0] = ses->ses_iv[0];
1144                                 ctx.pc_iv[1] = ses->ses_iv[1];
1145                         }
1146
1147                         if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
1148                                 if (crp->crp_flags & CRYPTO_F_IMBUF)
1149                                         m_copyback(q->q_src_m,
1150                                             enccrd->crd_inject,
1151                                             8, (caddr_t)ctx.pc_iv);
1152                                 else if (crp->crp_flags & CRYPTO_F_IOV)
1153                                         cuio_copyback(q->q_src_io,
1154                                             enccrd->crd_inject,
1155                                             8, (caddr_t)ctx.pc_iv);
1156                         }
1157                 } else {
1158                         ctx.pc_flags |= htole16(UBS_PKTCTX_INBOUND);
1159
1160                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
1161                                 bcopy(enccrd->crd_iv, ctx.pc_iv, 8);
1162                         else if (crp->crp_flags & CRYPTO_F_IMBUF)
1163                                 m_copydata(q->q_src_m, enccrd->crd_inject,
1164                                     8, (caddr_t)ctx.pc_iv);
1165                         else if (crp->crp_flags & CRYPTO_F_IOV)
1166                                 cuio_copydata(q->q_src_io,
1167                                     enccrd->crd_inject, 8,
1168                                     (caddr_t)ctx.pc_iv);
1169                 }
1170
1171                 ctx.pc_deskey[0] = ses->ses_deskey[0];
1172                 ctx.pc_deskey[1] = ses->ses_deskey[1];
1173                 ctx.pc_deskey[2] = ses->ses_deskey[2];
1174                 ctx.pc_deskey[3] = ses->ses_deskey[3];
1175                 ctx.pc_deskey[4] = ses->ses_deskey[4];
1176                 ctx.pc_deskey[5] = ses->ses_deskey[5];
1177                 SWAP32(ctx.pc_iv[0]);
1178                 SWAP32(ctx.pc_iv[1]);
1179         }
1180
1181         if (maccrd) {
1182                 macoffset = maccrd->crd_skip;
1183
1184                 if (maccrd->crd_alg == CRYPTO_MD5_HMAC)
1185                         ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_MD5);
1186                 else
1187                         ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_SHA1);
1188
1189                 for (i = 0; i < 5; i++) {
1190                         ctx.pc_hminner[i] = ses->ses_hminner[i];
1191                         ctx.pc_hmouter[i] = ses->ses_hmouter[i];
1192
1193                         HTOLE32(ctx.pc_hminner[i]);
1194                         HTOLE32(ctx.pc_hmouter[i]);
1195                 }
1196         }
1197
1198         if (enccrd && maccrd) {
1199                 /*
1200                  * ubsec cannot handle packets where the end of encryption
1201                  * and authentication are not the same, or where the
1202                  * encrypted part begins before the authenticated part.
1203                  */
1204                 if ((encoffset + enccrd->crd_len) !=
1205                     (macoffset + maccrd->crd_len)) {
1206                         ubsecstats.hst_lenmismatch++;
1207                         err = EINVAL;
1208                         goto errout;
1209                 }
1210                 if (enccrd->crd_skip < maccrd->crd_skip) {
1211                         ubsecstats.hst_skipmismatch++;
1212                         err = EINVAL;
1213                         goto errout;
1214                 }
1215                 sskip = maccrd->crd_skip;
1216                 cpskip = dskip = enccrd->crd_skip;
1217                 stheend = maccrd->crd_len;
1218                 dtheend = enccrd->crd_len;
1219                 coffset = enccrd->crd_skip - maccrd->crd_skip;
1220                 cpoffset = cpskip + dtheend;
1221 #ifdef UBSEC_DEBUG
1222                 if (ubsec_debug) {
1223                         printf("mac: skip %d, len %d, inject %d\n",
1224                             maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject);
1225                         printf("enc: skip %d, len %d, inject %d\n",
1226                             enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject);
1227                         printf("src: skip %d, len %d\n", sskip, stheend);
1228                         printf("dst: skip %d, len %d\n", dskip, dtheend);
1229                         printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n",
1230                             coffset, stheend, cpskip, cpoffset);
1231                 }
1232 #endif
1233         } else {
1234                 cpskip = dskip = sskip = macoffset + encoffset;
1235                 dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len;
1236                 cpoffset = cpskip + dtheend;
1237                 coffset = 0;
1238         }
1239         ctx.pc_offset = htole16(coffset >> 2);
1240
1241         if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &q->q_src_map)) {
1242                 ubsecstats.hst_nomap++;
1243                 err = ENOMEM;
1244                 goto errout;
1245         }
1246         if (crp->crp_flags & CRYPTO_F_IMBUF) {
1247                 if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map,
1248                     q->q_src_m, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) {
1249                         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1250                         q->q_src_map = NULL;
1251                         ubsecstats.hst_noload++;
1252                         err = ENOMEM;
1253                         goto errout;
1254                 }
1255         } else if (crp->crp_flags & CRYPTO_F_IOV) {
1256                 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map,
1257                     q->q_src_io, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) {
1258                         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1259                         q->q_src_map = NULL;
1260                         ubsecstats.hst_noload++;
1261                         err = ENOMEM;
1262                         goto errout;
1263                 }
1264         }
1265         nicealign = ubsec_dmamap_aligned(&q->q_src);
1266
1267         dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend);
1268
1269 #ifdef UBSEC_DEBUG
1270         if (ubsec_debug)
1271                 printf("src skip: %d nicealign: %u\n", sskip, nicealign);
1272 #endif
1273         for (i = j = 0; i < q->q_src_nsegs; i++) {
1274                 struct ubsec_pktbuf *pb;
1275                 bus_size_t packl = q->q_src_segs[i].ds_len;
1276                 bus_addr_t packp = q->q_src_segs[i].ds_addr;
1277
1278                 if (sskip >= packl) {
1279                         sskip -= packl;
1280                         continue;
1281                 }
1282
1283                 packl -= sskip;
1284                 packp += sskip;
1285                 sskip = 0;
1286
1287                 if (packl > 0xfffc) {
1288                         err = EIO;
1289                         goto errout;
1290                 }
1291
1292                 if (j == 0)
1293                         pb = &dmap->d_dma->d_mcr.mcr_ipktbuf;
1294                 else
1295                         pb = &dmap->d_dma->d_sbuf[j - 1];
1296
1297                 pb->pb_addr = htole32(packp);
1298
1299                 if (stheend) {
1300                         if (packl > stheend) {
1301                                 pb->pb_len = htole32(stheend);
1302                                 stheend = 0;
1303                         } else {
1304                                 pb->pb_len = htole32(packl);
1305                                 stheend -= packl;
1306                         }
1307                 } else
1308                         pb->pb_len = htole32(packl);
1309
1310                 if ((i + 1) == q->q_src_nsegs)
1311                         pb->pb_next = 0;
1312                 else
1313                         pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1314                             offsetof(struct ubsec_dmachunk, d_sbuf[j]));
1315                 j++;
1316         }
1317
1318         if (enccrd == NULL && maccrd != NULL) {
1319                 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0;
1320                 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0;
1321                 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr +
1322                     offsetof(struct ubsec_dmachunk, d_macbuf[0]));
1323 #ifdef UBSEC_DEBUG
1324                 if (ubsec_debug)
1325                         printf("opkt: %x %x %x\n",
1326                             dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr,
1327                             dmap->d_dma->d_mcr.mcr_opktbuf.pb_len,
1328                             dmap->d_dma->d_mcr.mcr_opktbuf.pb_next);
1329 #endif
1330         } else {
1331                 if (crp->crp_flags & CRYPTO_F_IOV) {
1332                         if (!nicealign) {
1333                                 ubsecstats.hst_iovmisaligned++;
1334                                 err = EINVAL;
1335                                 goto errout;
1336                         }
1337                         if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT,
1338                              &q->q_dst_map)) {
1339                                 ubsecstats.hst_nomap++;
1340                                 err = ENOMEM;
1341                                 goto errout;
1342                         }
1343                         if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map,
1344                             q->q_dst_io, ubsec_op_cb, &q->q_dst, BUS_DMA_NOWAIT) != 0) {
1345                                 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
1346                                 q->q_dst_map = NULL;
1347                                 ubsecstats.hst_noload++;
1348                                 err = ENOMEM;
1349                                 goto errout;
1350                         }
1351                 } else if (crp->crp_flags & CRYPTO_F_IMBUF) {
1352                         if (nicealign) {
1353                                 q->q_dst = q->q_src;
1354                         } else {
1355                                 int totlen, len;
1356                                 struct mbuf *m, *top, **mp;
1357
1358                                 ubsecstats.hst_unaligned++;
1359                                 totlen = q->q_src_mapsize;
1360                                 if (q->q_src_m->m_flags & M_PKTHDR) {
1361                                         len = MHLEN;
1362                                         MGETHDR(m, M_DONTWAIT, MT_DATA);
1363                                         if (m && !m_dup_pkthdr(m, q->q_src_m, M_DONTWAIT)) {
1364                                                 m_free(m);
1365                                                 m = NULL;
1366                                         }
1367                                 } else {
1368                                         len = MLEN;
1369                                         MGET(m, M_DONTWAIT, MT_DATA);
1370                                 }
1371                                 if (m == NULL) {
1372                                         ubsecstats.hst_nombuf++;
1373                                         err = sc->sc_nqueue ? ERESTART : ENOMEM;
1374                                         goto errout;
1375                                 }
1376                                 if (totlen >= MINCLSIZE) {
1377                                         MCLGET(m, M_DONTWAIT);
1378                                         if ((m->m_flags & M_EXT) == 0) {
1379                                                 m_free(m);
1380                                                 ubsecstats.hst_nomcl++;
1381                                                 err = sc->sc_nqueue ? ERESTART : ENOMEM;
1382                                                 goto errout;
1383                                         }
1384                                         len = MCLBYTES;
1385                                 }
1386                                 m->m_len = len;
1387                                 top = NULL;
1388                                 mp = &top;
1389
1390                                 while (totlen > 0) {
1391                                         if (top) {
1392                                                 MGET(m, M_DONTWAIT, MT_DATA);
1393                                                 if (m == NULL) {
1394                                                         m_freem(top);
1395                                                         ubsecstats.hst_nombuf++;
1396                                                         err = sc->sc_nqueue ? ERESTART : ENOMEM;
1397                                                         goto errout;
1398                                                 }
1399                                                 len = MLEN;
1400                                         }
1401                                         if (top && totlen >= MINCLSIZE) {
1402                                                 MCLGET(m, M_DONTWAIT);
1403                                                 if ((m->m_flags & M_EXT) == 0) {
1404                                                         *mp = m;
1405                                                         m_freem(top);
1406                                                         ubsecstats.hst_nomcl++;
1407                                                         err = sc->sc_nqueue ? ERESTART : ENOMEM;
1408                                                         goto errout;
1409                                                 }
1410                                                 len = MCLBYTES;
1411                                         }
1412                                         m->m_len = len = min(totlen, len);
1413                                         totlen -= len;
1414                                         *mp = m;
1415                                         mp = &m->m_next;
1416                                 }
1417                                 q->q_dst_m = top;
1418                                 ubsec_mcopy(q->q_src_m, q->q_dst_m,
1419                                     cpskip, cpoffset);
1420                                 if (bus_dmamap_create(sc->sc_dmat,
1421                                     BUS_DMA_NOWAIT, &q->q_dst_map) != 0) {
1422                                         ubsecstats.hst_nomap++;
1423                                         err = ENOMEM;
1424                                         goto errout;
1425                                 }
1426                                 if (bus_dmamap_load_mbuf(sc->sc_dmat,
1427                                     q->q_dst_map, q->q_dst_m,
1428                                     ubsec_op_cb, &q->q_dst,
1429                                     BUS_DMA_NOWAIT) != 0) {
1430                                         bus_dmamap_destroy(sc->sc_dmat,
1431                                         q->q_dst_map);
1432                                         q->q_dst_map = NULL;
1433                                         ubsecstats.hst_noload++;
1434                                         err = ENOMEM;
1435                                         goto errout;
1436                                 }
1437                         }
1438                 } else {
1439                         ubsecstats.hst_badflags++;
1440                         err = EINVAL;
1441                         goto errout;
1442                 }
1443
1444 #ifdef UBSEC_DEBUG
1445                 if (ubsec_debug)
1446                         printf("dst skip: %d\n", dskip);
1447 #endif
1448                 for (i = j = 0; i < q->q_dst_nsegs; i++) {
1449                         struct ubsec_pktbuf *pb;
1450                         bus_size_t packl = q->q_dst_segs[i].ds_len;
1451                         bus_addr_t packp = q->q_dst_segs[i].ds_addr;
1452
1453                         if (dskip >= packl) {
1454                                 dskip -= packl;
1455                                 continue;
1456                         }
1457
1458                         packl -= dskip;
1459                         packp += dskip;
1460                         dskip = 0;
1461
1462                         if (packl > 0xfffc) {
1463                                 err = EIO;
1464                                 goto errout;
1465                         }
1466
1467                         if (j == 0)
1468                                 pb = &dmap->d_dma->d_mcr.mcr_opktbuf;
1469                         else
1470                                 pb = &dmap->d_dma->d_dbuf[j - 1];
1471
1472                         pb->pb_addr = htole32(packp);
1473
1474                         if (dtheend) {
1475                                 if (packl > dtheend) {
1476                                         pb->pb_len = htole32(dtheend);
1477                                         dtheend = 0;
1478                                 } else {
1479                                         pb->pb_len = htole32(packl);
1480                                         dtheend -= packl;
1481                                 }
1482                         } else
1483                                 pb->pb_len = htole32(packl);
1484
1485                         if ((i + 1) == q->q_dst_nsegs) {
1486                                 if (maccrd)
1487                                         pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1488                                             offsetof(struct ubsec_dmachunk, d_macbuf[0]));
1489                                 else
1490                                         pb->pb_next = 0;
1491                         } else
1492                                 pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1493                                     offsetof(struct ubsec_dmachunk, d_dbuf[j]));
1494                         j++;
1495                 }
1496         }
1497
1498         dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr +
1499             offsetof(struct ubsec_dmachunk, d_ctx));
1500
1501         if (sc->sc_flags & UBS_FLAGS_LONGCTX) {
1502                 struct ubsec_pktctx_long *ctxl;
1503
1504                 ctxl = (struct ubsec_pktctx_long *)(dmap->d_alloc.dma_vaddr +
1505                     offsetof(struct ubsec_dmachunk, d_ctx));
1506
1507                 /* transform small context into long context */
1508                 ctxl->pc_len = htole16(sizeof(struct ubsec_pktctx_long));
1509                 ctxl->pc_type = htole16(UBS_PKTCTX_TYPE_IPSEC);
1510                 ctxl->pc_flags = ctx.pc_flags;
1511                 ctxl->pc_offset = ctx.pc_offset;
1512                 for (i = 0; i < 6; i++)
1513                         ctxl->pc_deskey[i] = ctx.pc_deskey[i];
1514                 for (i = 0; i < 5; i++)
1515                         ctxl->pc_hminner[i] = ctx.pc_hminner[i];
1516                 for (i = 0; i < 5; i++)
1517                         ctxl->pc_hmouter[i] = ctx.pc_hmouter[i];
1518                 ctxl->pc_iv[0] = ctx.pc_iv[0];
1519                 ctxl->pc_iv[1] = ctx.pc_iv[1];
1520         } else
1521                 bcopy(&ctx, dmap->d_alloc.dma_vaddr +
1522                     offsetof(struct ubsec_dmachunk, d_ctx),
1523                     sizeof(struct ubsec_pktctx));
1524
1525         mtx_lock(&sc->sc_mcr1lock);
1526         SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next);
1527         sc->sc_nqueue++;
1528         ubsecstats.hst_ipackets++;
1529         ubsecstats.hst_ibytes += dmap->d_alloc.dma_size;
1530         if ((hint & CRYPTO_HINT_MORE) == 0 || sc->sc_nqueue >= UBS_MAX_AGGR)
1531                 ubsec_feed(sc);
1532         mtx_unlock(&sc->sc_mcr1lock);
1533         return (0);
1534
1535 errout:
1536         if (q != NULL) {
1537                 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
1538                         m_freem(q->q_dst_m);
1539
1540                 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
1541                         bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
1542                         bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
1543                 }
1544                 if (q->q_src_map != NULL) {
1545                         bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
1546                         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1547                 }
1548
1549                 mtx_lock(&sc->sc_freeqlock);
1550                 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
1551                 mtx_unlock(&sc->sc_freeqlock);
1552         }
1553         if (err != ERESTART) {
1554                 crp->crp_etype = err;
1555                 crypto_done(crp);
1556         } else {
1557                 sc->sc_needwakeup |= CRYPTO_SYMQ;
1558         }
1559         return (err);
1560 }
1561
1562 static void
1563 ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q)
1564 {
1565         struct cryptop *crp = (struct cryptop *)q->q_crp;
1566         struct cryptodesc *crd;
1567         struct ubsec_dma *dmap = q->q_dma;
1568
1569         ubsecstats.hst_opackets++;
1570         ubsecstats.hst_obytes += dmap->d_alloc.dma_size;
1571
1572         ubsec_dma_sync(&dmap->d_alloc,
1573             BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
1574         if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
1575                 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map,
1576                     BUS_DMASYNC_POSTREAD);
1577                 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
1578                 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
1579         }
1580         bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_POSTWRITE);
1581         bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
1582         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1583
1584         if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) {
1585                 m_freem(q->q_src_m);
1586                 crp->crp_buf = (caddr_t)q->q_dst_m;
1587         }
1588         ubsecstats.hst_obytes += ((struct mbuf *)crp->crp_buf)->m_len;
1589
1590         /* copy out IV for future use */
1591         if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) {
1592                 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1593                         if (crd->crd_alg != CRYPTO_DES_CBC &&
1594                             crd->crd_alg != CRYPTO_3DES_CBC)
1595                                 continue;
1596                         if (crp->crp_flags & CRYPTO_F_IMBUF)
1597                                 m_copydata((struct mbuf *)crp->crp_buf,
1598                                     crd->crd_skip + crd->crd_len - 8, 8,
1599                                     (caddr_t)sc->sc_sessions[q->q_sesn].ses_iv);
1600                         else if (crp->crp_flags & CRYPTO_F_IOV) {
1601                                 cuio_copydata((struct uio *)crp->crp_buf,
1602                                     crd->crd_skip + crd->crd_len - 8, 8,
1603                                     (caddr_t)sc->sc_sessions[q->q_sesn].ses_iv);
1604                         }
1605                         break;
1606                 }
1607         }
1608
1609         for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1610                 if (crd->crd_alg != CRYPTO_MD5_HMAC &&
1611                     crd->crd_alg != CRYPTO_SHA1_HMAC)
1612                         continue;
1613                 if (crp->crp_flags & CRYPTO_F_IMBUF)
1614                         m_copyback((struct mbuf *)crp->crp_buf,
1615                             crd->crd_inject,
1616                             sc->sc_sessions[q->q_sesn].ses_mlen,
1617                             (caddr_t)dmap->d_dma->d_macbuf);
1618                 else if (crp->crp_flags & CRYPTO_F_IOV && crp->crp_mac)
1619                         bcopy((caddr_t)dmap->d_dma->d_macbuf,
1620                             crp->crp_mac,
1621                             sc->sc_sessions[q->q_sesn].ses_mlen);
1622                 break;
1623         }
1624         mtx_lock(&sc->sc_freeqlock);
1625         SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
1626         mtx_unlock(&sc->sc_freeqlock);
1627         crypto_done(crp);
1628 }
1629
1630 static void
1631 ubsec_mcopy(struct mbuf *srcm, struct mbuf *dstm, int hoffset, int toffset)
1632 {
1633         int i, j, dlen, slen;
1634         caddr_t dptr, sptr;
1635
1636         j = 0;
1637         sptr = srcm->m_data;
1638         slen = srcm->m_len;
1639         dptr = dstm->m_data;
1640         dlen = dstm->m_len;
1641
1642         while (1) {
1643                 for (i = 0; i < min(slen, dlen); i++) {
1644                         if (j < hoffset || j >= toffset)
1645                                 *dptr++ = *sptr++;
1646                         slen--;
1647                         dlen--;
1648                         j++;
1649                 }
1650                 if (slen == 0) {
1651                         srcm = srcm->m_next;
1652                         if (srcm == NULL)
1653                                 return;
1654                         sptr = srcm->m_data;
1655                         slen = srcm->m_len;
1656                 }
1657                 if (dlen == 0) {
1658                         dstm = dstm->m_next;
1659                         if (dstm == NULL)
1660                                 return;
1661                         dptr = dstm->m_data;
1662                         dlen = dstm->m_len;
1663                 }
1664         }
1665 }
1666
1667 /*
1668  * feed the key generator, must be called at splimp() or higher.
1669  */
1670 static int
1671 ubsec_feed2(struct ubsec_softc *sc)
1672 {
1673         struct ubsec_q2 *q;
1674
1675         while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) {
1676                 if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL)
1677                         break;
1678                 q = SIMPLEQ_FIRST(&sc->sc_queue2);
1679
1680                 ubsec_dma_sync(&q->q_mcr,
1681                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1682                 ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_PREWRITE);
1683
1684                 WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr);
1685                 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, q, q_next);
1686                 --sc->sc_nqueue2;
1687                 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next);
1688         }
1689         return (0);
1690 }
1691
1692 /*
1693  * Callback for handling random numbers
1694  */
1695 static void
1696 ubsec_callback2(struct ubsec_softc *sc, struct ubsec_q2 *q)
1697 {
1698         struct cryptkop *krp;
1699         struct ubsec_ctx_keyop *ctx;
1700
1701         ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr;
1702         ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_POSTWRITE);
1703
1704         switch (q->q_type) {
1705 #ifndef UBSEC_NO_RNG
1706         case UBS_CTXOP_RNGBYPASS: {
1707                 struct ubsec_q2_rng *rng = (struct ubsec_q2_rng *)q;
1708
1709                 ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_POSTREAD);
1710                 (*sc->sc_harvest)(sc->sc_rndtest,
1711                         rng->rng_buf.dma_vaddr,
1712                         UBSEC_RNG_BUFSIZ*sizeof (u_int32_t));
1713                 rng->rng_used = 0;
1714                 callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
1715                 break;
1716         }
1717 #endif
1718         case UBS_CTXOP_MODEXP: {
1719                 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q;
1720                 u_int rlen, clen;
1721
1722                 krp = me->me_krp;
1723                 rlen = (me->me_modbits + 7) / 8;
1724                 clen = (krp->krp_param[krp->krp_iparams].crp_nbits + 7) / 8;
1725
1726                 ubsec_dma_sync(&me->me_M, BUS_DMASYNC_POSTWRITE);
1727                 ubsec_dma_sync(&me->me_E, BUS_DMASYNC_POSTWRITE);
1728                 ubsec_dma_sync(&me->me_C, BUS_DMASYNC_POSTREAD);
1729                 ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_POSTWRITE);
1730
1731                 if (clen < rlen)
1732                         krp->krp_status = E2BIG;
1733                 else {
1734                         if (sc->sc_flags & UBS_FLAGS_HWNORM) {
1735                                 bzero(krp->krp_param[krp->krp_iparams].crp_p,
1736                                     (krp->krp_param[krp->krp_iparams].crp_nbits
1737                                         + 7) / 8);
1738                                 bcopy(me->me_C.dma_vaddr,
1739                                     krp->krp_param[krp->krp_iparams].crp_p,
1740                                     (me->me_modbits + 7) / 8);
1741                         } else
1742                                 ubsec_kshift_l(me->me_shiftbits,
1743                                     me->me_C.dma_vaddr, me->me_normbits,
1744                                     krp->krp_param[krp->krp_iparams].crp_p,
1745                                     krp->krp_param[krp->krp_iparams].crp_nbits);
1746                 }
1747
1748                 crypto_kdone(krp);
1749
1750                 /* bzero all potentially sensitive data */
1751                 bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
1752                 bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
1753                 bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
1754                 bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
1755
1756                 /* Can't free here, so put us on the free list. */
1757                 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next);
1758                 break;
1759         }
1760         case UBS_CTXOP_RSAPRIV: {
1761                 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q;
1762                 u_int len;
1763
1764                 krp = rp->rpr_krp;
1765                 ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_POSTWRITE);
1766                 ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_POSTREAD);
1767
1768                 len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + 7) / 8;
1769                 bcopy(rp->rpr_msgout.dma_vaddr,
1770                     krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len);
1771
1772                 crypto_kdone(krp);
1773
1774                 bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size);
1775                 bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size);
1776                 bzero(rp->rpr_q.q_ctx.dma_vaddr, rp->rpr_q.q_ctx.dma_size);
1777
1778                 /* Can't free here, so put us on the free list. */
1779                 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next);
1780                 break;
1781         }
1782         default:
1783                 device_printf(sc->sc_dev, "unknown ctx op: %x\n",
1784                     letoh16(ctx->ctx_op));
1785                 break;
1786         }
1787 }
1788
1789 #ifndef UBSEC_NO_RNG
1790 static void
1791 ubsec_rng(void *vsc)
1792 {
1793         struct ubsec_softc *sc = vsc;
1794         struct ubsec_q2_rng *rng = &sc->sc_rng;
1795         struct ubsec_mcr *mcr;
1796         struct ubsec_ctx_rngbypass *ctx;
1797
1798         mtx_lock(&sc->sc_mcr2lock);
1799         if (rng->rng_used) {
1800                 mtx_unlock(&sc->sc_mcr2lock);
1801                 return;
1802         }
1803         sc->sc_nqueue2++;
1804         if (sc->sc_nqueue2 >= UBS_MAX_NQUEUE)
1805                 goto out;
1806
1807         mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr;
1808         ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr;
1809
1810         mcr->mcr_pkts = htole16(1);
1811         mcr->mcr_flags = 0;
1812         mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr);
1813         mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = 0;
1814         mcr->mcr_ipktbuf.pb_len = 0;
1815         mcr->mcr_reserved = mcr->mcr_pktlen = 0;
1816         mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr);
1817         mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) &
1818             UBS_PKTBUF_LEN);
1819         mcr->mcr_opktbuf.pb_next = 0;
1820
1821         ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass));
1822         ctx->rbp_op = htole16(UBS_CTXOP_RNGBYPASS);
1823         rng->rng_q.q_type = UBS_CTXOP_RNGBYPASS;
1824
1825         ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_PREREAD);
1826
1827         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next);
1828         rng->rng_used = 1;
1829         ubsec_feed2(sc);
1830         ubsecstats.hst_rng++;
1831         mtx_unlock(&sc->sc_mcr2lock);
1832
1833         return;
1834
1835 out:
1836         /*
1837          * Something weird happened, generate our own call back.
1838          */
1839         sc->sc_nqueue2--;
1840         mtx_unlock(&sc->sc_mcr2lock);
1841         callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
1842 }
1843 #endif /* UBSEC_NO_RNG */
1844
1845 static void
1846 ubsec_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
1847 {
1848         bus_addr_t *paddr = (bus_addr_t*) arg;
1849         *paddr = segs->ds_addr;
1850 }
1851
1852 static int
1853 ubsec_dma_malloc(
1854         struct ubsec_softc *sc,
1855         bus_size_t size,
1856         struct ubsec_dma_alloc *dma,
1857         int mapflags
1858 )
1859 {
1860         int r;
1861
1862         /* XXX could specify sc_dmat as parent but that just adds overhead */
1863         r = bus_dma_tag_create(NULL,                    /* parent */
1864                                1, 0,                    /* alignment, bounds */
1865                                BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
1866                                BUS_SPACE_MAXADDR,       /* highaddr */
1867                                NULL, NULL,              /* filter, filterarg */
1868                                size,                    /* maxsize */
1869                                1,                       /* nsegments */
1870                                size,                    /* maxsegsize */
1871                                BUS_DMA_ALLOCNOW,        /* flags */
1872                                NULL, NULL,              /* lockfunc, lockarg */
1873                                &dma->dma_tag);
1874         if (r != 0) {
1875                 device_printf(sc->sc_dev, "ubsec_dma_malloc: "
1876                         "bus_dma_tag_create failed; error %u\n", r);
1877                 goto fail_0;
1878         }
1879
1880         r = bus_dmamap_create(dma->dma_tag, BUS_DMA_NOWAIT, &dma->dma_map);
1881         if (r != 0) {
1882                 device_printf(sc->sc_dev, "ubsec_dma_malloc: "
1883                         "bus_dmamap_create failed; error %u\n", r);
1884                 goto fail_1;
1885         }
1886
1887         r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
1888                              BUS_DMA_NOWAIT, &dma->dma_map);
1889         if (r != 0) {
1890                 device_printf(sc->sc_dev, "ubsec_dma_malloc: "
1891                         "bus_dmammem_alloc failed; size %zu, error %u\n",
1892                         size, r);
1893                 goto fail_2;
1894         }
1895
1896         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
1897                             size,
1898                             ubsec_dmamap_cb,
1899                             &dma->dma_paddr,
1900                             mapflags | BUS_DMA_NOWAIT);
1901         if (r != 0) {
1902                 device_printf(sc->sc_dev, "ubsec_dma_malloc: "
1903                         "bus_dmamap_load failed; error %u\n", r);
1904                 goto fail_3;
1905         }
1906
1907         dma->dma_size = size;
1908         return (0);
1909
1910 fail_3:
1911         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
1912 fail_2:
1913         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1914 fail_1:
1915         bus_dmamap_destroy(dma->dma_tag, dma->dma_map);
1916         bus_dma_tag_destroy(dma->dma_tag);
1917 fail_0:
1918         dma->dma_map = NULL;
1919         dma->dma_tag = NULL;
1920         return (r);
1921 }
1922
1923 static void
1924 ubsec_dma_free(struct ubsec_softc *sc, struct ubsec_dma_alloc *dma)
1925 {
1926         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
1927         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1928         bus_dmamap_destroy(dma->dma_tag, dma->dma_map);
1929         bus_dma_tag_destroy(dma->dma_tag);
1930 }
1931
1932 /*
1933  * Resets the board.  Values in the regesters are left as is
1934  * from the reset (i.e. initial values are assigned elsewhere).
1935  */
1936 static void
1937 ubsec_reset_board(struct ubsec_softc *sc)
1938 {
1939     volatile u_int32_t ctrl;
1940
1941     ctrl = READ_REG(sc, BS_CTRL);
1942     ctrl |= BS_CTRL_RESET;
1943     WRITE_REG(sc, BS_CTRL, ctrl);
1944
1945     /*
1946      * Wait aprox. 30 PCI clocks = 900 ns = 0.9 us
1947      */
1948     DELAY(10);
1949 }
1950
1951 /*
1952  * Init Broadcom registers
1953  */
1954 static void
1955 ubsec_init_board(struct ubsec_softc *sc)
1956 {
1957         u_int32_t ctrl;
1958
1959         ctrl = READ_REG(sc, BS_CTRL);
1960         ctrl &= ~(BS_CTRL_BE32 | BS_CTRL_BE64);
1961         ctrl |= BS_CTRL_LITTLE_ENDIAN | BS_CTRL_MCR1INT;
1962
1963         if (sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG))
1964                 ctrl |= BS_CTRL_MCR2INT;
1965         else
1966                 ctrl &= ~BS_CTRL_MCR2INT;
1967
1968         if (sc->sc_flags & UBS_FLAGS_HWNORM)
1969                 ctrl &= ~BS_CTRL_SWNORM;
1970
1971         WRITE_REG(sc, BS_CTRL, ctrl);
1972 }
1973
1974 /*
1975  * Init Broadcom PCI registers
1976  */
1977 static void
1978 ubsec_init_pciregs(device_t dev)
1979 {
1980 #if 0
1981         u_int32_t misc;
1982
1983         misc = pci_conf_read(pc, pa->pa_tag, BS_RTY_TOUT);
1984         misc = (misc & ~(UBS_PCI_RTY_MASK << UBS_PCI_RTY_SHIFT))
1985             | ((UBS_DEF_RTY & 0xff) << UBS_PCI_RTY_SHIFT);
1986         misc = (misc & ~(UBS_PCI_TOUT_MASK << UBS_PCI_TOUT_SHIFT))
1987             | ((UBS_DEF_TOUT & 0xff) << UBS_PCI_TOUT_SHIFT);
1988         pci_conf_write(pc, pa->pa_tag, BS_RTY_TOUT, misc);
1989 #endif
1990
1991         /*
1992          * This will set the cache line size to 1, this will
1993          * force the BCM58xx chip just to do burst read/writes.
1994          * Cache line read/writes are to slow
1995          */
1996         pci_write_config(dev, PCIR_CACHELNSZ, UBS_DEF_CACHELINE, 1);
1997 }
1998
1999 /*
2000  * Clean up after a chip crash.
2001  * It is assumed that the caller in splimp()
2002  */
2003 static void
2004 ubsec_cleanchip(struct ubsec_softc *sc)
2005 {
2006         struct ubsec_q *q;
2007
2008         while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
2009                 q = SIMPLEQ_FIRST(&sc->sc_qchip);
2010                 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q, q_next);
2011                 ubsec_free_q(sc, q);
2012         }
2013         sc->sc_nqchip = 0;
2014 }
2015
2016 /*
2017  * free a ubsec_q
2018  * It is assumed that the caller is within splimp().
2019  */
2020 static int
2021 ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q)
2022 {
2023         struct ubsec_q *q2;
2024         struct cryptop *crp;
2025         int npkts;
2026         int i;
2027
2028         npkts = q->q_nstacked_mcrs;
2029
2030         for (i = 0; i < npkts; i++) {
2031                 if(q->q_stacked_mcr[i]) {
2032                         q2 = q->q_stacked_mcr[i];
2033
2034                         if ((q2->q_dst_m != NULL) && (q2->q_src_m != q2->q_dst_m))
2035                                 m_freem(q2->q_dst_m);
2036
2037                         crp = (struct cryptop *)q2->q_crp;
2038
2039                         SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next);
2040
2041                         crp->crp_etype = EFAULT;
2042                         crypto_done(crp);
2043                 } else {
2044                         break;
2045                 }
2046         }
2047
2048         /*
2049          * Free header MCR
2050          */
2051         if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
2052                 m_freem(q->q_dst_m);
2053
2054         crp = (struct cryptop *)q->q_crp;
2055
2056         SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
2057
2058         crp->crp_etype = EFAULT;
2059         crypto_done(crp);
2060         return(0);
2061 }
2062
2063 /*
2064  * Routine to reset the chip and clean up.
2065  * It is assumed that the caller is in splimp()
2066  */
2067 static void
2068 ubsec_totalreset(struct ubsec_softc *sc)
2069 {
2070         ubsec_reset_board(sc);
2071         ubsec_init_board(sc);
2072         ubsec_cleanchip(sc);
2073 }
2074
2075 static int
2076 ubsec_dmamap_aligned(struct ubsec_operand *op)
2077 {
2078         int i;
2079
2080         for (i = 0; i < op->nsegs; i++) {
2081                 if (op->segs[i].ds_addr & 3)
2082                         return (0);
2083                 if ((i != (op->nsegs - 1)) &&
2084                     (op->segs[i].ds_len & 3))
2085                         return (0);
2086         }
2087         return (1);
2088 }
2089
2090 static void
2091 ubsec_kfree(struct ubsec_softc *sc, struct ubsec_q2 *q)
2092 {
2093         switch (q->q_type) {
2094         case UBS_CTXOP_MODEXP: {
2095                 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q;
2096
2097                 ubsec_dma_free(sc, &me->me_q.q_mcr);
2098                 ubsec_dma_free(sc, &me->me_q.q_ctx);
2099                 ubsec_dma_free(sc, &me->me_M);
2100                 ubsec_dma_free(sc, &me->me_E);
2101                 ubsec_dma_free(sc, &me->me_C);
2102                 ubsec_dma_free(sc, &me->me_epb);
2103                 free(me, M_DEVBUF);
2104                 break;
2105         }
2106         case UBS_CTXOP_RSAPRIV: {
2107                 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q;
2108
2109                 ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
2110                 ubsec_dma_free(sc, &rp->rpr_q.q_ctx);
2111                 ubsec_dma_free(sc, &rp->rpr_msgin);
2112                 ubsec_dma_free(sc, &rp->rpr_msgout);
2113                 free(rp, M_DEVBUF);
2114                 break;
2115         }
2116         default:
2117                 device_printf(sc->sc_dev, "invalid kfree 0x%x\n", q->q_type);
2118                 break;
2119         }
2120 }
2121
2122 static int
2123 ubsec_kprocess(void *arg, struct cryptkop *krp, int hint)
2124 {
2125         struct ubsec_softc *sc = arg;
2126         int r;
2127
2128         if (krp == NULL || krp->krp_callback == NULL)
2129                 return (EINVAL);
2130
2131         while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) {
2132                 struct ubsec_q2 *q;
2133
2134                 q = SIMPLEQ_FIRST(&sc->sc_q2free);
2135                 SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, q, q_next);
2136                 ubsec_kfree(sc, q);
2137         }
2138
2139         switch (krp->krp_op) {
2140         case CRK_MOD_EXP:
2141                 if (sc->sc_flags & UBS_FLAGS_HWNORM)
2142                         r = ubsec_kprocess_modexp_hw(sc, krp, hint);
2143                 else
2144                         r = ubsec_kprocess_modexp_sw(sc, krp, hint);
2145                 break;
2146         case CRK_MOD_EXP_CRT:
2147                 return (ubsec_kprocess_rsapriv(sc, krp, hint));
2148         default:
2149                 device_printf(sc->sc_dev, "kprocess: invalid op 0x%x\n",
2150                     krp->krp_op);
2151                 krp->krp_status = EOPNOTSUPP;
2152                 crypto_kdone(krp);
2153                 return (0);
2154         }
2155         return (0);                     /* silence compiler */
2156 }
2157
2158 /*
2159  * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization)
2160  */
2161 static int
2162 ubsec_kprocess_modexp_sw(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
2163 {
2164         struct ubsec_q2_modexp *me;
2165         struct ubsec_mcr *mcr;
2166         struct ubsec_ctx_modexp *ctx;
2167         struct ubsec_pktbuf *epb;
2168         int err = 0;
2169         u_int nbits, normbits, mbits, shiftbits, ebits;
2170
2171         me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT);
2172         if (me == NULL) {
2173                 err = ENOMEM;
2174                 goto errout;
2175         }
2176         bzero(me, sizeof *me);
2177         me->me_krp = krp;
2178         me->me_q.q_type = UBS_CTXOP_MODEXP;
2179
2180         nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
2181         if (nbits <= 512)
2182                 normbits = 512;
2183         else if (nbits <= 768)
2184                 normbits = 768;
2185         else if (nbits <= 1024)
2186                 normbits = 1024;
2187         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536)
2188                 normbits = 1536;
2189         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048)
2190                 normbits = 2048;
2191         else {
2192                 err = E2BIG;
2193                 goto errout;
2194         }
2195
2196         shiftbits = normbits - nbits;
2197
2198         me->me_modbits = nbits;
2199         me->me_shiftbits = shiftbits;
2200         me->me_normbits = normbits;
2201
2202         /* Sanity check: result bits must be >= true modulus bits. */
2203         if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
2204                 err = ERANGE;
2205                 goto errout;
2206         }
2207
2208         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2209             &me->me_q.q_mcr, 0)) {
2210                 err = ENOMEM;
2211                 goto errout;
2212         }
2213         mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
2214
2215         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
2216             &me->me_q.q_ctx, 0)) {
2217                 err = ENOMEM;
2218                 goto errout;
2219         }
2220
2221         mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
2222         if (mbits > nbits) {
2223                 err = E2BIG;
2224                 goto errout;
2225         }
2226         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) {
2227                 err = ENOMEM;
2228                 goto errout;
2229         }
2230         ubsec_kshift_r(shiftbits,
2231             krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits,
2232             me->me_M.dma_vaddr, normbits);
2233
2234         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) {
2235                 err = ENOMEM;
2236                 goto errout;
2237         }
2238         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
2239
2240         ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
2241         if (ebits > nbits) {
2242                 err = E2BIG;
2243                 goto errout;
2244         }
2245         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) {
2246                 err = ENOMEM;
2247                 goto errout;
2248         }
2249         ubsec_kshift_r(shiftbits,
2250             krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits,
2251             me->me_E.dma_vaddr, normbits);
2252
2253         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
2254             &me->me_epb, 0)) {
2255                 err = ENOMEM;
2256                 goto errout;
2257         }
2258         epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
2259         epb->pb_addr = htole32(me->me_E.dma_paddr);
2260         epb->pb_next = 0;
2261         epb->pb_len = htole32(normbits / 8);
2262
2263 #ifdef UBSEC_DEBUG
2264         if (ubsec_debug) {
2265                 printf("Epb ");
2266                 ubsec_dump_pb(epb);
2267         }
2268 #endif
2269
2270         mcr->mcr_pkts = htole16(1);
2271         mcr->mcr_flags = 0;
2272         mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
2273         mcr->mcr_reserved = 0;
2274         mcr->mcr_pktlen = 0;
2275
2276         mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
2277         mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8);
2278         mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
2279
2280         mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
2281         mcr->mcr_opktbuf.pb_next = 0;
2282         mcr->mcr_opktbuf.pb_len = htole32(normbits / 8);
2283
2284 #ifdef DIAGNOSTIC
2285         /* Misaligned output buffer will hang the chip. */
2286         if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0)
2287                 panic("%s: modexp invalid addr 0x%x\n",
2288                     device_get_nameunit(sc->sc_dev),
2289                     letoh32(mcr->mcr_opktbuf.pb_addr));
2290         if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0)
2291                 panic("%s: modexp invalid len 0x%x\n",
2292                     device_get_nameunit(sc->sc_dev),
2293                     letoh32(mcr->mcr_opktbuf.pb_len));
2294 #endif
2295
2296         ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
2297         bzero(ctx, sizeof(*ctx));
2298         ubsec_kshift_r(shiftbits,
2299             krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits,
2300             ctx->me_N, normbits);
2301         ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t)));
2302         ctx->me_op = htole16(UBS_CTXOP_MODEXP);
2303         ctx->me_E_len = htole16(nbits);
2304         ctx->me_N_len = htole16(nbits);
2305
2306 #ifdef UBSEC_DEBUG
2307         if (ubsec_debug) {
2308                 ubsec_dump_mcr(mcr);
2309                 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
2310         }
2311 #endif
2312
2313         /*
2314          * ubsec_feed2 will sync mcr and ctx, we just need to sync
2315          * everything else.
2316          */
2317         ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE);
2318         ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE);
2319         ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD);
2320         ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE);
2321
2322         /* Enqueue and we're done... */
2323         mtx_lock(&sc->sc_mcr2lock);
2324         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
2325         ubsec_feed2(sc);
2326         ubsecstats.hst_modexp++;
2327         mtx_unlock(&sc->sc_mcr2lock);
2328
2329         return (0);
2330
2331 errout:
2332         if (me != NULL) {
2333                 if (me->me_q.q_mcr.dma_map != NULL)
2334                         ubsec_dma_free(sc, &me->me_q.q_mcr);
2335                 if (me->me_q.q_ctx.dma_map != NULL) {
2336                         bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
2337                         ubsec_dma_free(sc, &me->me_q.q_ctx);
2338                 }
2339                 if (me->me_M.dma_map != NULL) {
2340                         bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
2341                         ubsec_dma_free(sc, &me->me_M);
2342                 }
2343                 if (me->me_E.dma_map != NULL) {
2344                         bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
2345                         ubsec_dma_free(sc, &me->me_E);
2346                 }
2347                 if (me->me_C.dma_map != NULL) {
2348                         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
2349                         ubsec_dma_free(sc, &me->me_C);
2350                 }
2351                 if (me->me_epb.dma_map != NULL)
2352                         ubsec_dma_free(sc, &me->me_epb);
2353                 free(me, M_DEVBUF);
2354         }
2355         krp->krp_status = err;
2356         crypto_kdone(krp);
2357         return (0);
2358 }
2359
2360 /*
2361  * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization)
2362  */
2363 static int
2364 ubsec_kprocess_modexp_hw(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
2365 {
2366         struct ubsec_q2_modexp *me;
2367         struct ubsec_mcr *mcr;
2368         struct ubsec_ctx_modexp *ctx;
2369         struct ubsec_pktbuf *epb;
2370         int err = 0;
2371         u_int nbits, normbits, mbits, shiftbits, ebits;
2372
2373         me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT);
2374         if (me == NULL) {
2375                 err = ENOMEM;
2376                 goto errout;
2377         }
2378         bzero(me, sizeof *me);
2379         me->me_krp = krp;
2380         me->me_q.q_type = UBS_CTXOP_MODEXP;
2381
2382         nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
2383         if (nbits <= 512)
2384                 normbits = 512;
2385         else if (nbits <= 768)
2386                 normbits = 768;
2387         else if (nbits <= 1024)
2388                 normbits = 1024;
2389         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536)
2390                 normbits = 1536;
2391         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048)
2392                 normbits = 2048;
2393         else {
2394                 err = E2BIG;
2395                 goto errout;
2396         }
2397
2398         shiftbits = normbits - nbits;
2399
2400         /* XXX ??? */
2401         me->me_modbits = nbits;
2402         me->me_shiftbits = shiftbits;
2403         me->me_normbits = normbits;
2404
2405         /* Sanity check: result bits must be >= true modulus bits. */
2406         if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
2407                 err = ERANGE;
2408                 goto errout;
2409         }
2410
2411         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2412             &me->me_q.q_mcr, 0)) {
2413                 err = ENOMEM;
2414                 goto errout;
2415         }
2416         mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
2417
2418         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
2419             &me->me_q.q_ctx, 0)) {
2420                 err = ENOMEM;
2421                 goto errout;
2422         }
2423
2424         mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
2425         if (mbits > nbits) {
2426                 err = E2BIG;
2427                 goto errout;
2428         }
2429         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) {
2430                 err = ENOMEM;
2431                 goto errout;
2432         }
2433         bzero(me->me_M.dma_vaddr, normbits / 8);
2434         bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p,
2435             me->me_M.dma_vaddr, (mbits + 7) / 8);
2436
2437         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) {
2438                 err = ENOMEM;
2439                 goto errout;
2440         }
2441         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
2442
2443         ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
2444         if (ebits > nbits) {
2445                 err = E2BIG;
2446                 goto errout;
2447         }
2448         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) {
2449                 err = ENOMEM;
2450                 goto errout;
2451         }
2452         bzero(me->me_E.dma_vaddr, normbits / 8);
2453         bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p,
2454             me->me_E.dma_vaddr, (ebits + 7) / 8);
2455
2456         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
2457             &me->me_epb, 0)) {
2458                 err = ENOMEM;
2459                 goto errout;
2460         }
2461         epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
2462         epb->pb_addr = htole32(me->me_E.dma_paddr);
2463         epb->pb_next = 0;
2464         epb->pb_len = htole32((ebits + 7) / 8);
2465
2466 #ifdef UBSEC_DEBUG
2467         if (ubsec_debug) {
2468                 printf("Epb ");
2469                 ubsec_dump_pb(epb);
2470         }
2471 #endif
2472
2473         mcr->mcr_pkts = htole16(1);
2474         mcr->mcr_flags = 0;
2475         mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
2476         mcr->mcr_reserved = 0;
2477         mcr->mcr_pktlen = 0;
2478
2479         mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
2480         mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8);
2481         mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
2482
2483         mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
2484         mcr->mcr_opktbuf.pb_next = 0;
2485         mcr->mcr_opktbuf.pb_len = htole32(normbits / 8);
2486
2487 #ifdef DIAGNOSTIC
2488         /* Misaligned output buffer will hang the chip. */
2489         if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0)
2490                 panic("%s: modexp invalid addr 0x%x\n",
2491                     device_get_nameunit(sc->sc_dev),
2492                     letoh32(mcr->mcr_opktbuf.pb_addr));
2493         if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0)
2494                 panic("%s: modexp invalid len 0x%x\n",
2495                     device_get_nameunit(sc->sc_dev),
2496                     letoh32(mcr->mcr_opktbuf.pb_len));
2497 #endif
2498
2499         ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
2500         bzero(ctx, sizeof(*ctx));
2501         bcopy(krp->krp_param[UBS_MODEXP_PAR_N].crp_p, ctx->me_N,
2502             (nbits + 7) / 8);
2503         ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t)));
2504         ctx->me_op = htole16(UBS_CTXOP_MODEXP);
2505         ctx->me_E_len = htole16(ebits);
2506         ctx->me_N_len = htole16(nbits);
2507
2508 #ifdef UBSEC_DEBUG
2509         if (ubsec_debug) {
2510                 ubsec_dump_mcr(mcr);
2511                 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
2512         }
2513 #endif
2514
2515         /*
2516          * ubsec_feed2 will sync mcr and ctx, we just need to sync
2517          * everything else.
2518          */
2519         ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE);
2520         ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE);
2521         ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD);
2522         ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE);
2523
2524         /* Enqueue and we're done... */
2525         mtx_lock(&sc->sc_mcr2lock);
2526         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
2527         ubsec_feed2(sc);
2528         mtx_unlock(&sc->sc_mcr2lock);
2529
2530         return (0);
2531
2532 errout:
2533         if (me != NULL) {
2534                 if (me->me_q.q_mcr.dma_map != NULL)
2535                         ubsec_dma_free(sc, &me->me_q.q_mcr);
2536                 if (me->me_q.q_ctx.dma_map != NULL) {
2537                         bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
2538                         ubsec_dma_free(sc, &me->me_q.q_ctx);
2539                 }
2540                 if (me->me_M.dma_map != NULL) {
2541                         bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
2542                         ubsec_dma_free(sc, &me->me_M);
2543                 }
2544                 if (me->me_E.dma_map != NULL) {
2545                         bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
2546                         ubsec_dma_free(sc, &me->me_E);
2547                 }
2548                 if (me->me_C.dma_map != NULL) {
2549                         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
2550                         ubsec_dma_free(sc, &me->me_C);
2551                 }
2552                 if (me->me_epb.dma_map != NULL)
2553                         ubsec_dma_free(sc, &me->me_epb);
2554                 free(me, M_DEVBUF);
2555         }
2556         krp->krp_status = err;
2557         crypto_kdone(krp);
2558         return (0);
2559 }
2560
2561 static int
2562 ubsec_kprocess_rsapriv(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
2563 {
2564         struct ubsec_q2_rsapriv *rp = NULL;
2565         struct ubsec_mcr *mcr;
2566         struct ubsec_ctx_rsapriv *ctx;
2567         int err = 0;
2568         u_int padlen, msglen;
2569
2570         msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]);
2571         padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]);
2572         if (msglen > padlen)
2573                 padlen = msglen;
2574
2575         if (padlen <= 256)
2576                 padlen = 256;
2577         else if (padlen <= 384)
2578                 padlen = 384;
2579         else if (padlen <= 512)
2580                 padlen = 512;
2581         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 768)
2582                 padlen = 768;
2583         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 1024)
2584                 padlen = 1024;
2585         else {
2586                 err = E2BIG;
2587                 goto errout;
2588         }
2589
2590         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) {
2591                 err = E2BIG;
2592                 goto errout;
2593         }
2594
2595         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) {
2596                 err = E2BIG;
2597                 goto errout;
2598         }
2599
2600         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) {
2601                 err = E2BIG;
2602                 goto errout;
2603         }
2604
2605         rp = (struct ubsec_q2_rsapriv *)malloc(sizeof *rp, M_DEVBUF, M_NOWAIT);
2606         if (rp == NULL)
2607                 return (ENOMEM);
2608         bzero(rp, sizeof *rp);
2609         rp->rpr_krp = krp;
2610         rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV;
2611
2612         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2613             &rp->rpr_q.q_mcr, 0)) {
2614                 err = ENOMEM;
2615                 goto errout;
2616         }
2617         mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr;
2618
2619         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv),
2620             &rp->rpr_q.q_ctx, 0)) {
2621                 err = ENOMEM;
2622                 goto errout;
2623         }
2624         ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr;
2625         bzero(ctx, sizeof *ctx);
2626
2627         /* Copy in p */
2628         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p,
2629             &ctx->rpr_buf[0 * (padlen / 8)],
2630             (krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + 7) / 8);
2631
2632         /* Copy in q */
2633         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p,
2634             &ctx->rpr_buf[1 * (padlen / 8)],
2635             (krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + 7) / 8);
2636
2637         /* Copy in dp */
2638         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p,
2639             &ctx->rpr_buf[2 * (padlen / 8)],
2640             (krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + 7) / 8);
2641
2642         /* Copy in dq */
2643         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p,
2644             &ctx->rpr_buf[3 * (padlen / 8)],
2645             (krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + 7) / 8);
2646
2647         /* Copy in pinv */
2648         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p,
2649             &ctx->rpr_buf[4 * (padlen / 8)],
2650             (krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + 7) / 8);
2651
2652         msglen = padlen * 2;
2653
2654         /* Copy in input message (aligned buffer/length). */
2655         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) {
2656                 /* Is this likely? */
2657                 err = E2BIG;
2658                 goto errout;
2659         }
2660         if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgin, 0)) {
2661                 err = ENOMEM;
2662                 goto errout;
2663         }
2664         bzero(rp->rpr_msgin.dma_vaddr, (msglen + 7) / 8);
2665         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p,
2666             rp->rpr_msgin.dma_vaddr,
2667             (krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + 7) / 8);
2668
2669         /* Prepare space for output message (aligned buffer/length). */
2670         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) {
2671                 /* Is this likely? */
2672                 err = E2BIG;
2673                 goto errout;
2674         }
2675         if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgout, 0)) {
2676                 err = ENOMEM;
2677                 goto errout;
2678         }
2679         bzero(rp->rpr_msgout.dma_vaddr, (msglen + 7) / 8);
2680
2681         mcr->mcr_pkts = htole16(1);
2682         mcr->mcr_flags = 0;
2683         mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr);
2684         mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr);
2685         mcr->mcr_ipktbuf.pb_next = 0;
2686         mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size);
2687         mcr->mcr_reserved = 0;
2688         mcr->mcr_pktlen = htole16(msglen);
2689         mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr);
2690         mcr->mcr_opktbuf.pb_next = 0;
2691         mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size);
2692
2693 #ifdef DIAGNOSTIC
2694         if (rp->rpr_msgin.dma_paddr & 3 || rp->rpr_msgin.dma_size & 3) {
2695                 panic("%s: rsapriv: invalid msgin %x(0x%jx)",
2696                     device_get_nameunit(sc->sc_dev),
2697                     rp->rpr_msgin.dma_paddr, (uintmax_t)rp->rpr_msgin.dma_size);
2698         }
2699         if (rp->rpr_msgout.dma_paddr & 3 || rp->rpr_msgout.dma_size & 3) {
2700                 panic("%s: rsapriv: invalid msgout %x(0x%jx)",
2701                     device_get_nameunit(sc->sc_dev),
2702                     rp->rpr_msgout.dma_paddr, (uintmax_t)rp->rpr_msgout.dma_size);
2703         }
2704 #endif
2705
2706         ctx->rpr_len = (sizeof(u_int16_t) * 4) + (5 * (padlen / 8));
2707         ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV);
2708         ctx->rpr_q_len = htole16(padlen);
2709         ctx->rpr_p_len = htole16(padlen);
2710
2711         /*
2712          * ubsec_feed2 will sync mcr and ctx, we just need to sync
2713          * everything else.
2714          */
2715         ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_PREWRITE);
2716         ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_PREREAD);
2717
2718         /* Enqueue and we're done... */
2719         mtx_lock(&sc->sc_mcr2lock);
2720         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next);
2721         ubsec_feed2(sc);
2722         ubsecstats.hst_modexpcrt++;
2723         mtx_unlock(&sc->sc_mcr2lock);
2724         return (0);
2725
2726 errout:
2727         if (rp != NULL) {
2728                 if (rp->rpr_q.q_mcr.dma_map != NULL)
2729                         ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
2730                 if (rp->rpr_msgin.dma_map != NULL) {
2731                         bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size);
2732                         ubsec_dma_free(sc, &rp->rpr_msgin);
2733                 }
2734                 if (rp->rpr_msgout.dma_map != NULL) {
2735                         bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size);
2736                         ubsec_dma_free(sc, &rp->rpr_msgout);
2737                 }
2738                 free(rp, M_DEVBUF);
2739         }
2740         krp->krp_status = err;
2741         crypto_kdone(krp);
2742         return (0);
2743 }
2744
2745 #ifdef UBSEC_DEBUG
2746 static void
2747 ubsec_dump_pb(volatile struct ubsec_pktbuf *pb)
2748 {
2749         printf("addr 0x%x (0x%x) next 0x%x\n",
2750             pb->pb_addr, pb->pb_len, pb->pb_next);
2751 }
2752
2753 static void
2754 ubsec_dump_ctx2(struct ubsec_ctx_keyop *c)
2755 {
2756         printf("CTX (0x%x):\n", c->ctx_len);
2757         switch (letoh16(c->ctx_op)) {
2758         case UBS_CTXOP_RNGBYPASS:
2759         case UBS_CTXOP_RNGSHA1:
2760                 break;
2761         case UBS_CTXOP_MODEXP:
2762         {
2763                 struct ubsec_ctx_modexp *cx = (void *)c;
2764                 int i, len;
2765
2766                 printf(" Elen %u, Nlen %u\n",
2767                     letoh16(cx->me_E_len), letoh16(cx->me_N_len));
2768                 len = (cx->me_N_len + 7)/8;
2769                 for (i = 0; i < len; i++)
2770                         printf("%s%02x", (i == 0) ? " N: " : ":", cx->me_N[i]);
2771                 printf("\n");
2772                 break;
2773         }
2774         default:
2775                 printf("unknown context: %x\n", c->ctx_op);
2776         }
2777         printf("END CTX\n");
2778 }
2779
2780 static void
2781 ubsec_dump_mcr(struct ubsec_mcr *mcr)
2782 {
2783         volatile struct ubsec_mcr_add *ma;
2784         int i;
2785
2786         printf("MCR:\n");
2787         printf(" pkts: %u, flags 0x%x\n",
2788             letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags));
2789         ma = (volatile struct ubsec_mcr_add *)&mcr->mcr_cmdctxp;
2790         for (i = 0; i < letoh16(mcr->mcr_pkts); i++) {
2791                 printf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i,
2792                     letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen),
2793                     letoh16(ma->mcr_reserved));
2794                 printf(" %d: ipkt ", i);
2795                 ubsec_dump_pb(&ma->mcr_ipktbuf);
2796                 printf(" %d: opkt ", i);
2797                 ubsec_dump_pb(&ma->mcr_opktbuf);
2798                 ma++;
2799         }
2800         printf("END MCR\n");
2801 }
2802 #endif /* UBSEC_DEBUG */
2803
2804 /*
2805  * Return the number of significant bits of a big number.
2806  */
2807 static int
2808 ubsec_ksigbits(struct crparam *cr)
2809 {
2810         u_int plen = (cr->crp_nbits + 7) / 8;
2811         int i, sig = plen * 8;
2812         u_int8_t c, *p = cr->crp_p;
2813
2814         for (i = plen - 1; i >= 0; i--) {
2815                 c = p[i];
2816                 if (c != 0) {
2817                         while ((c & 0x80) == 0) {
2818                                 sig--;
2819                                 c <<= 1;
2820                         }
2821                         break;
2822                 }
2823                 sig -= 8;
2824         }
2825         return (sig);
2826 }
2827
2828 static void
2829 ubsec_kshift_r(
2830         u_int shiftbits,
2831         u_int8_t *src, u_int srcbits,
2832         u_int8_t *dst, u_int dstbits)
2833 {
2834         u_int slen, dlen;
2835         int i, si, di, n;
2836
2837         slen = (srcbits + 7) / 8;
2838         dlen = (dstbits + 7) / 8;
2839
2840         for (i = 0; i < slen; i++)
2841                 dst[i] = src[i];
2842         for (i = 0; i < dlen - slen; i++)
2843                 dst[slen + i] = 0;
2844
2845         n = shiftbits / 8;
2846         if (n != 0) {
2847                 si = dlen - n - 1;
2848                 di = dlen - 1;
2849                 while (si >= 0)
2850                         dst[di--] = dst[si--];
2851                 while (di >= 0)
2852                         dst[di--] = 0;
2853         }
2854
2855         n = shiftbits % 8;
2856         if (n != 0) {
2857                 for (i = dlen - 1; i > 0; i--)
2858                         dst[i] = (dst[i] << n) |
2859                             (dst[i - 1] >> (8 - n));
2860                 dst[0] = dst[0] << n;
2861         }
2862 }
2863
2864 static void
2865 ubsec_kshift_l(
2866         u_int shiftbits,
2867         u_int8_t *src, u_int srcbits,
2868         u_int8_t *dst, u_int dstbits)
2869 {
2870         int slen, dlen, i, n;
2871
2872         slen = (srcbits + 7) / 8;
2873         dlen = (dstbits + 7) / 8;
2874
2875         n = shiftbits / 8;
2876         for (i = 0; i < slen; i++)
2877                 dst[i] = src[i + n];
2878         for (i = 0; i < dlen - slen; i++)
2879                 dst[slen + i] = 0;
2880
2881         n = shiftbits % 8;
2882         if (n != 0) {
2883                 for (i = 0; i < (dlen - 1); i++)
2884                         dst[i] = (dst[i] >> n) | (dst[i + 1] << (8 - n));
2885                 dst[dlen - 1] = dst[dlen - 1] >> n;
2886         }
2887 }