sys/dev/ubsec/ubsec.c

   1 /*      $OpenBSD: ubsec.c,v 1.115 2002/09/24 18:33:26 jason Exp $       */
   2
   3 /*-
   4  * Copyright (c) 2000 Jason L. Wright (jason@thought.net)
   5  * Copyright (c) 2000 Theo de Raadt (deraadt@openbsd.org)
   6  * Copyright (c) 2001 Patrik Lindergren (patrik@ipunplugged.com)
   7  *
   8  * All rights reserved.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  * 3. All advertising materials mentioning features or use of this software
  19  *    must display the following acknowledgement:
  20  *      This product includes software developed by Jason L. Wright
  21  * 4. The name of the author may not be used to endorse or promote products
  22  *    derived from this software without specific prior written permission.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  25  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  26  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  27  * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
  28  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  29  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  30  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  31  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  32  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  33  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  34  * POSSIBILITY OF SUCH DAMAGE.
  35  *
  36  * Effort sponsored in part by the Defense Advanced Research Projects
  37  * Agency (DARPA) and Air Force Research Laboratory, Air Force
  38  * Materiel Command, USAF, under agreement number F30602-01-2-0537.
  39  */
  40
  41 #include <sys/cdefs.h>
  42 __FBSDID("$FreeBSD$");
  43
  44 /*
  45  * uBsec 5[56]01, 58xx hardware crypto accelerator
  46  */
  47
  48 #include "opt_ubsec.h"
  49
  50 #include <sys/param.h>
  51 #include <sys/systm.h>
  52 #include <sys/proc.h>
  53 #include <sys/errno.h>
  54 #include <sys/malloc.h>
  55 #include <sys/kernel.h>
  56 #include <sys/module.h>
  57 #include <sys/mbuf.h>
  58 #include <sys/lock.h>
  59 #include <sys/mutex.h>
  60 #include <sys/sysctl.h>
  61 #include <sys/endian.h>
  62
  63 #include <vm/vm.h>
  64 #include <vm/pmap.h>
  65
  66 #include <machine/bus.h>
  67 #include <machine/resource.h>
  68 #include <sys/bus.h>
  69 #include <sys/rman.h>
  70
  71 #include <crypto/sha1.h>
  72 #include <opencrypto/cryptodev.h>
  73 #include <opencrypto/cryptosoft.h>
  74 #include <sys/md5.h>
  75 #include <sys/random.h>
  76 #include <sys/kobj.h>
  77
  78 #include "cryptodev_if.h"
  79
  80 #include <dev/pci/pcivar.h>
  81 #include <dev/pci/pcireg.h>
  82
  83 /* grr, #defines for gratuitous incompatibility in queue.h */
  84 #define SIMPLEQ_HEAD            STAILQ_HEAD
  85 #define SIMPLEQ_ENTRY           STAILQ_ENTRY
  86 #define SIMPLEQ_INIT            STAILQ_INIT
  87 #define SIMPLEQ_INSERT_TAIL     STAILQ_INSERT_TAIL
  88 #define SIMPLEQ_EMPTY           STAILQ_EMPTY
  89 #define SIMPLEQ_FIRST           STAILQ_FIRST
  90 #define SIMPLEQ_REMOVE_HEAD     STAILQ_REMOVE_HEAD
  91 #define SIMPLEQ_FOREACH         STAILQ_FOREACH
  92 /* ditto for endian.h */
  93 #define letoh16(x)              le16toh(x)
  94 #define letoh32(x)              le32toh(x)
  95
  96 #ifdef UBSEC_RNDTEST
  97 #include <dev/rndtest/rndtest.h>
  98 #endif
  99 #include <dev/ubsec/ubsecreg.h>
 100 #include <dev/ubsec/ubsecvar.h>
 101
 102 /*
 103  * Prototypes and count for the pci_device structure
 104  */
 105 static  int ubsec_probe(device_t);
 106 static  int ubsec_attach(device_t);
 107 static  int ubsec_detach(device_t);
 108 static  int ubsec_suspend(device_t);
 109 static  int ubsec_resume(device_t);
 110 static  int ubsec_shutdown(device_t);
 111
 112 static  int ubsec_newsession(device_t, u_int32_t *, struct cryptoini *);
 113 static  int ubsec_freesession(device_t, u_int64_t);
 114 static  int ubsec_process(device_t, struct cryptop *, int);
 115 static  int ubsec_kprocess(device_t, struct cryptkop *, int);
 116
 117 static device_method_t ubsec_methods[] = {
 118         /* Device interface */
 119         DEVMETHOD(device_probe,         ubsec_probe),
 120         DEVMETHOD(device_attach,        ubsec_attach),
 121         DEVMETHOD(device_detach,        ubsec_detach),
 122         DEVMETHOD(device_suspend,       ubsec_suspend),
 123         DEVMETHOD(device_resume,        ubsec_resume),
 124         DEVMETHOD(device_shutdown,      ubsec_shutdown),
 125
 126         /* crypto device methods */
 127         DEVMETHOD(cryptodev_newsession, ubsec_newsession),
 128         DEVMETHOD(cryptodev_freesession,ubsec_freesession),
 129         DEVMETHOD(cryptodev_process,    ubsec_process),
 130         DEVMETHOD(cryptodev_kprocess,   ubsec_kprocess),
 131
 132         DEVMETHOD_END
 133 };
 134 static driver_t ubsec_driver = {
 135         "ubsec",
 136         ubsec_methods,
 137         sizeof (struct ubsec_softc)
 138 };
 139 static devclass_t ubsec_devclass;
 140
 141 DRIVER_MODULE(ubsec, pci, ubsec_driver, ubsec_devclass, 0, 0);
 142 MODULE_DEPEND(ubsec, crypto, 1, 1, 1);
 143 #ifdef UBSEC_RNDTEST
 144 MODULE_DEPEND(ubsec, rndtest, 1, 1, 1);
 145 #endif
 146
 147 static  void ubsec_intr(void *);
 148 static  void ubsec_callback(struct ubsec_softc *, struct ubsec_q *);
 149 static  void ubsec_feed(struct ubsec_softc *);
 150 static  void ubsec_mcopy(struct mbuf *, struct mbuf *, int, int);
 151 static  void ubsec_callback2(struct ubsec_softc *, struct ubsec_q2 *);
 152 static  int ubsec_feed2(struct ubsec_softc *);
 153 static  void ubsec_rng(void *);
 154 static  int ubsec_dma_malloc(struct ubsec_softc *, bus_size_t,
 155                              struct ubsec_dma_alloc *, int);
 156 #define ubsec_dma_sync(_dma, _flags) \
 157         bus_dmamap_sync((_dma)->dma_tag, (_dma)->dma_map, (_flags))
 158 static  void ubsec_dma_free(struct ubsec_softc *, struct ubsec_dma_alloc *);
 159 static  int ubsec_dmamap_aligned(struct ubsec_operand *op);
 160
 161 static  void ubsec_reset_board(struct ubsec_softc *sc);
 162 static  void ubsec_init_board(struct ubsec_softc *sc);
 163 static  void ubsec_init_pciregs(device_t dev);
 164 static  void ubsec_totalreset(struct ubsec_softc *sc);
 165
 166 static  int ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q);
 167
 168 static  int ubsec_kprocess_modexp_hw(struct ubsec_softc *, struct cryptkop *, int);
 169 static  int ubsec_kprocess_modexp_sw(struct ubsec_softc *, struct cryptkop *, int);
 170 static  int ubsec_kprocess_rsapriv(struct ubsec_softc *, struct cryptkop *, int);
 171 static  void ubsec_kfree(struct ubsec_softc *, struct ubsec_q2 *);
 172 static  int ubsec_ksigbits(struct crparam *);
 173 static  void ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
 174 static  void ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
 175
 176 static SYSCTL_NODE(_hw, OID_AUTO, ubsec, CTLFLAG_RD, 0,
 177     "Broadcom driver parameters");
 178
 179 #ifdef UBSEC_DEBUG
 180 static  void ubsec_dump_pb(volatile struct ubsec_pktbuf *);
 181 static  void ubsec_dump_mcr(struct ubsec_mcr *);
 182 static  void ubsec_dump_ctx2(struct ubsec_ctx_keyop *);
 183
 184 static  int ubsec_debug = 0;
 185 SYSCTL_INT(_hw_ubsec, OID_AUTO, debug, CTLFLAG_RW, &ubsec_debug,
 186             0, "control debugging msgs");
 187 #endif
 188
 189 #define READ_REG(sc,r) \
 190         bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r))
 191
 192 #define WRITE_REG(sc,reg,val) \
 193         bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val)
 194
 195 #define SWAP32(x) (x) = htole32(ntohl((x)))
 196 #define HTOLE32(x) (x) = htole32(x)
 197
 198 struct ubsec_stats ubsecstats;
 199 SYSCTL_STRUCT(_hw_ubsec, OID_AUTO, stats, CTLFLAG_RD, &ubsecstats,
 200             ubsec_stats, "driver statistics");
 201
 202 static int
 203 ubsec_probe(device_t dev)
 204 {
 205         if (pci_get_vendor(dev) == PCI_VENDOR_SUN &&
 206             (pci_get_device(dev) == PCI_PRODUCT_SUN_5821 ||
 207              pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K))
 208                 return (BUS_PROBE_DEFAULT);
 209         if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL &&
 210             (pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5501 ||
 211              pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601))
 212                 return (BUS_PROBE_DEFAULT);
 213         if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 214             (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5801 ||
 215              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 ||
 216              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805 ||
 217              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820 ||
 218              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 ||
 219              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 ||
 220              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823 ||
 221              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5825
 222              ))
 223                 return (BUS_PROBE_DEFAULT);
 224         return (ENXIO);
 225 }
 226
 227 static const char*
 228 ubsec_partname(struct ubsec_softc *sc)
 229 {
 230         /* XXX sprintf numbers when not decoded */
 231         switch (pci_get_vendor(sc->sc_dev)) {
 232         case PCI_VENDOR_BROADCOM:
 233                 switch (pci_get_device(sc->sc_dev)) {
 234                 case PCI_PRODUCT_BROADCOM_5801: return "Broadcom 5801";
 235                 case PCI_PRODUCT_BROADCOM_5802: return "Broadcom 5802";
 236                 case PCI_PRODUCT_BROADCOM_5805: return "Broadcom 5805";
 237                 case PCI_PRODUCT_BROADCOM_5820: return "Broadcom 5820";
 238                 case PCI_PRODUCT_BROADCOM_5821: return "Broadcom 5821";
 239                 case PCI_PRODUCT_BROADCOM_5822: return "Broadcom 5822";
 240                 case PCI_PRODUCT_BROADCOM_5823: return "Broadcom 5823";
 241                 case PCI_PRODUCT_BROADCOM_5825: return "Broadcom 5825";
 242                 }
 243                 return "Broadcom unknown-part";
 244         case PCI_VENDOR_BLUESTEEL:
 245                 switch (pci_get_device(sc->sc_dev)) {
 246                 case PCI_PRODUCT_BLUESTEEL_5601: return "Bluesteel 5601";
 247                 }
 248                 return "Bluesteel unknown-part";
 249         case PCI_VENDOR_SUN:
 250                 switch (pci_get_device(sc->sc_dev)) {
 251                 case PCI_PRODUCT_SUN_5821: return "Sun Crypto 5821";
 252                 case PCI_PRODUCT_SUN_SCA1K: return "Sun Crypto 1K";
 253                 }
 254                 return "Sun unknown-part";
 255         }
 256         return "Unknown-vendor unknown-part";
 257 }
 258
 259 static void
 260 default_harvest(struct rndtest_state *rsp, void *buf, u_int count)
 261 {
 262         random_harvest(buf, count, count*NBBY, 0, RANDOM_PURE);
 263 }
 264
 265 static int
 266 ubsec_attach(device_t dev)
 267 {
 268         struct ubsec_softc *sc = device_get_softc(dev);
 269         struct ubsec_dma *dmap;
 270         u_int32_t i;
 271         int rid;
 272
 273         bzero(sc, sizeof (*sc));
 274         sc->sc_dev = dev;
 275
 276         SIMPLEQ_INIT(&sc->sc_queue);
 277         SIMPLEQ_INIT(&sc->sc_qchip);
 278         SIMPLEQ_INIT(&sc->sc_queue2);
 279         SIMPLEQ_INIT(&sc->sc_qchip2);
 280         SIMPLEQ_INIT(&sc->sc_q2free);
 281
 282         /* XXX handle power management */
 283
 284         sc->sc_statmask = BS_STAT_MCR1_DONE | BS_STAT_DMAERR;
 285
 286         if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL &&
 287             pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601)
 288                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG;
 289
 290         if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 291             (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 ||
 292              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805))
 293                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG;
 294
 295         if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 296             pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820)
 297                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG |
 298                     UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY;
 299
 300         if ((pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 301              (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 ||
 302               pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 ||
 303               pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823 ||
 304               pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5825)) ||
 305             (pci_get_vendor(dev) == PCI_VENDOR_SUN &&
 306              (pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K ||
 307               pci_get_device(dev) == PCI_PRODUCT_SUN_5821))) {
 308                 /* NB: the 5821/5822 defines some additional status bits */
 309                 sc->sc_statmask |= BS_STAT_MCR1_ALLEMPTY |
 310                     BS_STAT_MCR2_ALLEMPTY;
 311                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG |
 312                     UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY;
 313         }
 314
 315         pci_enable_busmaster(dev);
 316
 317         /*
 318          * Setup memory-mapping of PCI registers.
 319          */
 320         rid = BS_BAR;
 321         sc->sc_sr = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 322                                            RF_ACTIVE);
 323         if (sc->sc_sr == NULL) {
 324                 device_printf(dev, "cannot map register space\n");
 325                 goto bad;
 326         }
 327         sc->sc_st = rman_get_bustag(sc->sc_sr);
 328         sc->sc_sh = rman_get_bushandle(sc->sc_sr);
 329
 330         /*
 331          * Arrange interrupt line.
 332          */
 333         rid = 0;
 334         sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 335                                             RF_SHAREABLE|RF_ACTIVE);
 336         if (sc->sc_irq == NULL) {
 337                 device_printf(dev, "could not map interrupt\n");
 338                 goto bad1;
 339         }
 340         /*
 341          * NB: Network code assumes we are blocked with splimp()
 342          *     so make sure the IRQ is mapped appropriately.
 343          */
 344         if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE,
 345                            NULL, ubsec_intr, sc, &sc->sc_ih)) {
 346                 device_printf(dev, "could not establish interrupt\n");
 347                 goto bad2;
 348         }
 349
 350         sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE);
 351         if (sc->sc_cid < 0) {
 352                 device_printf(dev, "could not get crypto driver id\n");
 353                 goto bad3;
 354         }
 355
 356         /*
 357          * Setup DMA descriptor area.
 358          */
 359         if (bus_dma_tag_create(bus_get_dma_tag(dev),    /* parent */
 360                                1, 0,                    /* alignment, bounds */
 361                                BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
 362                                BUS_SPACE_MAXADDR,       /* highaddr */
 363                                NULL, NULL,              /* filter, filterarg */
 364                                0x3ffff,                 /* maxsize */
 365                                UBS_MAX_SCATTER,         /* nsegments */
 366                                0xffff,                  /* maxsegsize */
 367                                BUS_DMA_ALLOCNOW,        /* flags */
 368                                NULL, NULL,              /* lockfunc, lockarg */
 369                                &sc->sc_dmat)) {
 370                 device_printf(dev, "cannot allocate DMA tag\n");
 371                 goto bad4;
 372         }
 373         SIMPLEQ_INIT(&sc->sc_freequeue);
 374         dmap = sc->sc_dmaa;
 375         for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) {
 376                 struct ubsec_q *q;
 377
 378                 q = (struct ubsec_q *)malloc(sizeof(struct ubsec_q),
 379                     M_DEVBUF, M_NOWAIT);
 380                 if (q == NULL) {
 381                         device_printf(dev, "cannot allocate queue buffers\n");
 382                         break;
 383                 }
 384
 385                 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk),
 386                     &dmap->d_alloc, 0)) {
 387                         device_printf(dev, "cannot allocate dma buffers\n");
 388                         free(q, M_DEVBUF);
 389                         break;
 390                 }
 391                 dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr;
 392
 393                 q->q_dma = dmap;
 394                 sc->sc_queuea[i] = q;
 395
 396                 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
 397         }
 398         mtx_init(&sc->sc_mcr1lock, device_get_nameunit(dev),
 399                 "mcr1 operations", MTX_DEF);
 400         mtx_init(&sc->sc_freeqlock, device_get_nameunit(dev),
 401                 "mcr1 free q", MTX_DEF);
 402
 403         device_printf(sc->sc_dev, "%s\n", ubsec_partname(sc));
 404
 405         crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
 406         crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
 407         crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
 408         crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
 409
 410         /*
 411          * Reset Broadcom chip
 412          */
 413         ubsec_reset_board(sc);
 414
 415         /*
 416          * Init Broadcom specific PCI settings
 417          */
 418         ubsec_init_pciregs(dev);
 419
 420         /*
 421          * Init Broadcom chip
 422          */
 423         ubsec_init_board(sc);
 424
 425 #ifndef UBSEC_NO_RNG
 426         if (sc->sc_flags & UBS_FLAGS_RNG) {
 427                 sc->sc_statmask |= BS_STAT_MCR2_DONE;
 428 #ifdef UBSEC_RNDTEST
 429                 sc->sc_rndtest = rndtest_attach(dev);
 430                 if (sc->sc_rndtest)
 431                         sc->sc_harvest = rndtest_harvest;
 432                 else
 433                         sc->sc_harvest = default_harvest;
 434 #else
 435                 sc->sc_harvest = default_harvest;
 436 #endif
 437
 438                 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
 439                     &sc->sc_rng.rng_q.q_mcr, 0))
 440                         goto skip_rng;
 441
 442                 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass),
 443                     &sc->sc_rng.rng_q.q_ctx, 0)) {
 444                         ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
 445                         goto skip_rng;
 446                 }
 447
 448                 if (ubsec_dma_malloc(sc, sizeof(u_int32_t) *
 449                     UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, 0)) {
 450                         ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx);
 451                         ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
 452                         goto skip_rng;
 453                 }
 454
 455                 if (hz >= 100)
 456                         sc->sc_rnghz = hz / 100;
 457                 else
 458                         sc->sc_rnghz = 1;
 459                 callout_init(&sc->sc_rngto, CALLOUT_MPSAFE);
 460                 callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
 461 skip_rng:
 462         ;
 463         }
 464 #endif /* UBSEC_NO_RNG */
 465         mtx_init(&sc->sc_mcr2lock, device_get_nameunit(dev),
 466                 "mcr2 operations", MTX_DEF);
 467
 468         if (sc->sc_flags & UBS_FLAGS_KEY) {
 469                 sc->sc_statmask |= BS_STAT_MCR2_DONE;
 470
 471                 crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0);
 472 #if 0
 473                 crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0);
 474 #endif
 475         }
 476         return (0);
 477 bad4:
 478         crypto_unregister_all(sc->sc_cid);
 479 bad3:
 480         bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
 481 bad2:
 482         bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
 483 bad1:
 484         bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
 485 bad:
 486         return (ENXIO);
 487 }
 488
 489 /*
 490  * Detach a device that successfully probed.
 491  */
 492 static int
 493 ubsec_detach(device_t dev)
 494 {
 495         struct ubsec_softc *sc = device_get_softc(dev);
 496
 497         /* XXX wait/abort active ops */
 498
 499         /* disable interrupts */
 500         WRITE_REG(sc, BS_CTRL, READ_REG(sc, BS_CTRL) &~
 501                 (BS_CTRL_MCR2INT | BS_CTRL_MCR1INT | BS_CTRL_DMAERR));
 502
 503         callout_stop(&sc->sc_rngto);
 504
 505         crypto_unregister_all(sc->sc_cid);
 506
 507 #ifdef UBSEC_RNDTEST
 508         if (sc->sc_rndtest)
 509                 rndtest_detach(sc->sc_rndtest);
 510 #endif
 511
 512         while (!SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
 513                 struct ubsec_q *q;
 514
 515                 q = SIMPLEQ_FIRST(&sc->sc_freequeue);
 516                 SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q_next);
 517                 ubsec_dma_free(sc, &q->q_dma->d_alloc);
 518                 free(q, M_DEVBUF);
 519         }
 520         mtx_destroy(&sc->sc_mcr1lock);
 521         mtx_destroy(&sc->sc_freeqlock);
 522 #ifndef UBSEC_NO_RNG
 523         if (sc->sc_flags & UBS_FLAGS_RNG) {
 524                 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
 525                 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx);
 526                 ubsec_dma_free(sc, &sc->sc_rng.rng_buf);
 527         }
 528 #endif /* UBSEC_NO_RNG */
 529         mtx_destroy(&sc->sc_mcr2lock);
 530
 531         bus_generic_detach(dev);
 532         bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
 533         bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
 534
 535         bus_dma_tag_destroy(sc->sc_dmat);
 536         bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
 537
 538         return (0);
 539 }
 540
 541 /*
 542  * Stop all chip i/o so that the kernel's probe routines don't
 543  * get confused by errant DMAs when rebooting.
 544  */
 545 static int
 546 ubsec_shutdown(device_t dev)
 547 {
 548 #ifdef notyet
 549         ubsec_stop(device_get_softc(dev));
 550 #endif
 551         return (0);
 552 }
 553
 554 /*
 555  * Device suspend routine.
 556  */
 557 static int
 558 ubsec_suspend(device_t dev)
 559 {
 560         struct ubsec_softc *sc = device_get_softc(dev);
 561
 562 #ifdef notyet
 563         /* XXX stop the device and save PCI settings */
 564 #endif
 565         sc->sc_suspended = 1;
 566
 567         return (0);
 568 }
 569
 570 static int
 571 ubsec_resume(device_t dev)
 572 {
 573         struct ubsec_softc *sc = device_get_softc(dev);
 574
 575 #ifdef notyet
 576         /* XXX retore PCI settings and start the device */
 577 #endif
 578         sc->sc_suspended = 0;
 579         return (0);
 580 }
 581
 582 /*
 583  * UBSEC Interrupt routine
 584  */
 585 static void
 586 ubsec_intr(void *arg)
 587 {
 588         struct ubsec_softc *sc = arg;
 589         volatile u_int32_t stat;
 590         struct ubsec_q *q;
 591         struct ubsec_dma *dmap;
 592         int npkts = 0, i;
 593
 594         stat = READ_REG(sc, BS_STAT);
 595         stat &= sc->sc_statmask;
 596         if (stat == 0)
 597                 return;
 598
 599         WRITE_REG(sc, BS_STAT, stat);           /* IACK */
 600
 601         /*
 602          * Check to see if we have any packets waiting for us
 603          */
 604         if ((stat & BS_STAT_MCR1_DONE)) {
 605                 mtx_lock(&sc->sc_mcr1lock);
 606                 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
 607                         q = SIMPLEQ_FIRST(&sc->sc_qchip);
 608                         dmap = q->q_dma;
 609
 610                         if ((dmap->d_dma->d_mcr.mcr_flags & htole16(UBS_MCR_DONE)) == 0)
 611                                 break;
 612
 613                         SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q_next);
 614
 615                         npkts = q->q_nstacked_mcrs;
 616                         sc->sc_nqchip -= 1+npkts;
 617                         /*
 618                          * search for further sc_qchip ubsec_q's that share
 619                          * the same MCR, and complete them too, they must be
 620                          * at the top.
 621                          */
 622                         for (i = 0; i < npkts; i++) {
 623                                 if(q->q_stacked_mcr[i]) {
 624                                         ubsec_callback(sc, q->q_stacked_mcr[i]);
 625                                 } else {
 626                                         break;
 627                                 }
 628                         }
 629                         ubsec_callback(sc, q);
 630                 }
 631                 /*
 632                  * Don't send any more packet to chip if there has been
 633                  * a DMAERR.
 634                  */
 635                 if (!(stat & BS_STAT_DMAERR))
 636                         ubsec_feed(sc);
 637                 mtx_unlock(&sc->sc_mcr1lock);
 638         }
 639
 640         /*
 641          * Check to see if we have any key setups/rng's waiting for us
 642          */
 643         if ((sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) &&
 644             (stat & BS_STAT_MCR2_DONE)) {
 645                 struct ubsec_q2 *q2;
 646                 struct ubsec_mcr *mcr;
 647
 648                 mtx_lock(&sc->sc_mcr2lock);
 649                 while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) {
 650                         q2 = SIMPLEQ_FIRST(&sc->sc_qchip2);
 651
 652                         ubsec_dma_sync(&q2->q_mcr,
 653                             BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
 654
 655                         mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr;
 656                         if ((mcr->mcr_flags & htole16(UBS_MCR_DONE)) == 0) {
 657                                 ubsec_dma_sync(&q2->q_mcr,
 658                                     BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
 659                                 break;
 660                         }
 661                         SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, q_next);
 662                         ubsec_callback2(sc, q2);
 663                         /*
 664                          * Don't send any more packet to chip if there has been
 665                          * a DMAERR.
 666                          */
 667                         if (!(stat & BS_STAT_DMAERR))
 668                                 ubsec_feed2(sc);
 669                 }
 670                 mtx_unlock(&sc->sc_mcr2lock);
 671         }
 672
 673         /*
 674          * Check to see if we got any DMA Error
 675          */
 676         if (stat & BS_STAT_DMAERR) {
 677 #ifdef UBSEC_DEBUG
 678                 if (ubsec_debug) {
 679                         volatile u_int32_t a = READ_REG(sc, BS_ERR);
 680
 681                         printf("dmaerr %s@%08x\n",
 682                             (a & BS_ERR_READ) ? "read" : "write",
 683                             a & BS_ERR_ADDR);
 684                 }
 685 #endif /* UBSEC_DEBUG */
 686                 ubsecstats.hst_dmaerr++;
 687                 mtx_lock(&sc->sc_mcr1lock);
 688                 ubsec_totalreset(sc);
 689                 ubsec_feed(sc);
 690                 mtx_unlock(&sc->sc_mcr1lock);
 691         }
 692
 693         if (sc->sc_needwakeup) {                /* XXX check high watermark */
 694                 int wakeup;
 695
 696                 mtx_lock(&sc->sc_freeqlock);
 697                 wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ);
 698 #ifdef UBSEC_DEBUG
 699                 if (ubsec_debug)
 700                         device_printf(sc->sc_dev, "wakeup crypto (%x)\n",
 701                                 sc->sc_needwakeup);
 702 #endif /* UBSEC_DEBUG */
 703                 sc->sc_needwakeup &= ~wakeup;
 704                 mtx_unlock(&sc->sc_freeqlock);
 705                 crypto_unblock(sc->sc_cid, wakeup);
 706         }
 707 }
 708
 709 /*
 710  * ubsec_feed() - aggregate and post requests to chip
 711  */
 712 static void
 713 ubsec_feed(struct ubsec_softc *sc)
 714 {
 715         struct ubsec_q *q, *q2;
 716         int npkts, i;
 717         void *v;
 718         u_int32_t stat;
 719
 720         /*
 721          * Decide how many ops to combine in a single MCR.  We cannot
 722          * aggregate more than UBS_MAX_AGGR because this is the number
 723          * of slots defined in the data structure.  Note that
 724          * aggregation only happens if ops are marked batch'able.
 725          * Aggregating ops reduces the number of interrupts to the host
 726          * but also (potentially) increases the latency for processing
 727          * completed ops as we only get an interrupt when all aggregated
 728          * ops have completed.
 729          */
 730         if (sc->sc_nqueue == 0)
 731                 return;
 732         if (sc->sc_nqueue > 1) {
 733                 npkts = 0;
 734                 SIMPLEQ_FOREACH(q, &sc->sc_queue, q_next) {
 735                         npkts++;
 736                         if ((q->q_crp->crp_flags & CRYPTO_F_BATCH) == 0)
 737                                 break;
 738                 }
 739         } else
 740                 npkts = 1;
 741         /*
 742          * Check device status before going any further.
 743          */
 744         if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) {
 745                 if (stat & BS_STAT_DMAERR) {
 746                         ubsec_totalreset(sc);
 747                         ubsecstats.hst_dmaerr++;
 748                 } else
 749                         ubsecstats.hst_mcr1full++;
 750                 return;
 751         }
 752         if (sc->sc_nqueue > ubsecstats.hst_maxqueue)
 753                 ubsecstats.hst_maxqueue = sc->sc_nqueue;
 754         if (npkts > UBS_MAX_AGGR)
 755                 npkts = UBS_MAX_AGGR;
 756         if (npkts < 2)                          /* special case 1 op */
 757                 goto feed1;
 758
 759         ubsecstats.hst_totbatch += npkts-1;
 760 #ifdef UBSEC_DEBUG
 761         if (ubsec_debug)
 762                 printf("merging %d records\n", npkts);
 763 #endif /* UBSEC_DEBUG */
 764
 765         q = SIMPLEQ_FIRST(&sc->sc_queue);
 766         SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next);
 767         --sc->sc_nqueue;
 768
 769         bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE);
 770         if (q->q_dst_map != NULL)
 771                 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD);
 772
 773         q->q_nstacked_mcrs = npkts - 1;         /* Number of packets stacked */
 774
 775         for (i = 0; i < q->q_nstacked_mcrs; i++) {
 776                 q2 = SIMPLEQ_FIRST(&sc->sc_queue);
 777                 bus_dmamap_sync(sc->sc_dmat, q2->q_src_map,
 778                     BUS_DMASYNC_PREWRITE);
 779                 if (q2->q_dst_map != NULL)
 780                         bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map,
 781                             BUS_DMASYNC_PREREAD);
 782                 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next);
 783                 --sc->sc_nqueue;
 784
 785                 v = (void*)(((char *)&q2->q_dma->d_dma->d_mcr) + sizeof(struct ubsec_mcr) -
 786                     sizeof(struct ubsec_mcr_add));
 787                 bcopy(v, &q->q_dma->d_dma->d_mcradd[i], sizeof(struct ubsec_mcr_add));
 788                 q->q_stacked_mcr[i] = q2;
 789         }
 790         q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts);
 791         SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
 792         sc->sc_nqchip += npkts;
 793         if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
 794                 ubsecstats.hst_maxqchip = sc->sc_nqchip;
 795         ubsec_dma_sync(&q->q_dma->d_alloc,
 796             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 797         WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
 798             offsetof(struct ubsec_dmachunk, d_mcr));
 799         return;
 800 feed1:
 801         q = SIMPLEQ_FIRST(&sc->sc_queue);
 802
 803         bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE);
 804         if (q->q_dst_map != NULL)
 805                 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD);
 806         ubsec_dma_sync(&q->q_dma->d_alloc,
 807             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 808
 809         WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
 810             offsetof(struct ubsec_dmachunk, d_mcr));
 811 #ifdef UBSEC_DEBUG
 812         if (ubsec_debug)
 813                 printf("feed1: q->chip %p %08x stat %08x\n",
 814                       q, (u_int32_t)vtophys(&q->q_dma->d_dma->d_mcr),
 815                       stat);
 816 #endif /* UBSEC_DEBUG */
 817         SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next);
 818         --sc->sc_nqueue;
 819         SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
 820         sc->sc_nqchip++;
 821         if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
 822                 ubsecstats.hst_maxqchip = sc->sc_nqchip;
 823         return;
 824 }
 825
 826 static void
 827 ubsec_setup_enckey(struct ubsec_session *ses, int algo, caddr_t key)
 828 {
 829
 830         /* Go ahead and compute key in ubsec's byte order */
 831         if (algo == CRYPTO_DES_CBC) {
 832                 bcopy(key, &ses->ses_deskey[0], 8);
 833                 bcopy(key, &ses->ses_deskey[2], 8);
 834                 bcopy(key, &ses->ses_deskey[4], 8);
 835         } else
 836                 bcopy(key, ses->ses_deskey, 24);
 837
 838         SWAP32(ses->ses_deskey[0]);
 839         SWAP32(ses->ses_deskey[1]);
 840         SWAP32(ses->ses_deskey[2]);
 841         SWAP32(ses->ses_deskey[3]);
 842         SWAP32(ses->ses_deskey[4]);
 843         SWAP32(ses->ses_deskey[5]);
 844 }
 845
 846 static void
 847 ubsec_setup_mackey(struct ubsec_session *ses, int algo, caddr_t key, int klen)
 848 {
 849         MD5_CTX md5ctx;
 850         SHA1_CTX sha1ctx;
 851         int i;
 852
 853         for (i = 0; i < klen; i++)
 854                 key[i] ^= HMAC_IPAD_VAL;
 855
 856         if (algo == CRYPTO_MD5_HMAC) {
 857                 MD5Init(&md5ctx);
 858                 MD5Update(&md5ctx, key, klen);
 859                 MD5Update(&md5ctx, hmac_ipad_buffer, MD5_HMAC_BLOCK_LEN - klen);
 860                 bcopy(md5ctx.state, ses->ses_hminner, sizeof(md5ctx.state));
 861         } else {
 862                 SHA1Init(&sha1ctx);
 863                 SHA1Update(&sha1ctx, key, klen);
 864                 SHA1Update(&sha1ctx, hmac_ipad_buffer,
 865                     SHA1_HMAC_BLOCK_LEN - klen);
 866                 bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32));
 867         }
 868
 869         for (i = 0; i < klen; i++)
 870                 key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
 871
 872         if (algo == CRYPTO_MD5_HMAC) {
 873                 MD5Init(&md5ctx);
 874                 MD5Update(&md5ctx, key, klen);
 875                 MD5Update(&md5ctx, hmac_opad_buffer, MD5_HMAC_BLOCK_LEN - klen);
 876                 bcopy(md5ctx.state, ses->ses_hmouter, sizeof(md5ctx.state));
 877         } else {
 878                 SHA1Init(&sha1ctx);
 879                 SHA1Update(&sha1ctx, key, klen);
 880                 SHA1Update(&sha1ctx, hmac_opad_buffer,
 881                     SHA1_HMAC_BLOCK_LEN - klen);
 882                 bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32));
 883         }
 884
 885         for (i = 0; i < klen; i++)
 886                 key[i] ^= HMAC_OPAD_VAL;
 887 }
 888
 889 /*
 890  * Allocate a new 'session' and return an encoded session id.  'sidp'
 891  * contains our registration id, and should contain an encoded session
 892  * id on successful allocation.
 893  */
 894 static int
 895 ubsec_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri)
 896 {
 897         struct ubsec_softc *sc = device_get_softc(dev);
 898         struct cryptoini *c, *encini = NULL, *macini = NULL;
 899         struct ubsec_session *ses = NULL;
 900         int sesn;
 901
 902         if (sidp == NULL || cri == NULL || sc == NULL)
 903                 return (EINVAL);
 904
 905         for (c = cri; c != NULL; c = c->cri_next) {
 906                 if (c->cri_alg == CRYPTO_MD5_HMAC ||
 907                     c->cri_alg == CRYPTO_SHA1_HMAC) {
 908                         if (macini)
 909                                 return (EINVAL);
 910                         macini = c;
 911                 } else if (c->cri_alg == CRYPTO_DES_CBC ||
 912                     c->cri_alg == CRYPTO_3DES_CBC) {
 913                         if (encini)
 914                                 return (EINVAL);
 915                         encini = c;
 916                 } else
 917                         return (EINVAL);
 918         }
 919         if (encini == NULL && macini == NULL)
 920                 return (EINVAL);
 921
 922         if (sc->sc_sessions == NULL) {
 923                 ses = sc->sc_sessions = (struct ubsec_session *)malloc(
 924                     sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT);
 925                 if (ses == NULL)
 926                         return (ENOMEM);
 927                 sesn = 0;
 928                 sc->sc_nsessions = 1;
 929         } else {
 930                 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
 931                         if (sc->sc_sessions[sesn].ses_used == 0) {
 932                                 ses = &sc->sc_sessions[sesn];
 933                                 break;
 934                         }
 935                 }
 936
 937                 if (ses == NULL) {
 938                         sesn = sc->sc_nsessions;
 939                         ses = (struct ubsec_session *)malloc((sesn + 1) *
 940                             sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT);
 941                         if (ses == NULL)
 942                                 return (ENOMEM);
 943                         bcopy(sc->sc_sessions, ses, sesn *
 944                             sizeof(struct ubsec_session));
 945                         bzero(sc->sc_sessions, sesn *
 946                             sizeof(struct ubsec_session));
 947                         free(sc->sc_sessions, M_DEVBUF);
 948                         sc->sc_sessions = ses;
 949                         ses = &sc->sc_sessions[sesn];
 950                         sc->sc_nsessions++;
 951                 }
 952         }
 953         bzero(ses, sizeof(struct ubsec_session));
 954         ses->ses_used = 1;
 955
 956         if (encini) {
 957                 /* get an IV, network byte order */
 958                 /* XXX may read fewer than requested */
 959                 read_random(ses->ses_iv, sizeof(ses->ses_iv));
 960
 961                 if (encini->cri_key != NULL) {
 962                         ubsec_setup_enckey(ses, encini->cri_alg,
 963                             encini->cri_key);
 964                 }
 965         }
 966
 967         if (macini) {
 968                 ses->ses_mlen = macini->cri_mlen;
 969                 if (ses->ses_mlen == 0) {
 970                         if (macini->cri_alg == CRYPTO_MD5_HMAC)
 971                                 ses->ses_mlen = MD5_HASH_LEN;
 972                         else
 973                                 ses->ses_mlen = SHA1_HASH_LEN;
 974                 }
 975
 976                 if (macini->cri_key != NULL) {
 977                         ubsec_setup_mackey(ses, macini->cri_alg,
 978                             macini->cri_key, macini->cri_klen / 8);
 979                 }
 980         }
 981
 982         *sidp = UBSEC_SID(device_get_unit(sc->sc_dev), sesn);
 983         return (0);
 984 }
 985
 986 /*
 987  * Deallocate a session.
 988  */
 989 static int
 990 ubsec_freesession(device_t dev, u_int64_t tid)
 991 {
 992         struct ubsec_softc *sc = device_get_softc(dev);
 993         int session, ret;
 994         u_int32_t sid = CRYPTO_SESID2LID(tid);
 995
 996         if (sc == NULL)
 997                 return (EINVAL);
 998
 999         session = UBSEC_SESSION(sid);
1000         if (session < sc->sc_nsessions) {
1001                 bzero(&sc->sc_sessions[session],
1002                         sizeof(sc->sc_sessions[session]));
1003                 ret = 0;
1004         } else
1005                 ret = EINVAL;
1006
1007         return (ret);
1008 }
1009
1010 static void
1011 ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error)
1012 {
1013         struct ubsec_operand *op = arg;
1014
1015         KASSERT(nsegs <= UBS_MAX_SCATTER,
1016                 ("Too many DMA segments returned when mapping operand"));
1017 #ifdef UBSEC_DEBUG
1018         if (ubsec_debug)
1019                 printf("ubsec_op_cb: mapsize %u nsegs %d error %d\n",
1020                         (u_int) mapsize, nsegs, error);
1021 #endif
1022         if (error != 0)
1023                 return;
1024         op->mapsize = mapsize;
1025         op->nsegs = nsegs;
1026         bcopy(seg, op->segs, nsegs * sizeof (seg[0]));
1027 }
1028
1029 static int
1030 ubsec_process(device_t dev, struct cryptop *crp, int hint)
1031 {
1032         struct ubsec_softc *sc = device_get_softc(dev);
1033         struct ubsec_q *q = NULL;
1034         int err = 0, i, j, nicealign;
1035         struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
1036         int encoffset = 0, macoffset = 0, cpskip, cpoffset;
1037         int sskip, dskip, stheend, dtheend;
1038         int16_t coffset;
1039         struct ubsec_session *ses;
1040         struct ubsec_pktctx ctx;
1041         struct ubsec_dma *dmap = NULL;
1042
1043         if (crp == NULL || crp->crp_callback == NULL || sc == NULL) {
1044                 ubsecstats.hst_invalid++;
1045                 return (EINVAL);
1046         }
1047         if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) {
1048                 ubsecstats.hst_badsession++;
1049                 return (EINVAL);
1050         }
1051
1052         mtx_lock(&sc->sc_freeqlock);
1053         if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
1054                 ubsecstats.hst_queuefull++;
1055                 sc->sc_needwakeup |= CRYPTO_SYMQ;
1056                 mtx_unlock(&sc->sc_freeqlock);
1057                 return (ERESTART);
1058         }
1059         q = SIMPLEQ_FIRST(&sc->sc_freequeue);
1060         SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q_next);
1061         mtx_unlock(&sc->sc_freeqlock);
1062
1063         dmap = q->q_dma; /* Save dma pointer */
1064         bzero(q, sizeof(struct ubsec_q));
1065         bzero(&ctx, sizeof(ctx));
1066
1067         q->q_sesn = UBSEC_SESSION(crp->crp_sid);
1068         q->q_dma = dmap;
1069         ses = &sc->sc_sessions[q->q_sesn];
1070
1071         if (crp->crp_flags & CRYPTO_F_IMBUF) {
1072                 q->q_src_m = (struct mbuf *)crp->crp_buf;
1073                 q->q_dst_m = (struct mbuf *)crp->crp_buf;
1074         } else if (crp->crp_flags & CRYPTO_F_IOV) {
1075                 q->q_src_io = (struct uio *)crp->crp_buf;
1076                 q->q_dst_io = (struct uio *)crp->crp_buf;
1077         } else {
1078                 ubsecstats.hst_badflags++;
1079                 err = EINVAL;
1080                 goto errout;    /* XXX we don't handle contiguous blocks! */
1081         }
1082
1083         bzero(&dmap->d_dma->d_mcr, sizeof(struct ubsec_mcr));
1084
1085         dmap->d_dma->d_mcr.mcr_pkts = htole16(1);
1086         dmap->d_dma->d_mcr.mcr_flags = 0;
1087         q->q_crp = crp;
1088
1089         crd1 = crp->crp_desc;
1090         if (crd1 == NULL) {
1091                 ubsecstats.hst_nodesc++;
1092                 err = EINVAL;
1093                 goto errout;
1094         }
1095         crd2 = crd1->crd_next;
1096
1097         if (crd2 == NULL) {
1098                 if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
1099                     crd1->crd_alg == CRYPTO_SHA1_HMAC) {
1100                         maccrd = crd1;
1101                         enccrd = NULL;
1102                 } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
1103                     crd1->crd_alg == CRYPTO_3DES_CBC) {
1104                         maccrd = NULL;
1105                         enccrd = crd1;
1106                 } else {
1107                         ubsecstats.hst_badalg++;
1108                         err = EINVAL;
1109                         goto errout;
1110                 }
1111         } else {
1112                 if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
1113                     crd1->crd_alg == CRYPTO_SHA1_HMAC) &&
1114                     (crd2->crd_alg == CRYPTO_DES_CBC ||
1115                         crd2->crd_alg == CRYPTO_3DES_CBC) &&
1116                     ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
1117                         maccrd = crd1;
1118                         enccrd = crd2;
1119                 } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
1120                     crd1->crd_alg == CRYPTO_3DES_CBC) &&
1121                     (crd2->crd_alg == CRYPTO_MD5_HMAC ||
1122                         crd2->crd_alg == CRYPTO_SHA1_HMAC) &&
1123                     (crd1->crd_flags & CRD_F_ENCRYPT)) {
1124                         enccrd = crd1;
1125                         maccrd = crd2;
1126                 } else {
1127                         /*
1128                          * We cannot order the ubsec as requested
1129                          */
1130                         ubsecstats.hst_badalg++;
1131                         err = EINVAL;
1132                         goto errout;
1133                 }
1134         }
1135
1136         if (enccrd) {
1137                 if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
1138                         ubsec_setup_enckey(ses, enccrd->crd_alg,
1139                             enccrd->crd_key);
1140                 }
1141
1142                 encoffset = enccrd->crd_skip;
1143                 ctx.pc_flags |= htole16(UBS_PKTCTX_ENC_3DES);
1144
1145                 if (enccrd->crd_flags & CRD_F_ENCRYPT) {
1146                         q->q_flags |= UBSEC_QFLAGS_COPYOUTIV;
1147
1148                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
1149                                 bcopy(enccrd->crd_iv, ctx.pc_iv, 8);
1150                         else {
1151                                 ctx.pc_iv[0] = ses->ses_iv[0];
1152                                 ctx.pc_iv[1] = ses->ses_iv[1];
1153                         }
1154
1155                         if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
1156                                 crypto_copyback(crp->crp_flags, crp->crp_buf,
1157                                     enccrd->crd_inject, 8, (caddr_t)ctx.pc_iv);
1158                         }
1159                 } else {
1160                         ctx.pc_flags |= htole16(UBS_PKTCTX_INBOUND);
1161
1162                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
1163                                 bcopy(enccrd->crd_iv, ctx.pc_iv, 8);
1164                         else {
1165                                 crypto_copydata(crp->crp_flags, crp->crp_buf,
1166                                     enccrd->crd_inject, 8, (caddr_t)ctx.pc_iv);
1167                         }
1168                 }
1169
1170                 ctx.pc_deskey[0] = ses->ses_deskey[0];
1171                 ctx.pc_deskey[1] = ses->ses_deskey[1];
1172                 ctx.pc_deskey[2] = ses->ses_deskey[2];
1173                 ctx.pc_deskey[3] = ses->ses_deskey[3];
1174                 ctx.pc_deskey[4] = ses->ses_deskey[4];
1175                 ctx.pc_deskey[5] = ses->ses_deskey[5];
1176                 SWAP32(ctx.pc_iv[0]);
1177                 SWAP32(ctx.pc_iv[1]);
1178         }
1179
1180         if (maccrd) {
1181                 if (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
1182                         ubsec_setup_mackey(ses, maccrd->crd_alg,
1183                             maccrd->crd_key, maccrd->crd_klen / 8);
1184                 }
1185
1186                 macoffset = maccrd->crd_skip;
1187
1188                 if (maccrd->crd_alg == CRYPTO_MD5_HMAC)
1189                         ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_MD5);
1190                 else
1191                         ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_SHA1);
1192
1193                 for (i = 0; i < 5; i++) {
1194                         ctx.pc_hminner[i] = ses->ses_hminner[i];
1195                         ctx.pc_hmouter[i] = ses->ses_hmouter[i];
1196
1197                         HTOLE32(ctx.pc_hminner[i]);
1198                         HTOLE32(ctx.pc_hmouter[i]);
1199                 }
1200         }
1201
1202         if (enccrd && maccrd) {
1203                 /*
1204                  * ubsec cannot handle packets where the end of encryption
1205                  * and authentication are not the same, or where the
1206                  * encrypted part begins before the authenticated part.
1207                  */
1208                 if ((encoffset + enccrd->crd_len) !=
1209                     (macoffset + maccrd->crd_len)) {
1210                         ubsecstats.hst_lenmismatch++;
1211                         err = EINVAL;
1212                         goto errout;
1213                 }
1214                 if (enccrd->crd_skip < maccrd->crd_skip) {
1215                         ubsecstats.hst_skipmismatch++;
1216                         err = EINVAL;
1217                         goto errout;
1218                 }
1219                 sskip = maccrd->crd_skip;
1220                 cpskip = dskip = enccrd->crd_skip;
1221                 stheend = maccrd->crd_len;
1222                 dtheend = enccrd->crd_len;
1223                 coffset = enccrd->crd_skip - maccrd->crd_skip;
1224                 cpoffset = cpskip + dtheend;
1225 #ifdef UBSEC_DEBUG
1226                 if (ubsec_debug) {
1227                         printf("mac: skip %d, len %d, inject %d\n",
1228                             maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject);
1229                         printf("enc: skip %d, len %d, inject %d\n",
1230                             enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject);
1231                         printf("src: skip %d, len %d\n", sskip, stheend);
1232                         printf("dst: skip %d, len %d\n", dskip, dtheend);
1233                         printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n",
1234                             coffset, stheend, cpskip, cpoffset);
1235                 }
1236 #endif
1237         } else {
1238                 cpskip = dskip = sskip = macoffset + encoffset;
1239                 dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len;
1240                 cpoffset = cpskip + dtheend;
1241                 coffset = 0;
1242         }
1243         ctx.pc_offset = htole16(coffset >> 2);
1244
1245         if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &q->q_src_map)) {
1246                 ubsecstats.hst_nomap++;
1247                 err = ENOMEM;
1248                 goto errout;
1249         }
1250         if (crp->crp_flags & CRYPTO_F_IMBUF) {
1251                 if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map,
1252                     q->q_src_m, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) {
1253                         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1254                         q->q_src_map = NULL;
1255                         ubsecstats.hst_noload++;
1256                         err = ENOMEM;
1257                         goto errout;
1258                 }
1259         } else if (crp->crp_flags & CRYPTO_F_IOV) {
1260                 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map,
1261                     q->q_src_io, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) {
1262                         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1263                         q->q_src_map = NULL;
1264                         ubsecstats.hst_noload++;
1265                         err = ENOMEM;
1266                         goto errout;
1267                 }
1268         }
1269         nicealign = ubsec_dmamap_aligned(&q->q_src);
1270
1271         dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend);
1272
1273 #ifdef UBSEC_DEBUG
1274         if (ubsec_debug)
1275                 printf("src skip: %d nicealign: %u\n", sskip, nicealign);
1276 #endif
1277         for (i = j = 0; i < q->q_src_nsegs; i++) {
1278                 struct ubsec_pktbuf *pb;
1279                 bus_size_t packl = q->q_src_segs[i].ds_len;
1280                 bus_addr_t packp = q->q_src_segs[i].ds_addr;
1281
1282                 if (sskip >= packl) {
1283                         sskip -= packl;
1284                         continue;
1285                 }
1286
1287                 packl -= sskip;
1288                 packp += sskip;
1289                 sskip = 0;
1290
1291                 if (packl > 0xfffc) {
1292                         err = EIO;
1293                         goto errout;
1294                 }
1295
1296                 if (j == 0)
1297                         pb = &dmap->d_dma->d_mcr.mcr_ipktbuf;
1298                 else
1299                         pb = &dmap->d_dma->d_sbuf[j - 1];
1300
1301                 pb->pb_addr = htole32(packp);
1302
1303                 if (stheend) {
1304                         if (packl > stheend) {
1305                                 pb->pb_len = htole32(stheend);
1306                                 stheend = 0;
1307                         } else {
1308                                 pb->pb_len = htole32(packl);
1309                                 stheend -= packl;
1310                         }
1311                 } else
1312                         pb->pb_len = htole32(packl);
1313
1314                 if ((i + 1) == q->q_src_nsegs)
1315                         pb->pb_next = 0;
1316                 else
1317                         pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1318                             offsetof(struct ubsec_dmachunk, d_sbuf[j]));
1319                 j++;
1320         }
1321
1322         if (enccrd == NULL && maccrd != NULL) {
1323                 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0;
1324                 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0;
1325                 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr +
1326                     offsetof(struct ubsec_dmachunk, d_macbuf[0]));
1327 #ifdef UBSEC_DEBUG
1328                 if (ubsec_debug)
1329                         printf("opkt: %x %x %x\n",
1330                             dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr,
1331                             dmap->d_dma->d_mcr.mcr_opktbuf.pb_len,
1332                             dmap->d_dma->d_mcr.mcr_opktbuf.pb_next);
1333 #endif
1334         } else {
1335                 if (crp->crp_flags & CRYPTO_F_IOV) {
1336                         if (!nicealign) {
1337                                 ubsecstats.hst_iovmisaligned++;
1338                                 err = EINVAL;
1339                                 goto errout;
1340                         }
1341                         if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT,
1342                              &q->q_dst_map)) {
1343                                 ubsecstats.hst_nomap++;
1344                                 err = ENOMEM;
1345                                 goto errout;
1346                         }
1347                         if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map,
1348                             q->q_dst_io, ubsec_op_cb, &q->q_dst, BUS_DMA_NOWAIT) != 0) {
1349                                 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
1350                                 q->q_dst_map = NULL;
1351                                 ubsecstats.hst_noload++;
1352                                 err = ENOMEM;
1353                                 goto errout;
1354                         }
1355                 } else if (crp->crp_flags & CRYPTO_F_IMBUF) {
1356                         if (nicealign) {
1357                                 q->q_dst = q->q_src;
1358                         } else {
1359                                 int totlen, len;
1360                                 struct mbuf *m, *top, **mp;
1361
1362                                 ubsecstats.hst_unaligned++;
1363                                 totlen = q->q_src_mapsize;
1364                                 if (totlen >= MINCLSIZE) {
1365                                         m = m_getcl(M_NOWAIT, MT_DATA,
1366                                             q->q_src_m->m_flags & M_PKTHDR);
1367                                         len = MCLBYTES;
1368                                 } else if (q->q_src_m->m_flags & M_PKTHDR) {
1369                                         m = m_gethdr(M_NOWAIT, MT_DATA);
1370                                         len = MHLEN;
1371                                 } else {
1372                                         m = m_get(M_NOWAIT, MT_DATA);
1373                                         len = MLEN;
1374                                 }
1375                                 if (m && q->q_src_m->m_flags & M_PKTHDR &&
1376                                     !m_dup_pkthdr(m, q->q_src_m, M_NOWAIT)) {
1377                                         m_free(m);
1378                                         m = NULL;
1379                                 }
1380                                 if (m == NULL) {
1381                                         ubsecstats.hst_nombuf++;
1382                                         err = sc->sc_nqueue ? ERESTART : ENOMEM;
1383                                         goto errout;
1384                                 }
1385                                 m->m_len = len = min(totlen, len);
1386                                 totlen -= len;
1387                                 top = m;
1388                                 mp = &top;
1389
1390                                 while (totlen > 0) {
1391                                         if (totlen >= MINCLSIZE) {
1392                                                 m = m_getcl(M_NOWAIT,
1393                                                     MT_DATA, 0);
1394                                                 len = MCLBYTES;
1395                                         } else {
1396                                                 m = m_get(M_NOWAIT, MT_DATA);
1397                                                 len = MLEN;
1398                                         }
1399                                         if (m == NULL) {
1400                                                 m_freem(top);
1401                                                 ubsecstats.hst_nombuf++;
1402                                                 err = sc->sc_nqueue ? ERESTART : ENOMEM;
1403                                                 goto errout;
1404                                         }
1405                                         m->m_len = len = min(totlen, len);
1406                                         totlen -= len;
1407                                         *mp = m;
1408                                         mp = &m->m_next;
1409                                 }
1410                                 q->q_dst_m = top;
1411                                 ubsec_mcopy(q->q_src_m, q->q_dst_m,
1412                                     cpskip, cpoffset);
1413                                 if (bus_dmamap_create(sc->sc_dmat,
1414                                     BUS_DMA_NOWAIT, &q->q_dst_map) != 0) {
1415                                         ubsecstats.hst_nomap++;
1416                                         err = ENOMEM;
1417                                         goto errout;
1418                                 }
1419                                 if (bus_dmamap_load_mbuf(sc->sc_dmat,
1420                                     q->q_dst_map, q->q_dst_m,
1421                                     ubsec_op_cb, &q->q_dst,
1422                                     BUS_DMA_NOWAIT) != 0) {
1423                                         bus_dmamap_destroy(sc->sc_dmat,
1424                                         q->q_dst_map);
1425                                         q->q_dst_map = NULL;
1426                                         ubsecstats.hst_noload++;
1427                                         err = ENOMEM;
1428                                         goto errout;
1429                                 }
1430                         }
1431                 } else {
1432                         ubsecstats.hst_badflags++;
1433                         err = EINVAL;
1434                         goto errout;
1435                 }
1436
1437 #ifdef UBSEC_DEBUG
1438                 if (ubsec_debug)
1439                         printf("dst skip: %d\n", dskip);
1440 #endif
1441                 for (i = j = 0; i < q->q_dst_nsegs; i++) {
1442                         struct ubsec_pktbuf *pb;
1443                         bus_size_t packl = q->q_dst_segs[i].ds_len;
1444                         bus_addr_t packp = q->q_dst_segs[i].ds_addr;
1445
1446                         if (dskip >= packl) {
1447                                 dskip -= packl;
1448                                 continue;
1449                         }
1450
1451                         packl -= dskip;
1452                         packp += dskip;
1453                         dskip = 0;
1454
1455                         if (packl > 0xfffc) {
1456                                 err = EIO;
1457                                 goto errout;
1458                         }
1459
1460                         if (j == 0)
1461                                 pb = &dmap->d_dma->d_mcr.mcr_opktbuf;
1462                         else
1463                                 pb = &dmap->d_dma->d_dbuf[j - 1];
1464
1465                         pb->pb_addr = htole32(packp);
1466
1467                         if (dtheend) {
1468                                 if (packl > dtheend) {
1469                                         pb->pb_len = htole32(dtheend);
1470                                         dtheend = 0;
1471                                 } else {
1472                                         pb->pb_len = htole32(packl);
1473                                         dtheend -= packl;
1474                                 }
1475                         } else
1476                                 pb->pb_len = htole32(packl);
1477
1478                         if ((i + 1) == q->q_dst_nsegs) {
1479                                 if (maccrd)
1480                                         pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1481                                             offsetof(struct ubsec_dmachunk, d_macbuf[0]));
1482                                 else
1483                                         pb->pb_next = 0;
1484                         } else
1485                                 pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1486                                     offsetof(struct ubsec_dmachunk, d_dbuf[j]));
1487                         j++;
1488                 }
1489         }
1490
1491         dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr +
1492             offsetof(struct ubsec_dmachunk, d_ctx));
1493
1494         if (sc->sc_flags & UBS_FLAGS_LONGCTX) {
1495                 struct ubsec_pktctx_long *ctxl;
1496
1497                 ctxl = (struct ubsec_pktctx_long *)(dmap->d_alloc.dma_vaddr +
1498                     offsetof(struct ubsec_dmachunk, d_ctx));
1499
1500                 /* transform small context into long context */
1501                 ctxl->pc_len = htole16(sizeof(struct ubsec_pktctx_long));
1502                 ctxl->pc_type = htole16(UBS_PKTCTX_TYPE_IPSEC);
1503                 ctxl->pc_flags = ctx.pc_flags;
1504                 ctxl->pc_offset = ctx.pc_offset;
1505                 for (i = 0; i < 6; i++)
1506                         ctxl->pc_deskey[i] = ctx.pc_deskey[i];
1507                 for (i = 0; i < 5; i++)
1508                         ctxl->pc_hminner[i] = ctx.pc_hminner[i];
1509                 for (i = 0; i < 5; i++)
1510                         ctxl->pc_hmouter[i] = ctx.pc_hmouter[i];
1511                 ctxl->pc_iv[0] = ctx.pc_iv[0];
1512                 ctxl->pc_iv[1] = ctx.pc_iv[1];
1513         } else
1514                 bcopy(&ctx, dmap->d_alloc.dma_vaddr +
1515                     offsetof(struct ubsec_dmachunk, d_ctx),
1516                     sizeof(struct ubsec_pktctx));
1517
1518         mtx_lock(&sc->sc_mcr1lock);
1519         SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next);
1520         sc->sc_nqueue++;
1521         ubsecstats.hst_ipackets++;
1522         ubsecstats.hst_ibytes += dmap->d_alloc.dma_size;
1523         if ((hint & CRYPTO_HINT_MORE) == 0 || sc->sc_nqueue >= UBS_MAX_AGGR)
1524                 ubsec_feed(sc);
1525         mtx_unlock(&sc->sc_mcr1lock);
1526         return (0);
1527
1528 errout:
1529         if (q != NULL) {
1530                 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
1531                         m_freem(q->q_dst_m);
1532
1533                 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
1534                         bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
1535                         bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
1536                 }
1537                 if (q->q_src_map != NULL) {
1538                         bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
1539                         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1540                 }
1541         }
1542         if (q != NULL || err == ERESTART) {
1543                 mtx_lock(&sc->sc_freeqlock);
1544                 if (q != NULL)
1545                         SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
1546                 if (err == ERESTART)
1547                         sc->sc_needwakeup |= CRYPTO_SYMQ;
1548                 mtx_unlock(&sc->sc_freeqlock);
1549         }
1550         if (err != ERESTART) {
1551                 crp->crp_etype = err;
1552                 crypto_done(crp);
1553         }
1554         return (err);
1555 }
1556
1557 static void
1558 ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q)
1559 {
1560         struct cryptop *crp = (struct cryptop *)q->q_crp;
1561         struct cryptodesc *crd;
1562         struct ubsec_dma *dmap = q->q_dma;
1563
1564         ubsecstats.hst_opackets++;
1565         ubsecstats.hst_obytes += dmap->d_alloc.dma_size;
1566
1567         ubsec_dma_sync(&dmap->d_alloc,
1568             BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
1569         if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
1570                 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map,
1571                     BUS_DMASYNC_POSTREAD);
1572                 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
1573                 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
1574         }
1575         bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_POSTWRITE);
1576         bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
1577         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1578
1579         if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) {
1580                 m_freem(q->q_src_m);
1581                 crp->crp_buf = (caddr_t)q->q_dst_m;
1582         }
1583
1584         /* copy out IV for future use */
1585         if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) {
1586                 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1587                         if (crd->crd_alg != CRYPTO_DES_CBC &&
1588                             crd->crd_alg != CRYPTO_3DES_CBC)
1589                                 continue;
1590                         crypto_copydata(crp->crp_flags, crp->crp_buf,
1591                             crd->crd_skip + crd->crd_len - 8, 8,
1592                             (caddr_t)sc->sc_sessions[q->q_sesn].ses_iv);
1593                         break;
1594                 }
1595         }
1596
1597         for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1598                 if (crd->crd_alg != CRYPTO_MD5_HMAC &&
1599                     crd->crd_alg != CRYPTO_SHA1_HMAC)
1600                         continue;
1601                 crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject,
1602                     sc->sc_sessions[q->q_sesn].ses_mlen,
1603                     (caddr_t)dmap->d_dma->d_macbuf);
1604                 break;
1605         }
1606         mtx_lock(&sc->sc_freeqlock);
1607         SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
1608         mtx_unlock(&sc->sc_freeqlock);
1609         crypto_done(crp);
1610 }
1611
1612 static void
1613 ubsec_mcopy(struct mbuf *srcm, struct mbuf *dstm, int hoffset, int toffset)
1614 {
1615         int i, j, dlen, slen;
1616         caddr_t dptr, sptr;
1617
1618         j = 0;
1619         sptr = srcm->m_data;
1620         slen = srcm->m_len;
1621         dptr = dstm->m_data;
1622         dlen = dstm->m_len;
1623
1624         while (1) {
1625                 for (i = 0; i < min(slen, dlen); i++) {
1626                         if (j < hoffset || j >= toffset)
1627                                 *dptr++ = *sptr++;
1628                         slen--;
1629                         dlen--;
1630                         j++;
1631                 }
1632                 if (slen == 0) {
1633                         srcm = srcm->m_next;
1634                         if (srcm == NULL)
1635                                 return;
1636                         sptr = srcm->m_data;
1637                         slen = srcm->m_len;
1638                 }
1639                 if (dlen == 0) {
1640                         dstm = dstm->m_next;
1641                         if (dstm == NULL)
1642                                 return;
1643                         dptr = dstm->m_data;
1644                         dlen = dstm->m_len;
1645                 }
1646         }
1647 }
1648
1649 /*
1650  * feed the key generator, must be called at splimp() or higher.
1651  */
1652 static int
1653 ubsec_feed2(struct ubsec_softc *sc)
1654 {
1655         struct ubsec_q2 *q;
1656
1657         while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) {
1658                 if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL)
1659                         break;
1660                 q = SIMPLEQ_FIRST(&sc->sc_queue2);
1661
1662                 ubsec_dma_sync(&q->q_mcr,
1663                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1664                 ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_PREWRITE);
1665
1666                 WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr);
1667                 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, q_next);
1668                 --sc->sc_nqueue2;
1669                 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next);
1670         }
1671         return (0);
1672 }
1673
1674 /*
1675  * Callback for handling random numbers
1676  */
1677 static void
1678 ubsec_callback2(struct ubsec_softc *sc, struct ubsec_q2 *q)
1679 {
1680         struct cryptkop *krp;
1681         struct ubsec_ctx_keyop *ctx;
1682
1683         ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr;
1684         ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_POSTWRITE);
1685
1686         switch (q->q_type) {
1687 #ifndef UBSEC_NO_RNG
1688         case UBS_CTXOP_RNGBYPASS: {
1689                 struct ubsec_q2_rng *rng = (struct ubsec_q2_rng *)q;
1690
1691                 ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_POSTREAD);
1692                 (*sc->sc_harvest)(sc->sc_rndtest,
1693                         rng->rng_buf.dma_vaddr,
1694                         UBSEC_RNG_BUFSIZ*sizeof (u_int32_t));
1695                 rng->rng_used = 0;
1696                 callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
1697                 break;
1698         }
1699 #endif
1700         case UBS_CTXOP_MODEXP: {
1701                 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q;
1702                 u_int rlen, clen;
1703
1704                 krp = me->me_krp;
1705                 rlen = (me->me_modbits + 7) / 8;
1706                 clen = (krp->krp_param[krp->krp_iparams].crp_nbits + 7) / 8;
1707
1708                 ubsec_dma_sync(&me->me_M, BUS_DMASYNC_POSTWRITE);
1709                 ubsec_dma_sync(&me->me_E, BUS_DMASYNC_POSTWRITE);
1710                 ubsec_dma_sync(&me->me_C, BUS_DMASYNC_POSTREAD);
1711                 ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_POSTWRITE);
1712
1713                 if (clen < rlen)
1714                         krp->krp_status = E2BIG;
1715                 else {
1716                         if (sc->sc_flags & UBS_FLAGS_HWNORM) {
1717                                 bzero(krp->krp_param[krp->krp_iparams].crp_p,
1718                                     (krp->krp_param[krp->krp_iparams].crp_nbits
1719                                         + 7) / 8);
1720                                 bcopy(me->me_C.dma_vaddr,
1721                                     krp->krp_param[krp->krp_iparams].crp_p,
1722                                     (me->me_modbits + 7) / 8);
1723                         } else
1724                                 ubsec_kshift_l(me->me_shiftbits,
1725                                     me->me_C.dma_vaddr, me->me_normbits,
1726                                     krp->krp_param[krp->krp_iparams].crp_p,
1727                                     krp->krp_param[krp->krp_iparams].crp_nbits);
1728                 }
1729
1730                 crypto_kdone(krp);
1731
1732                 /* bzero all potentially sensitive data */
1733                 bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
1734                 bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
1735                 bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
1736                 bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
1737
1738                 /* Can't free here, so put us on the free list. */
1739                 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next);
1740                 break;
1741         }
1742         case UBS_CTXOP_RSAPRIV: {
1743                 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q;
1744                 u_int len;
1745
1746                 krp = rp->rpr_krp;
1747                 ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_POSTWRITE);
1748                 ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_POSTREAD);
1749
1750                 len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + 7) / 8;
1751                 bcopy(rp->rpr_msgout.dma_vaddr,
1752                     krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len);
1753
1754                 crypto_kdone(krp);
1755
1756                 bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size);
1757                 bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size);
1758                 bzero(rp->rpr_q.q_ctx.dma_vaddr, rp->rpr_q.q_ctx.dma_size);
1759
1760                 /* Can't free here, so put us on the free list. */
1761                 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next);
1762                 break;
1763         }
1764         default:
1765                 device_printf(sc->sc_dev, "unknown ctx op: %x\n",
1766                     letoh16(ctx->ctx_op));
1767                 break;
1768         }
1769 }
1770
1771 #ifndef UBSEC_NO_RNG
1772 static void
1773 ubsec_rng(void *vsc)
1774 {
1775         struct ubsec_softc *sc = vsc;
1776         struct ubsec_q2_rng *rng = &sc->sc_rng;
1777         struct ubsec_mcr *mcr;
1778         struct ubsec_ctx_rngbypass *ctx;
1779
1780         mtx_lock(&sc->sc_mcr2lock);
1781         if (rng->rng_used) {
1782                 mtx_unlock(&sc->sc_mcr2lock);
1783                 return;
1784         }
1785         sc->sc_nqueue2++;
1786         if (sc->sc_nqueue2 >= UBS_MAX_NQUEUE)
1787                 goto out;
1788
1789         mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr;
1790         ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr;
1791
1792         mcr->mcr_pkts = htole16(1);
1793         mcr->mcr_flags = 0;
1794         mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr);
1795         mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = 0;
1796         mcr->mcr_ipktbuf.pb_len = 0;
1797         mcr->mcr_reserved = mcr->mcr_pktlen = 0;
1798         mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr);
1799         mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) &
1800             UBS_PKTBUF_LEN);
1801         mcr->mcr_opktbuf.pb_next = 0;
1802
1803         ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass));
1804         ctx->rbp_op = htole16(UBS_CTXOP_RNGBYPASS);
1805         rng->rng_q.q_type = UBS_CTXOP_RNGBYPASS;
1806
1807         ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_PREREAD);
1808
1809         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next);
1810         rng->rng_used = 1;
1811         ubsec_feed2(sc);
1812         ubsecstats.hst_rng++;
1813         mtx_unlock(&sc->sc_mcr2lock);
1814
1815         return;
1816
1817 out:
1818         /*
1819          * Something weird happened, generate our own call back.
1820          */
1821         sc->sc_nqueue2--;
1822         mtx_unlock(&sc->sc_mcr2lock);
1823         callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
1824 }
1825 #endif /* UBSEC_NO_RNG */
1826
1827 static void
1828 ubsec_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
1829 {
1830         bus_addr_t *paddr = (bus_addr_t*) arg;
1831         *paddr = segs->ds_addr;
1832 }
1833
1834 static int
1835 ubsec_dma_malloc(
1836         struct ubsec_softc *sc,
1837         bus_size_t size,
1838         struct ubsec_dma_alloc *dma,
1839         int mapflags
1840 )
1841 {
1842         int r;
1843
1844         /* XXX could specify sc_dmat as parent but that just adds overhead */
1845         r = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev),     /* parent */
1846                                1, 0,                    /* alignment, bounds */
1847                                BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
1848                                BUS_SPACE_MAXADDR,       /* highaddr */
1849                                NULL, NULL,              /* filter, filterarg */
1850                                size,                    /* maxsize */
1851                                1,                       /* nsegments */
1852                                size,                    /* maxsegsize */
1853                                BUS_DMA_ALLOCNOW,        /* flags */
1854                                NULL, NULL,              /* lockfunc, lockarg */
1855                                &dma->dma_tag);
1856         if (r != 0) {
1857                 device_printf(sc->sc_dev, "ubsec_dma_malloc: "
1858                         "bus_dma_tag_create failed; error %u\n", r);
1859                 goto fail_0;
1860         }
1861
1862         r = bus_dmamap_create(dma->dma_tag, BUS_DMA_NOWAIT, &dma->dma_map);
1863         if (r != 0) {
1864                 device_printf(sc->sc_dev, "ubsec_dma_malloc: "
1865                         "bus_dmamap_create failed; error %u\n", r);
1866                 goto fail_1;
1867         }
1868
1869         r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
1870                              BUS_DMA_NOWAIT, &dma->dma_map);
1871         if (r != 0) {
1872                 device_printf(sc->sc_dev, "ubsec_dma_malloc: "
1873                         "bus_dmammem_alloc failed; size %ju, error %u\n",
1874                         (intmax_t)size, r);
1875                 goto fail_2;
1876         }
1877
1878         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
1879                             size,
1880                             ubsec_dmamap_cb,
1881                             &dma->dma_paddr,
1882                             mapflags | BUS_DMA_NOWAIT);
1883         if (r != 0) {
1884                 device_printf(sc->sc_dev, "ubsec_dma_malloc: "
1885                         "bus_dmamap_load failed; error %u\n", r);
1886                 goto fail_3;
1887         }
1888
1889         dma->dma_size = size;
1890         return (0);
1891
1892 fail_3:
1893         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
1894 fail_2:
1895         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1896 fail_1:
1897         bus_dmamap_destroy(dma->dma_tag, dma->dma_map);
1898         bus_dma_tag_destroy(dma->dma_tag);
1899 fail_0:
1900         dma->dma_map = NULL;
1901         dma->dma_tag = NULL;
1902         return (r);
1903 }
1904
1905 static void
1906 ubsec_dma_free(struct ubsec_softc *sc, struct ubsec_dma_alloc *dma)
1907 {
1908         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
1909         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1910         bus_dmamap_destroy(dma->dma_tag, dma->dma_map);
1911         bus_dma_tag_destroy(dma->dma_tag);
1912 }
1913
1914 /*
1915  * Resets the board.  Values in the regesters are left as is
1916  * from the reset (i.e. initial values are assigned elsewhere).
1917  */
1918 static void
1919 ubsec_reset_board(struct ubsec_softc *sc)
1920 {
1921     volatile u_int32_t ctrl;
1922
1923     ctrl = READ_REG(sc, BS_CTRL);
1924     ctrl |= BS_CTRL_RESET;
1925     WRITE_REG(sc, BS_CTRL, ctrl);
1926
1927     /*
1928      * Wait aprox. 30 PCI clocks = 900 ns = 0.9 us
1929      */
1930     DELAY(10);
1931 }
1932
1933 /*
1934  * Init Broadcom registers
1935  */
1936 static void
1937 ubsec_init_board(struct ubsec_softc *sc)
1938 {
1939         u_int32_t ctrl;
1940
1941         ctrl = READ_REG(sc, BS_CTRL);
1942         ctrl &= ~(BS_CTRL_BE32 | BS_CTRL_BE64);
1943         ctrl |= BS_CTRL_LITTLE_ENDIAN | BS_CTRL_MCR1INT;
1944
1945         if (sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG))
1946                 ctrl |= BS_CTRL_MCR2INT;
1947         else
1948                 ctrl &= ~BS_CTRL_MCR2INT;
1949
1950         if (sc->sc_flags & UBS_FLAGS_HWNORM)
1951                 ctrl &= ~BS_CTRL_SWNORM;
1952
1953         WRITE_REG(sc, BS_CTRL, ctrl);
1954 }
1955
1956 /*
1957  * Init Broadcom PCI registers
1958  */
1959 static void
1960 ubsec_init_pciregs(device_t dev)
1961 {
1962 #if 0
1963         u_int32_t misc;
1964
1965         misc = pci_conf_read(pc, pa->pa_tag, BS_RTY_TOUT);
1966         misc = (misc & ~(UBS_PCI_RTY_MASK << UBS_PCI_RTY_SHIFT))
1967             | ((UBS_DEF_RTY & 0xff) << UBS_PCI_RTY_SHIFT);
1968         misc = (misc & ~(UBS_PCI_TOUT_MASK << UBS_PCI_TOUT_SHIFT))
1969             | ((UBS_DEF_TOUT & 0xff) << UBS_PCI_TOUT_SHIFT);
1970         pci_conf_write(pc, pa->pa_tag, BS_RTY_TOUT, misc);
1971 #endif
1972
1973         /*
1974          * This will set the cache line size to 1, this will
1975          * force the BCM58xx chip just to do burst read/writes.
1976          * Cache line read/writes are to slow
1977          */
1978         pci_write_config(dev, PCIR_CACHELNSZ, UBS_DEF_CACHELINE, 1);
1979 }
1980
1981 /*
1982  * Clean up after a chip crash.
1983  * It is assumed that the caller in splimp()
1984  */
1985 static void
1986 ubsec_cleanchip(struct ubsec_softc *sc)
1987 {
1988         struct ubsec_q *q;
1989
1990         while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
1991                 q = SIMPLEQ_FIRST(&sc->sc_qchip);
1992                 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q_next);
1993                 ubsec_free_q(sc, q);
1994         }
1995         sc->sc_nqchip = 0;
1996 }
1997
1998 /*
1999  * free a ubsec_q
2000  * It is assumed that the caller is within splimp().
2001  */
2002 static int
2003 ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q)
2004 {
2005         struct ubsec_q *q2;
2006         struct cryptop *crp;
2007         int npkts;
2008         int i;
2009
2010         npkts = q->q_nstacked_mcrs;
2011
2012         for (i = 0; i < npkts; i++) {
2013                 if(q->q_stacked_mcr[i]) {
2014                         q2 = q->q_stacked_mcr[i];
2015
2016                         if ((q2->q_dst_m != NULL) && (q2->q_src_m != q2->q_dst_m))
2017                                 m_freem(q2->q_dst_m);
2018
2019                         crp = (struct cryptop *)q2->q_crp;
2020
2021                         SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next);
2022
2023                         crp->crp_etype = EFAULT;
2024                         crypto_done(crp);
2025                 } else {
2026                         break;
2027                 }
2028         }
2029
2030         /*
2031          * Free header MCR
2032          */
2033         if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
2034                 m_freem(q->q_dst_m);
2035
2036         crp = (struct cryptop *)q->q_crp;
2037
2038         SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
2039
2040         crp->crp_etype = EFAULT;
2041         crypto_done(crp);
2042         return(0);
2043 }
2044
2045 /*
2046  * Routine to reset the chip and clean up.
2047  * It is assumed that the caller is in splimp()
2048  */
2049 static void
2050 ubsec_totalreset(struct ubsec_softc *sc)
2051 {
2052         ubsec_reset_board(sc);
2053         ubsec_init_board(sc);
2054         ubsec_cleanchip(sc);
2055 }
2056
2057 static int
2058 ubsec_dmamap_aligned(struct ubsec_operand *op)
2059 {
2060         int i;
2061
2062         for (i = 0; i < op->nsegs; i++) {
2063                 if (op->segs[i].ds_addr & 3)
2064                         return (0);
2065                 if ((i != (op->nsegs - 1)) &&
2066                     (op->segs[i].ds_len & 3))
2067                         return (0);
2068         }
2069         return (1);
2070 }
2071
2072 static void
2073 ubsec_kfree(struct ubsec_softc *sc, struct ubsec_q2 *q)
2074 {
2075         switch (q->q_type) {
2076         case UBS_CTXOP_MODEXP: {
2077                 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q;
2078
2079                 ubsec_dma_free(sc, &me->me_q.q_mcr);
2080                 ubsec_dma_free(sc, &me->me_q.q_ctx);
2081                 ubsec_dma_free(sc, &me->me_M);
2082                 ubsec_dma_free(sc, &me->me_E);
2083                 ubsec_dma_free(sc, &me->me_C);
2084                 ubsec_dma_free(sc, &me->me_epb);
2085                 free(me, M_DEVBUF);
2086                 break;
2087         }
2088         case UBS_CTXOP_RSAPRIV: {
2089                 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q;
2090
2091                 ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
2092                 ubsec_dma_free(sc, &rp->rpr_q.q_ctx);
2093                 ubsec_dma_free(sc, &rp->rpr_msgin);
2094                 ubsec_dma_free(sc, &rp->rpr_msgout);
2095                 free(rp, M_DEVBUF);
2096                 break;
2097         }
2098         default:
2099                 device_printf(sc->sc_dev, "invalid kfree 0x%x\n", q->q_type);
2100                 break;
2101         }
2102 }
2103
2104 static int
2105 ubsec_kprocess(device_t dev, struct cryptkop *krp, int hint)
2106 {
2107         struct ubsec_softc *sc = device_get_softc(dev);
2108         int r;
2109
2110         if (krp == NULL || krp->krp_callback == NULL)
2111                 return (EINVAL);
2112
2113         while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) {
2114                 struct ubsec_q2 *q;
2115
2116                 q = SIMPLEQ_FIRST(&sc->sc_q2free);
2117                 SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, q_next);
2118                 ubsec_kfree(sc, q);
2119         }
2120
2121         switch (krp->krp_op) {
2122         case CRK_MOD_EXP:
2123                 if (sc->sc_flags & UBS_FLAGS_HWNORM)
2124                         r = ubsec_kprocess_modexp_hw(sc, krp, hint);
2125                 else
2126                         r = ubsec_kprocess_modexp_sw(sc, krp, hint);
2127                 break;
2128         case CRK_MOD_EXP_CRT:
2129                 return (ubsec_kprocess_rsapriv(sc, krp, hint));
2130         default:
2131                 device_printf(sc->sc_dev, "kprocess: invalid op 0x%x\n",
2132                     krp->krp_op);
2133                 krp->krp_status = EOPNOTSUPP;
2134                 crypto_kdone(krp);
2135                 return (0);
2136         }
2137         return (0);                     /* silence compiler */
2138 }
2139
2140 /*
2141  * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization)
2142  */
2143 static int
2144 ubsec_kprocess_modexp_sw(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
2145 {
2146         struct ubsec_q2_modexp *me;
2147         struct ubsec_mcr *mcr;
2148         struct ubsec_ctx_modexp *ctx;
2149         struct ubsec_pktbuf *epb;
2150         int err = 0;
2151         u_int nbits, normbits, mbits, shiftbits, ebits;
2152
2153         me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT);
2154         if (me == NULL) {
2155                 err = ENOMEM;
2156                 goto errout;
2157         }
2158         bzero(me, sizeof *me);
2159         me->me_krp = krp;
2160         me->me_q.q_type = UBS_CTXOP_MODEXP;
2161
2162         nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
2163         if (nbits <= 512)
2164                 normbits = 512;
2165         else if (nbits <= 768)
2166                 normbits = 768;
2167         else if (nbits <= 1024)
2168                 normbits = 1024;
2169         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536)
2170                 normbits = 1536;
2171         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048)
2172                 normbits = 2048;
2173         else {
2174                 err = E2BIG;
2175                 goto errout;
2176         }
2177
2178         shiftbits = normbits - nbits;
2179
2180         me->me_modbits = nbits;
2181         me->me_shiftbits = shiftbits;
2182         me->me_normbits = normbits;
2183
2184         /* Sanity check: result bits must be >= true modulus bits. */
2185         if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
2186                 err = ERANGE;
2187                 goto errout;
2188         }
2189
2190         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2191             &me->me_q.q_mcr, 0)) {
2192                 err = ENOMEM;
2193                 goto errout;
2194         }
2195         mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
2196
2197         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
2198             &me->me_q.q_ctx, 0)) {
2199                 err = ENOMEM;
2200                 goto errout;
2201         }
2202
2203         mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
2204         if (mbits > nbits) {
2205                 err = E2BIG;
2206                 goto errout;
2207         }
2208         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) {
2209                 err = ENOMEM;
2210                 goto errout;
2211         }
2212         ubsec_kshift_r(shiftbits,
2213             krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits,
2214             me->me_M.dma_vaddr, normbits);
2215
2216         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) {
2217                 err = ENOMEM;
2218                 goto errout;
2219         }
2220         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
2221
2222         ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
2223         if (ebits > nbits) {
2224                 err = E2BIG;
2225                 goto errout;
2226         }
2227         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) {
2228                 err = ENOMEM;
2229                 goto errout;
2230         }
2231         ubsec_kshift_r(shiftbits,
2232             krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits,
2233             me->me_E.dma_vaddr, normbits);
2234
2235         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
2236             &me->me_epb, 0)) {
2237                 err = ENOMEM;
2238                 goto errout;
2239         }
2240         epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
2241         epb->pb_addr = htole32(me->me_E.dma_paddr);
2242         epb->pb_next = 0;
2243         epb->pb_len = htole32(normbits / 8);
2244
2245 #ifdef UBSEC_DEBUG
2246         if (ubsec_debug) {
2247                 printf("Epb ");
2248                 ubsec_dump_pb(epb);
2249         }
2250 #endif
2251
2252         mcr->mcr_pkts = htole16(1);
2253         mcr->mcr_flags = 0;
2254         mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
2255         mcr->mcr_reserved = 0;
2256         mcr->mcr_pktlen = 0;
2257
2258         mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
2259         mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8);
2260         mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
2261
2262         mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
2263         mcr->mcr_opktbuf.pb_next = 0;
2264         mcr->mcr_opktbuf.pb_len = htole32(normbits / 8);
2265
2266 #ifdef DIAGNOSTIC
2267         /* Misaligned output buffer will hang the chip. */
2268         if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0)
2269                 panic("%s: modexp invalid addr 0x%x\n",
2270                     device_get_nameunit(sc->sc_dev),
2271                     letoh32(mcr->mcr_opktbuf.pb_addr));
2272         if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0)
2273                 panic("%s: modexp invalid len 0x%x\n",
2274                     device_get_nameunit(sc->sc_dev),
2275                     letoh32(mcr->mcr_opktbuf.pb_len));
2276 #endif
2277
2278         ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
2279         bzero(ctx, sizeof(*ctx));
2280         ubsec_kshift_r(shiftbits,
2281             krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits,
2282             ctx->me_N, normbits);
2283         ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t)));
2284         ctx->me_op = htole16(UBS_CTXOP_MODEXP);
2285         ctx->me_E_len = htole16(nbits);
2286         ctx->me_N_len = htole16(nbits);
2287
2288 #ifdef UBSEC_DEBUG
2289         if (ubsec_debug) {
2290                 ubsec_dump_mcr(mcr);
2291                 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
2292         }
2293 #endif
2294
2295         /*
2296          * ubsec_feed2 will sync mcr and ctx, we just need to sync
2297          * everything else.
2298          */
2299         ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE);
2300         ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE);
2301         ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD);
2302         ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE);
2303
2304         /* Enqueue and we're done... */
2305         mtx_lock(&sc->sc_mcr2lock);
2306         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
2307         ubsec_feed2(sc);
2308         ubsecstats.hst_modexp++;
2309         mtx_unlock(&sc->sc_mcr2lock);
2310
2311         return (0);
2312
2313 errout:
2314         if (me != NULL) {
2315                 if (me->me_q.q_mcr.dma_map != NULL)
2316                         ubsec_dma_free(sc, &me->me_q.q_mcr);
2317                 if (me->me_q.q_ctx.dma_map != NULL) {
2318                         bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
2319                         ubsec_dma_free(sc, &me->me_q.q_ctx);
2320                 }
2321                 if (me->me_M.dma_map != NULL) {
2322                         bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
2323                         ubsec_dma_free(sc, &me->me_M);
2324                 }
2325                 if (me->me_E.dma_map != NULL) {
2326                         bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
2327                         ubsec_dma_free(sc, &me->me_E);
2328                 }
2329                 if (me->me_C.dma_map != NULL) {
2330                         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
2331                         ubsec_dma_free(sc, &me->me_C);
2332                 }
2333                 if (me->me_epb.dma_map != NULL)
2334                         ubsec_dma_free(sc, &me->me_epb);
2335                 free(me, M_DEVBUF);
2336         }
2337         krp->krp_status = err;
2338         crypto_kdone(krp);
2339         return (0);
2340 }
2341
2342 /*
2343  * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization)
2344  */
2345 static int
2346 ubsec_kprocess_modexp_hw(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
2347 {
2348         struct ubsec_q2_modexp *me;
2349         struct ubsec_mcr *mcr;
2350         struct ubsec_ctx_modexp *ctx;
2351         struct ubsec_pktbuf *epb;
2352         int err = 0;
2353         u_int nbits, normbits, mbits, shiftbits, ebits;
2354
2355         me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT);
2356         if (me == NULL) {
2357                 err = ENOMEM;
2358                 goto errout;
2359         }
2360         bzero(me, sizeof *me);
2361         me->me_krp = krp;
2362         me->me_q.q_type = UBS_CTXOP_MODEXP;
2363
2364         nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
2365         if (nbits <= 512)
2366                 normbits = 512;
2367         else if (nbits <= 768)
2368                 normbits = 768;
2369         else if (nbits <= 1024)
2370                 normbits = 1024;
2371         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536)
2372                 normbits = 1536;
2373         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048)
2374                 normbits = 2048;
2375         else {
2376                 err = E2BIG;
2377                 goto errout;
2378         }
2379
2380         shiftbits = normbits - nbits;
2381
2382         /* XXX ??? */
2383         me->me_modbits = nbits;
2384         me->me_shiftbits = shiftbits;
2385         me->me_normbits = normbits;
2386
2387         /* Sanity check: result bits must be >= true modulus bits. */
2388         if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
2389                 err = ERANGE;
2390                 goto errout;
2391         }
2392
2393         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2394             &me->me_q.q_mcr, 0)) {
2395                 err = ENOMEM;
2396                 goto errout;
2397         }
2398         mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
2399
2400         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
2401             &me->me_q.q_ctx, 0)) {
2402                 err = ENOMEM;
2403                 goto errout;
2404         }
2405
2406         mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
2407         if (mbits > nbits) {
2408                 err = E2BIG;
2409                 goto errout;
2410         }
2411         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) {
2412                 err = ENOMEM;
2413                 goto errout;
2414         }
2415         bzero(me->me_M.dma_vaddr, normbits / 8);
2416         bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p,
2417             me->me_M.dma_vaddr, (mbits + 7) / 8);
2418
2419         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) {
2420                 err = ENOMEM;
2421                 goto errout;
2422         }
2423         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
2424
2425         ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
2426         if (ebits > nbits) {
2427                 err = E2BIG;
2428                 goto errout;
2429         }
2430         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) {
2431                 err = ENOMEM;
2432                 goto errout;
2433         }
2434         bzero(me->me_E.dma_vaddr, normbits / 8);
2435         bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p,
2436             me->me_E.dma_vaddr, (ebits + 7) / 8);
2437
2438         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
2439             &me->me_epb, 0)) {
2440                 err = ENOMEM;
2441                 goto errout;
2442         }
2443         epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
2444         epb->pb_addr = htole32(me->me_E.dma_paddr);
2445         epb->pb_next = 0;
2446         epb->pb_len = htole32((ebits + 7) / 8);
2447
2448 #ifdef UBSEC_DEBUG
2449         if (ubsec_debug) {
2450                 printf("Epb ");
2451                 ubsec_dump_pb(epb);
2452         }
2453 #endif
2454
2455         mcr->mcr_pkts = htole16(1);
2456         mcr->mcr_flags = 0;
2457         mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
2458         mcr->mcr_reserved = 0;
2459         mcr->mcr_pktlen = 0;
2460
2461         mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
2462         mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8);
2463         mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
2464
2465         mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
2466         mcr->mcr_opktbuf.pb_next = 0;
2467         mcr->mcr_opktbuf.pb_len = htole32(normbits / 8);
2468
2469 #ifdef DIAGNOSTIC
2470         /* Misaligned output buffer will hang the chip. */
2471         if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0)
2472                 panic("%s: modexp invalid addr 0x%x\n",
2473                     device_get_nameunit(sc->sc_dev),
2474                     letoh32(mcr->mcr_opktbuf.pb_addr));
2475         if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0)
2476                 panic("%s: modexp invalid len 0x%x\n",
2477                     device_get_nameunit(sc->sc_dev),
2478                     letoh32(mcr->mcr_opktbuf.pb_len));
2479 #endif
2480
2481         ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
2482         bzero(ctx, sizeof(*ctx));
2483         bcopy(krp->krp_param[UBS_MODEXP_PAR_N].crp_p, ctx->me_N,
2484             (nbits + 7) / 8);
2485         ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t)));
2486         ctx->me_op = htole16(UBS_CTXOP_MODEXP);
2487         ctx->me_E_len = htole16(ebits);
2488         ctx->me_N_len = htole16(nbits);
2489
2490 #ifdef UBSEC_DEBUG
2491         if (ubsec_debug) {
2492                 ubsec_dump_mcr(mcr);
2493                 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
2494         }
2495 #endif
2496
2497         /*
2498          * ubsec_feed2 will sync mcr and ctx, we just need to sync
2499          * everything else.
2500          */
2501         ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE);
2502         ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE);
2503         ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD);
2504         ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE);
2505
2506         /* Enqueue and we're done... */
2507         mtx_lock(&sc->sc_mcr2lock);
2508         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
2509         ubsec_feed2(sc);
2510         mtx_unlock(&sc->sc_mcr2lock);
2511
2512         return (0);
2513
2514 errout:
2515         if (me != NULL) {
2516                 if (me->me_q.q_mcr.dma_map != NULL)
2517                         ubsec_dma_free(sc, &me->me_q.q_mcr);
2518                 if (me->me_q.q_ctx.dma_map != NULL) {
2519                         bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
2520                         ubsec_dma_free(sc, &me->me_q.q_ctx);
2521                 }
2522                 if (me->me_M.dma_map != NULL) {
2523                         bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
2524                         ubsec_dma_free(sc, &me->me_M);
2525                 }
2526                 if (me->me_E.dma_map != NULL) {
2527                         bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
2528                         ubsec_dma_free(sc, &me->me_E);
2529                 }
2530                 if (me->me_C.dma_map != NULL) {
2531                         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
2532                         ubsec_dma_free(sc, &me->me_C);
2533                 }
2534                 if (me->me_epb.dma_map != NULL)
2535                         ubsec_dma_free(sc, &me->me_epb);
2536                 free(me, M_DEVBUF);
2537         }
2538         krp->krp_status = err;
2539         crypto_kdone(krp);
2540         return (0);
2541 }
2542
2543 static int
2544 ubsec_kprocess_rsapriv(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
2545 {
2546         struct ubsec_q2_rsapriv *rp = NULL;
2547         struct ubsec_mcr *mcr;
2548         struct ubsec_ctx_rsapriv *ctx;
2549         int err = 0;
2550         u_int padlen, msglen;
2551
2552         msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]);
2553         padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]);
2554         if (msglen > padlen)
2555                 padlen = msglen;
2556
2557         if (padlen <= 256)
2558                 padlen = 256;
2559         else if (padlen <= 384)
2560                 padlen = 384;
2561         else if (padlen <= 512)
2562                 padlen = 512;
2563         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 768)
2564                 padlen = 768;
2565         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 1024)
2566                 padlen = 1024;
2567         else {
2568                 err = E2BIG;
2569                 goto errout;
2570         }
2571
2572         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) {
2573                 err = E2BIG;
2574                 goto errout;
2575         }
2576
2577         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) {
2578                 err = E2BIG;
2579                 goto errout;
2580         }
2581
2582         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) {
2583                 err = E2BIG;
2584                 goto errout;
2585         }
2586
2587         rp = (struct ubsec_q2_rsapriv *)malloc(sizeof *rp, M_DEVBUF, M_NOWAIT);
2588         if (rp == NULL)
2589                 return (ENOMEM);
2590         bzero(rp, sizeof *rp);
2591         rp->rpr_krp = krp;
2592         rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV;
2593
2594         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2595             &rp->rpr_q.q_mcr, 0)) {
2596                 err = ENOMEM;
2597                 goto errout;
2598         }
2599         mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr;
2600
2601         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv),
2602             &rp->rpr_q.q_ctx, 0)) {
2603                 err = ENOMEM;
2604                 goto errout;
2605         }
2606         ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr;
2607         bzero(ctx, sizeof *ctx);
2608
2609         /* Copy in p */
2610         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p,
2611             &ctx->rpr_buf[0 * (padlen / 8)],
2612             (krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + 7) / 8);
2613
2614         /* Copy in q */
2615         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p,
2616             &ctx->rpr_buf[1 * (padlen / 8)],
2617             (krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + 7) / 8);
2618
2619         /* Copy in dp */
2620         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p,
2621             &ctx->rpr_buf[2 * (padlen / 8)],
2622             (krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + 7) / 8);
2623
2624         /* Copy in dq */
2625         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p,
2626             &ctx->rpr_buf[3 * (padlen / 8)],
2627             (krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + 7) / 8);
2628
2629         /* Copy in pinv */
2630         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p,
2631             &ctx->rpr_buf[4 * (padlen / 8)],
2632             (krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + 7) / 8);
2633
2634         msglen = padlen * 2;
2635
2636         /* Copy in input message (aligned buffer/length). */
2637         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) {
2638                 /* Is this likely? */
2639                 err = E2BIG;
2640                 goto errout;
2641         }
2642         if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgin, 0)) {
2643                 err = ENOMEM;
2644                 goto errout;
2645         }
2646         bzero(rp->rpr_msgin.dma_vaddr, (msglen + 7) / 8);
2647         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p,
2648             rp->rpr_msgin.dma_vaddr,
2649             (krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + 7) / 8);
2650
2651         /* Prepare space for output message (aligned buffer/length). */
2652         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) {
2653                 /* Is this likely? */
2654                 err = E2BIG;
2655                 goto errout;
2656         }
2657         if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgout, 0)) {
2658                 err = ENOMEM;
2659                 goto errout;
2660         }
2661         bzero(rp->rpr_msgout.dma_vaddr, (msglen + 7) / 8);
2662
2663         mcr->mcr_pkts = htole16(1);
2664         mcr->mcr_flags = 0;
2665         mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr);
2666         mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr);
2667         mcr->mcr_ipktbuf.pb_next = 0;
2668         mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size);
2669         mcr->mcr_reserved = 0;
2670         mcr->mcr_pktlen = htole16(msglen);
2671         mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr);
2672         mcr->mcr_opktbuf.pb_next = 0;
2673         mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size);
2674
2675 #ifdef DIAGNOSTIC
2676         if (rp->rpr_msgin.dma_paddr & 3 || rp->rpr_msgin.dma_size & 3) {
2677                 panic("%s: rsapriv: invalid msgin %x(0x%jx)",
2678                     device_get_nameunit(sc->sc_dev),
2679                     rp->rpr_msgin.dma_paddr, (uintmax_t)rp->rpr_msgin.dma_size);
2680         }
2681         if (rp->rpr_msgout.dma_paddr & 3 || rp->rpr_msgout.dma_size & 3) {
2682                 panic("%s: rsapriv: invalid msgout %x(0x%jx)",
2683                     device_get_nameunit(sc->sc_dev),
2684                     rp->rpr_msgout.dma_paddr, (uintmax_t)rp->rpr_msgout.dma_size);
2685         }
2686 #endif
2687
2688         ctx->rpr_len = (sizeof(u_int16_t) * 4) + (5 * (padlen / 8));
2689         ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV);
2690         ctx->rpr_q_len = htole16(padlen);
2691         ctx->rpr_p_len = htole16(padlen);
2692
2693         /*
2694          * ubsec_feed2 will sync mcr and ctx, we just need to sync
2695          * everything else.
2696          */
2697         ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_PREWRITE);
2698         ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_PREREAD);
2699
2700         /* Enqueue and we're done... */
2701         mtx_lock(&sc->sc_mcr2lock);
2702         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next);
2703         ubsec_feed2(sc);
2704         ubsecstats.hst_modexpcrt++;
2705         mtx_unlock(&sc->sc_mcr2lock);
2706         return (0);
2707
2708 errout:
2709         if (rp != NULL) {
2710                 if (rp->rpr_q.q_mcr.dma_map != NULL)
2711                         ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
2712                 if (rp->rpr_msgin.dma_map != NULL) {
2713                         bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size);
2714                         ubsec_dma_free(sc, &rp->rpr_msgin);
2715                 }
2716                 if (rp->rpr_msgout.dma_map != NULL) {
2717                         bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size);
2718                         ubsec_dma_free(sc, &rp->rpr_msgout);
2719                 }
2720                 free(rp, M_DEVBUF);
2721         }
2722         krp->krp_status = err;
2723         crypto_kdone(krp);
2724         return (0);
2725 }
2726
2727 #ifdef UBSEC_DEBUG
2728 static void
2729 ubsec_dump_pb(volatile struct ubsec_pktbuf *pb)
2730 {
2731         printf("addr 0x%x (0x%x) next 0x%x\n",
2732             pb->pb_addr, pb->pb_len, pb->pb_next);
2733 }
2734
2735 static void
2736 ubsec_dump_ctx2(struct ubsec_ctx_keyop *c)
2737 {
2738         printf("CTX (0x%x):\n", c->ctx_len);
2739         switch (letoh16(c->ctx_op)) {
2740         case UBS_CTXOP_RNGBYPASS:
2741         case UBS_CTXOP_RNGSHA1:
2742                 break;
2743         case UBS_CTXOP_MODEXP:
2744         {
2745                 struct ubsec_ctx_modexp *cx = (void *)c;
2746                 int i, len;
2747
2748                 printf(" Elen %u, Nlen %u\n",
2749                     letoh16(cx->me_E_len), letoh16(cx->me_N_len));
2750                 len = (cx->me_N_len + 7)/8;
2751                 for (i = 0; i < len; i++)
2752                         printf("%s%02x", (i == 0) ? " N: " : ":", cx->me_N[i]);
2753                 printf("\n");
2754                 break;
2755         }
2756         default:
2757                 printf("unknown context: %x\n", c->ctx_op);
2758         }
2759         printf("END CTX\n");
2760 }
2761
2762 static void
2763 ubsec_dump_mcr(struct ubsec_mcr *mcr)
2764 {
2765         volatile struct ubsec_mcr_add *ma;
2766         int i;
2767
2768         printf("MCR:\n");
2769         printf(" pkts: %u, flags 0x%x\n",
2770             letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags));
2771         ma = (volatile struct ubsec_mcr_add *)&mcr->mcr_cmdctxp;
2772         for (i = 0; i < letoh16(mcr->mcr_pkts); i++) {
2773                 printf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i,
2774                     letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen),
2775                     letoh16(ma->mcr_reserved));
2776                 printf(" %d: ipkt ", i);
2777                 ubsec_dump_pb(&ma->mcr_ipktbuf);
2778                 printf(" %d: opkt ", i);
2779                 ubsec_dump_pb(&ma->mcr_opktbuf);
2780                 ma++;
2781         }
2782         printf("END MCR\n");
2783 }
2784 #endif /* UBSEC_DEBUG */
2785
2786 /*
2787  * Return the number of significant bits of a big number.
2788  */
2789 static int
2790 ubsec_ksigbits(struct crparam *cr)
2791 {
2792         u_int plen = (cr->crp_nbits + 7) / 8;
2793         int i, sig = plen * 8;
2794         u_int8_t c, *p = cr->crp_p;
2795
2796         for (i = plen - 1; i >= 0; i--) {
2797                 c = p[i];
2798                 if (c != 0) {
2799                         while ((c & 0x80) == 0) {
2800                                 sig--;
2801                                 c <<= 1;
2802                         }
2803                         break;
2804                 }
2805                 sig -= 8;
2806         }
2807         return (sig);
2808 }
2809
2810 static void
2811 ubsec_kshift_r(
2812         u_int shiftbits,
2813         u_int8_t *src, u_int srcbits,
2814         u_int8_t *dst, u_int dstbits)
2815 {
2816         u_int slen, dlen;
2817         int i, si, di, n;
2818
2819         slen = (srcbits + 7) / 8;
2820         dlen = (dstbits + 7) / 8;
2821
2822         for (i = 0; i < slen; i++)
2823                 dst[i] = src[i];
2824         for (i = 0; i < dlen - slen; i++)
2825                 dst[slen + i] = 0;
2826
2827         n = shiftbits / 8;
2828         if (n != 0) {
2829                 si = dlen - n - 1;
2830                 di = dlen - 1;
2831                 while (si >= 0)
2832                         dst[di--] = dst[si--];
2833                 while (di >= 0)
2834                         dst[di--] = 0;
2835         }
2836
2837         n = shiftbits % 8;
2838         if (n != 0) {
2839                 for (i = dlen - 1; i > 0; i--)
2840                         dst[i] = (dst[i] << n) |
2841                             (dst[i - 1] >> (8 - n));
2842                 dst[0] = dst[0] << n;
2843         }
2844 }
2845
2846 static void
2847 ubsec_kshift_l(
2848         u_int shiftbits,
2849         u_int8_t *src, u_int srcbits,
2850         u_int8_t *dst, u_int dstbits)
2851 {
2852         int slen, dlen, i, n;
2853
2854         slen = (srcbits + 7) / 8;
2855         dlen = (dstbits + 7) / 8;
2856
2857         n = shiftbits / 8;
2858         for (i = 0; i < slen; i++)
2859                 dst[i] = src[i + n];
2860         for (i = 0; i < dlen - slen; i++)
2861                 dst[slen + i] = 0;
2862
2863         n = shiftbits % 8;
2864         if (n != 0) {
2865                 for (i = 0; i < (dlen - 1); i++)
2866                         dst[i] = (dst[i] >> n) | (dst[i + 1] << (8 - n));
2867                 dst[dlen - 1] = dst[dlen - 1] >> n;
2868         }
2869 }