sys/dev/ubsec/ubsec.c

   1 /*      $OpenBSD: ubsec.c,v 1.115 2002/09/24 18:33:26 jason Exp $       */
   2
   3 /*-
   4  * Copyright (c) 2000 Jason L. Wright (jason@thought.net)
   5  * Copyright (c) 2000 Theo de Raadt (deraadt@openbsd.org)
   6  * Copyright (c) 2001 Patrik Lindergren (patrik@ipunplugged.com)
   7  *
   8  * All rights reserved.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  * 3. All advertising materials mentioning features or use of this software
  19  *    must display the following acknowledgement:
  20  *      This product includes software developed by Jason L. Wright
  21  * 4. The name of the author may not be used to endorse or promote products
  22  *    derived from this software without specific prior written permission.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  25  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  26  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  27  * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
  28  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  29  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  30  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  31  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  32  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  33  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  34  * POSSIBILITY OF SUCH DAMAGE.
  35  *
  36  * Effort sponsored in part by the Defense Advanced Research Projects
  37  * Agency (DARPA) and Air Force Research Laboratory, Air Force
  38  * Materiel Command, USAF, under agreement number F30602-01-2-0537.
  39  */
  40
  41 #include <sys/cdefs.h>
  42 __FBSDID("$FreeBSD$");
  43
  44 /*
  45  * uBsec 5[56]01, 58xx hardware crypto accelerator
  46  */
  47
  48 #include "opt_ubsec.h"
  49
  50 #include <sys/param.h>
  51 #include <sys/systm.h>
  52 #include <sys/proc.h>
  53 #include <sys/errno.h>
  54 #include <sys/malloc.h>
  55 #include <sys/kernel.h>
  56 #include <sys/module.h>
  57 #include <sys/mbuf.h>
  58 #include <sys/lock.h>
  59 #include <sys/mutex.h>
  60 #include <sys/sysctl.h>
  61 #include <sys/endian.h>
  62
  63 #include <vm/vm.h>
  64 #include <vm/pmap.h>
  65
  66 #include <machine/bus.h>
  67 #include <machine/resource.h>
  68 #include <sys/bus.h>
  69 #include <sys/rman.h>
  70
  71 #include <crypto/sha1.h>
  72 #include <opencrypto/cryptodev.h>
  73 #include <opencrypto/cryptosoft.h>
  74 #include <sys/md5.h>
  75 #include <sys/random.h>
  76 #include <sys/kobj.h>
  77
  78 #include "cryptodev_if.h"
  79
  80 #include <dev/pci/pcivar.h>
  81 #include <dev/pci/pcireg.h>
  82
  83 /* grr, #defines for gratuitous incompatibility in queue.h */
  84 #define SIMPLEQ_HEAD            STAILQ_HEAD
  85 #define SIMPLEQ_ENTRY           STAILQ_ENTRY
  86 #define SIMPLEQ_INIT            STAILQ_INIT
  87 #define SIMPLEQ_INSERT_TAIL     STAILQ_INSERT_TAIL
  88 #define SIMPLEQ_EMPTY           STAILQ_EMPTY
  89 #define SIMPLEQ_FIRST           STAILQ_FIRST
  90 #define SIMPLEQ_REMOVE_HEAD     STAILQ_REMOVE_HEAD
  91 #define SIMPLEQ_FOREACH         STAILQ_FOREACH
  92 /* ditto for endian.h */
  93 #define letoh16(x)              le16toh(x)
  94 #define letoh32(x)              le32toh(x)
  95
  96 #ifdef UBSEC_RNDTEST
  97 #include <dev/rndtest/rndtest.h>
  98 #endif
  99 #include <dev/ubsec/ubsecreg.h>
 100 #include <dev/ubsec/ubsecvar.h>
 101
 102 /*
 103  * Prototypes and count for the pci_device structure
 104  */
 105 static  int ubsec_probe(device_t);
 106 static  int ubsec_attach(device_t);
 107 static  int ubsec_detach(device_t);
 108 static  int ubsec_suspend(device_t);
 109 static  int ubsec_resume(device_t);
 110 static  int ubsec_shutdown(device_t);
 111
 112 static  int ubsec_newsession(device_t, u_int32_t *, struct cryptoini *);
 113 static  int ubsec_freesession(device_t, u_int64_t);
 114 static  int ubsec_process(device_t, struct cryptop *, int);
 115 static  int ubsec_kprocess(device_t, struct cryptkop *, int);
 116
 117 static device_method_t ubsec_methods[] = {
 118         /* Device interface */
 119         DEVMETHOD(device_probe,         ubsec_probe),
 120         DEVMETHOD(device_attach,        ubsec_attach),
 121         DEVMETHOD(device_detach,        ubsec_detach),
 122         DEVMETHOD(device_suspend,       ubsec_suspend),
 123         DEVMETHOD(device_resume,        ubsec_resume),
 124         DEVMETHOD(device_shutdown,      ubsec_shutdown),
 125
 126         /* crypto device methods */
 127         DEVMETHOD(cryptodev_newsession, ubsec_newsession),
 128         DEVMETHOD(cryptodev_freesession,ubsec_freesession),
 129         DEVMETHOD(cryptodev_process,    ubsec_process),
 130         DEVMETHOD(cryptodev_kprocess,   ubsec_kprocess),
 131
 132         DEVMETHOD_END
 133 };
 134 static driver_t ubsec_driver = {
 135         "ubsec",
 136         ubsec_methods,
 137         sizeof (struct ubsec_softc)
 138 };
 139 static devclass_t ubsec_devclass;
 140
 141 DRIVER_MODULE(ubsec, pci, ubsec_driver, ubsec_devclass, 0, 0);
 142 MODULE_DEPEND(ubsec, crypto, 1, 1, 1);
 143 #ifdef UBSEC_RNDTEST
 144 MODULE_DEPEND(ubsec, rndtest, 1, 1, 1);
 145 #endif
 146
 147 static  void ubsec_intr(void *);
 148 static  void ubsec_callback(struct ubsec_softc *, struct ubsec_q *);
 149 static  void ubsec_feed(struct ubsec_softc *);
 150 static  void ubsec_mcopy(struct mbuf *, struct mbuf *, int, int);
 151 static  void ubsec_callback2(struct ubsec_softc *, struct ubsec_q2 *);
 152 static  int ubsec_feed2(struct ubsec_softc *);
 153 static  void ubsec_rng(void *);
 154 static  int ubsec_dma_malloc(struct ubsec_softc *, bus_size_t,
 155                              struct ubsec_dma_alloc *, int);
 156 #define ubsec_dma_sync(_dma, _flags) \
 157         bus_dmamap_sync((_dma)->dma_tag, (_dma)->dma_map, (_flags))
 158 static  void ubsec_dma_free(struct ubsec_softc *, struct ubsec_dma_alloc *);
 159 static  int ubsec_dmamap_aligned(struct ubsec_operand *op);
 160
 161 static  void ubsec_reset_board(struct ubsec_softc *sc);
 162 static  void ubsec_init_board(struct ubsec_softc *sc);
 163 static  void ubsec_init_pciregs(device_t dev);
 164 static  void ubsec_totalreset(struct ubsec_softc *sc);
 165
 166 static  int ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q);
 167
 168 static  int ubsec_kprocess_modexp_hw(struct ubsec_softc *, struct cryptkop *, int);
 169 static  int ubsec_kprocess_modexp_sw(struct ubsec_softc *, struct cryptkop *, int);
 170 static  int ubsec_kprocess_rsapriv(struct ubsec_softc *, struct cryptkop *, int);
 171 static  void ubsec_kfree(struct ubsec_softc *, struct ubsec_q2 *);
 172 static  int ubsec_ksigbits(struct crparam *);
 173 static  void ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
 174 static  void ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
 175
 176 static SYSCTL_NODE(_hw, OID_AUTO, ubsec, CTLFLAG_RD, 0,
 177     "Broadcom driver parameters");
 178
 179 #ifdef UBSEC_DEBUG
 180 static  void ubsec_dump_pb(volatile struct ubsec_pktbuf *);
 181 static  void ubsec_dump_mcr(struct ubsec_mcr *);
 182 static  void ubsec_dump_ctx2(struct ubsec_ctx_keyop *);
 183
 184 static  int ubsec_debug = 0;
 185 SYSCTL_INT(_hw_ubsec, OID_AUTO, debug, CTLFLAG_RW, &ubsec_debug,
 186             0, "control debugging msgs");
 187 #endif
 188
 189 #define READ_REG(sc,r) \
 190         bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r))
 191
 192 #define WRITE_REG(sc,reg,val) \
 193         bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val)
 194
 195 #define SWAP32(x) (x) = htole32(ntohl((x)))
 196 #define HTOLE32(x) (x) = htole32(x)
 197
 198 struct ubsec_stats ubsecstats;
 199 SYSCTL_STRUCT(_hw_ubsec, OID_AUTO, stats, CTLFLAG_RD, &ubsecstats,
 200             ubsec_stats, "driver statistics");
 201
 202 static int
 203 ubsec_probe(device_t dev)
 204 {
 205         if (pci_get_vendor(dev) == PCI_VENDOR_SUN &&
 206             (pci_get_device(dev) == PCI_PRODUCT_SUN_5821 ||
 207              pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K))
 208                 return (BUS_PROBE_DEFAULT);
 209         if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL &&
 210             (pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5501 ||
 211              pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601))
 212                 return (BUS_PROBE_DEFAULT);
 213         if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 214             (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5801 ||
 215              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 ||
 216              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805 ||
 217              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820 ||
 218              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 ||
 219              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 ||
 220              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823 ||
 221              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5825
 222              ))
 223                 return (BUS_PROBE_DEFAULT);
 224         return (ENXIO);
 225 }
 226
 227 static const char*
 228 ubsec_partname(struct ubsec_softc *sc)
 229 {
 230         /* XXX sprintf numbers when not decoded */
 231         switch (pci_get_vendor(sc->sc_dev)) {
 232         case PCI_VENDOR_BROADCOM:
 233                 switch (pci_get_device(sc->sc_dev)) {
 234                 case PCI_PRODUCT_BROADCOM_5801: return "Broadcom 5801";
 235                 case PCI_PRODUCT_BROADCOM_5802: return "Broadcom 5802";
 236                 case PCI_PRODUCT_BROADCOM_5805: return "Broadcom 5805";
 237                 case PCI_PRODUCT_BROADCOM_5820: return "Broadcom 5820";
 238                 case PCI_PRODUCT_BROADCOM_5821: return "Broadcom 5821";
 239                 case PCI_PRODUCT_BROADCOM_5822: return "Broadcom 5822";
 240                 case PCI_PRODUCT_BROADCOM_5823: return "Broadcom 5823";
 241                 case PCI_PRODUCT_BROADCOM_5825: return "Broadcom 5825";
 242                 }
 243                 return "Broadcom unknown-part";
 244         case PCI_VENDOR_BLUESTEEL:
 245                 switch (pci_get_device(sc->sc_dev)) {
 246                 case PCI_PRODUCT_BLUESTEEL_5601: return "Bluesteel 5601";
 247                 }
 248                 return "Bluesteel unknown-part";
 249         case PCI_VENDOR_SUN:
 250                 switch (pci_get_device(sc->sc_dev)) {
 251                 case PCI_PRODUCT_SUN_5821: return "Sun Crypto 5821";
 252                 case PCI_PRODUCT_SUN_SCA1K: return "Sun Crypto 1K";
 253                 }
 254                 return "Sun unknown-part";
 255         }
 256         return "Unknown-vendor unknown-part";
 257 }
 258
 259 static void
 260 default_harvest(struct rndtest_state *rsp, void *buf, u_int count)
 261 {
 262         random_harvest(buf, count, count*NBBY/2, RANDOM_PURE_UBSEC);
 263 }
 264
 265 static int
 266 ubsec_attach(device_t dev)
 267 {
 268         struct ubsec_softc *sc = device_get_softc(dev);
 269         struct ubsec_dma *dmap;
 270         u_int32_t i;
 271         int rid;
 272
 273         bzero(sc, sizeof (*sc));
 274         sc->sc_dev = dev;
 275
 276         SIMPLEQ_INIT(&sc->sc_queue);
 277         SIMPLEQ_INIT(&sc->sc_qchip);
 278         SIMPLEQ_INIT(&sc->sc_queue2);
 279         SIMPLEQ_INIT(&sc->sc_qchip2);
 280         SIMPLEQ_INIT(&sc->sc_q2free);
 281
 282         /* XXX handle power management */
 283
 284         sc->sc_statmask = BS_STAT_MCR1_DONE | BS_STAT_DMAERR;
 285
 286         if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL &&
 287             pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601)
 288                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG;
 289
 290         if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 291             (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 ||
 292              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805))
 293                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG;
 294
 295         if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 296             pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820)
 297                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG |
 298                     UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY;
 299
 300         if ((pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 301              (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 ||
 302               pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 ||
 303               pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823 ||
 304               pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5825)) ||
 305             (pci_get_vendor(dev) == PCI_VENDOR_SUN &&
 306              (pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K ||
 307               pci_get_device(dev) == PCI_PRODUCT_SUN_5821))) {
 308                 /* NB: the 5821/5822 defines some additional status bits */
 309                 sc->sc_statmask |= BS_STAT_MCR1_ALLEMPTY |
 310                     BS_STAT_MCR2_ALLEMPTY;
 311                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG |
 312                     UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY;
 313         }
 314
 315         pci_enable_busmaster(dev);
 316
 317         /*
 318          * Setup memory-mapping of PCI registers.
 319          */
 320         rid = BS_BAR;
 321         sc->sc_sr = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 322                                            RF_ACTIVE);
 323         if (sc->sc_sr == NULL) {
 324                 device_printf(dev, "cannot map register space\n");
 325                 goto bad;
 326         }
 327         sc->sc_st = rman_get_bustag(sc->sc_sr);
 328         sc->sc_sh = rman_get_bushandle(sc->sc_sr);
 329
 330         /*
 331          * Arrange interrupt line.
 332          */
 333         rid = 0;
 334         sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 335                                             RF_SHAREABLE|RF_ACTIVE);
 336         if (sc->sc_irq == NULL) {
 337                 device_printf(dev, "could not map interrupt\n");
 338                 goto bad1;
 339         }
 340         /*
 341          * NB: Network code assumes we are blocked with splimp()
 342          *     so make sure the IRQ is mapped appropriately.
 343          */
 344         if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE,
 345                            NULL, ubsec_intr, sc, &sc->sc_ih)) {
 346                 device_printf(dev, "could not establish interrupt\n");
 347                 goto bad2;
 348         }
 349
 350         sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE);
 351         if (sc->sc_cid < 0) {
 352                 device_printf(dev, "could not get crypto driver id\n");
 353                 goto bad3;
 354         }
 355
 356         /*
 357          * Setup DMA descriptor area.
 358          */
 359         if (bus_dma_tag_create(bus_get_dma_tag(dev),    /* parent */
 360                                1, 0,                    /* alignment, bounds */
 361                                BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
 362                                BUS_SPACE_MAXADDR,       /* highaddr */
 363                                NULL, NULL,              /* filter, filterarg */
 364                                0x3ffff,                 /* maxsize */
 365                                UBS_MAX_SCATTER,         /* nsegments */
 366                                0xffff,                  /* maxsegsize */
 367                                BUS_DMA_ALLOCNOW,        /* flags */
 368                                NULL, NULL,              /* lockfunc, lockarg */
 369                                &sc->sc_dmat)) {
 370                 device_printf(dev, "cannot allocate DMA tag\n");
 371                 goto bad4;
 372         }
 373         SIMPLEQ_INIT(&sc->sc_freequeue);
 374         dmap = sc->sc_dmaa;
 375         for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) {
 376                 struct ubsec_q *q;
 377
 378                 q = (struct ubsec_q *)malloc(sizeof(struct ubsec_q),
 379                     M_DEVBUF, M_NOWAIT);
 380                 if (q == NULL) {
 381                         device_printf(dev, "cannot allocate queue buffers\n");
 382                         break;
 383                 }
 384
 385                 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk),
 386                     &dmap->d_alloc, 0)) {
 387                         device_printf(dev, "cannot allocate dma buffers\n");
 388                         free(q, M_DEVBUF);
 389                         break;
 390                 }
 391                 dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr;
 392
 393                 q->q_dma = dmap;
 394                 sc->sc_queuea[i] = q;
 395
 396                 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
 397         }
 398         mtx_init(&sc->sc_mcr1lock, device_get_nameunit(dev),
 399                 "mcr1 operations", MTX_DEF);
 400         mtx_init(&sc->sc_freeqlock, device_get_nameunit(dev),
 401                 "mcr1 free q", MTX_DEF);
 402
 403         device_printf(sc->sc_dev, "%s\n", ubsec_partname(sc));
 404
 405         crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
 406         crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
 407         crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
 408         crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
 409
 410         /*
 411          * Reset Broadcom chip
 412          */
 413         ubsec_reset_board(sc);
 414
 415         /*
 416          * Init Broadcom specific PCI settings
 417          */
 418         ubsec_init_pciregs(dev);
 419
 420         /*
 421          * Init Broadcom chip
 422          */
 423         ubsec_init_board(sc);
 424
 425 #ifndef UBSEC_NO_RNG
 426         if (sc->sc_flags & UBS_FLAGS_RNG) {
 427                 sc->sc_statmask |= BS_STAT_MCR2_DONE;
 428 #ifdef UBSEC_RNDTEST
 429                 sc->sc_rndtest = rndtest_attach(dev);
 430                 if (sc->sc_rndtest)
 431                         sc->sc_harvest = rndtest_harvest;
 432                 else
 433                         sc->sc_harvest = default_harvest;
 434 #else
 435                 sc->sc_harvest = default_harvest;
 436 #endif
 437
 438                 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
 439                     &sc->sc_rng.rng_q.q_mcr, 0))
 440                         goto skip_rng;
 441
 442                 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass),
 443                     &sc->sc_rng.rng_q.q_ctx, 0)) {
 444                         ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
 445                         goto skip_rng;
 446                 }
 447
 448                 if (ubsec_dma_malloc(sc, sizeof(u_int32_t) *
 449                     UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, 0)) {
 450                         ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx);
 451                         ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
 452                         goto skip_rng;
 453                 }
 454
 455                 if (hz >= 100)
 456                         sc->sc_rnghz = hz / 100;
 457                 else
 458                         sc->sc_rnghz = 1;
 459                 callout_init(&sc->sc_rngto, CALLOUT_MPSAFE);
 460                 callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
 461 skip_rng:
 462         ;
 463         }
 464 #endif /* UBSEC_NO_RNG */
 465         mtx_init(&sc->sc_mcr2lock, device_get_nameunit(dev),
 466                 "mcr2 operations", MTX_DEF);
 467
 468         if (sc->sc_flags & UBS_FLAGS_KEY) {
 469                 sc->sc_statmask |= BS_STAT_MCR2_DONE;
 470
 471                 crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0);
 472 #if 0
 473                 crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0);
 474 #endif
 475         }
 476         return (0);
 477 bad4:
 478         crypto_unregister_all(sc->sc_cid);
 479 bad3:
 480         bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
 481 bad2:
 482         bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
 483 bad1:
 484         bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
 485 bad:
 486         return (ENXIO);
 487 }
 488
 489 /*
 490  * Detach a device that successfully probed.
 491  */
 492 static int
 493 ubsec_detach(device_t dev)
 494 {
 495         struct ubsec_softc *sc = device_get_softc(dev);
 496
 497         /* XXX wait/abort active ops */
 498
 499         /* disable interrupts */
 500         WRITE_REG(sc, BS_CTRL, READ_REG(sc, BS_CTRL) &~
 501                 (BS_CTRL_MCR2INT | BS_CTRL_MCR1INT | BS_CTRL_DMAERR));
 502
 503         callout_stop(&sc->sc_rngto);
 504
 505         crypto_unregister_all(sc->sc_cid);
 506
 507 #ifdef UBSEC_RNDTEST
 508         if (sc->sc_rndtest)
 509                 rndtest_detach(sc->sc_rndtest);
 510 #endif
 511
 512         while (!SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
 513                 struct ubsec_q *q;
 514
 515                 q = SIMPLEQ_FIRST(&sc->sc_freequeue);
 516                 SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q_next);
 517                 ubsec_dma_free(sc, &q->q_dma->d_alloc);
 518                 free(q, M_DEVBUF);
 519         }
 520         mtx_destroy(&sc->sc_mcr1lock);
 521         mtx_destroy(&sc->sc_freeqlock);
 522 #ifndef UBSEC_NO_RNG
 523         if (sc->sc_flags & UBS_FLAGS_RNG) {
 524                 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
 525                 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx);
 526                 ubsec_dma_free(sc, &sc->sc_rng.rng_buf);
 527         }
 528 #endif /* UBSEC_NO_RNG */
 529         mtx_destroy(&sc->sc_mcr2lock);
 530
 531         bus_generic_detach(dev);
 532         bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
 533         bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
 534
 535         bus_dma_tag_destroy(sc->sc_dmat);
 536         bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
 537
 538         return (0);
 539 }
 540
 541 /*
 542  * Stop all chip i/o so that the kernel's probe routines don't
 543  * get confused by errant DMAs when rebooting.
 544  */
 545 static int
 546 ubsec_shutdown(device_t dev)
 547 {
 548 #ifdef notyet
 549         ubsec_stop(device_get_softc(dev));
 550 #endif
 551         return (0);
 552 }
 553
 554 /*
 555  * Device suspend routine.
 556  */
 557 static int
 558 ubsec_suspend(device_t dev)
 559 {
 560         struct ubsec_softc *sc = device_get_softc(dev);
 561
 562 #ifdef notyet
 563         /* XXX stop the device and save PCI settings */
 564 #endif
 565         sc->sc_suspended = 1;
 566
 567         return (0);
 568 }
 569
 570 static int
 571 ubsec_resume(device_t dev)
 572 {
 573         struct ubsec_softc *sc = device_get_softc(dev);
 574
 575 #ifdef notyet
 576         /* XXX retore PCI settings and start the device */
 577 #endif
 578         sc->sc_suspended = 0;
 579         return (0);
 580 }
 581
 582 /*
 583  * UBSEC Interrupt routine
 584  */
 585 static void
 586 ubsec_intr(void *arg)
 587 {
 588         struct ubsec_softc *sc = arg;
 589         volatile u_int32_t stat;
 590         struct ubsec_q *q;
 591         struct ubsec_dma *dmap;
 592         int npkts = 0, i;
 593
 594         stat = READ_REG(sc, BS_STAT);
 595         stat &= sc->sc_statmask;
 596         if (stat == 0)
 597                 return;
 598
 599         WRITE_REG(sc, BS_STAT, stat);           /* IACK */
 600
 601         /*
 602          * Check to see if we have any packets waiting for us
 603          */
 604         if ((stat & BS_STAT_MCR1_DONE)) {
 605                 mtx_lock(&sc->sc_mcr1lock);
 606                 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
 607                         q = SIMPLEQ_FIRST(&sc->sc_qchip);
 608                         dmap = q->q_dma;
 609
 610                         if ((dmap->d_dma->d_mcr.mcr_flags & htole16(UBS_MCR_DONE)) == 0)
 611                                 break;
 612
 613                         SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q_next);
 614
 615                         npkts = q->q_nstacked_mcrs;
 616                         sc->sc_nqchip -= 1+npkts;
 617                         /*
 618                          * search for further sc_qchip ubsec_q's that share
 619                          * the same MCR, and complete them too, they must be
 620                          * at the top.
 621                          */
 622                         for (i = 0; i < npkts; i++) {
 623                                 if(q->q_stacked_mcr[i]) {
 624                                         ubsec_callback(sc, q->q_stacked_mcr[i]);
 625                                 } else {
 626                                         break;
 627                                 }
 628                         }
 629                         ubsec_callback(sc, q);
 630                 }
 631                 /*
 632                  * Don't send any more packet to chip if there has been
 633                  * a DMAERR.
 634                  */
 635                 if (!(stat & BS_STAT_DMAERR))
 636                         ubsec_feed(sc);
 637                 mtx_unlock(&sc->sc_mcr1lock);
 638         }
 639
 640         /*
 641          * Check to see if we have any key setups/rng's waiting for us
 642          */
 643         if ((sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) &&
 644             (stat & BS_STAT_MCR2_DONE)) {
 645                 struct ubsec_q2 *q2;
 646                 struct ubsec_mcr *mcr;
 647
 648                 mtx_lock(&sc->sc_mcr2lock);
 649                 while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) {
 650                         q2 = SIMPLEQ_FIRST(&sc->sc_qchip2);
 651
 652                         ubsec_dma_sync(&q2->q_mcr,
 653                             BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
 654
 655                         mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr;
 656                         if ((mcr->mcr_flags & htole16(UBS_MCR_DONE)) == 0) {
 657                                 ubsec_dma_sync(&q2->q_mcr,
 658                                     BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
 659                                 break;
 660                         }
 661                         SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, q_next);
 662                         ubsec_callback2(sc, q2);
 663                         /*
 664                          * Don't send any more packet to chip if there has been
 665                          * a DMAERR.
 666                          */
 667                         if (!(stat & BS_STAT_DMAERR))
 668                                 ubsec_feed2(sc);
 669                 }
 670                 mtx_unlock(&sc->sc_mcr2lock);
 671         }
 672
 673         /*
 674          * Check to see if we got any DMA Error
 675          */
 676         if (stat & BS_STAT_DMAERR) {
 677 #ifdef UBSEC_DEBUG
 678                 if (ubsec_debug) {
 679                         volatile u_int32_t a = READ_REG(sc, BS_ERR);
 680
 681                         printf("dmaerr %s@%08x\n",
 682                             (a & BS_ERR_READ) ? "read" : "write",
 683                             a & BS_ERR_ADDR);
 684                 }
 685 #endif /* UBSEC_DEBUG */
 686                 ubsecstats.hst_dmaerr++;
 687                 mtx_lock(&sc->sc_mcr1lock);
 688                 ubsec_totalreset(sc);
 689                 ubsec_feed(sc);
 690                 mtx_unlock(&sc->sc_mcr1lock);
 691         }
 692
 693         if (sc->sc_needwakeup) {                /* XXX check high watermark */
 694                 int wakeup;
 695
 696                 mtx_lock(&sc->sc_freeqlock);
 697                 wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ);
 698 #ifdef UBSEC_DEBUG
 699                 if (ubsec_debug)
 700                         device_printf(sc->sc_dev, "wakeup crypto (%x)\n",
 701                                 sc->sc_needwakeup);
 702 #endif /* UBSEC_DEBUG */
 703                 sc->sc_needwakeup &= ~wakeup;
 704                 mtx_unlock(&sc->sc_freeqlock);
 705                 crypto_unblock(sc->sc_cid, wakeup);
 706         }
 707 }
 708
 709 /*
 710  * ubsec_feed() - aggregate and post requests to chip
 711  */
 712 static void
 713 ubsec_feed(struct ubsec_softc *sc)
 714 {
 715         struct ubsec_q *q, *q2;
 716         int npkts, i;
 717         void *v;
 718         u_int32_t stat;
 719
 720         /*
 721          * Decide how many ops to combine in a single MCR.  We cannot
 722          * aggregate more than UBS_MAX_AGGR because this is the number
 723          * of slots defined in the data structure.  Note that
 724          * aggregation only happens if ops are marked batch'able.
 725          * Aggregating ops reduces the number of interrupts to the host
 726          * but also (potentially) increases the latency for processing
 727          * completed ops as we only get an interrupt when all aggregated
 728          * ops have completed.
 729          */
 730         if (sc->sc_nqueue == 0)
 731                 return;
 732         if (sc->sc_nqueue > 1) {
 733                 npkts = 0;
 734                 SIMPLEQ_FOREACH(q, &sc->sc_queue, q_next) {
 735                         npkts++;
 736                         if ((q->q_crp->crp_flags & CRYPTO_F_BATCH) == 0)
 737                                 break;
 738                 }
 739         } else
 740                 npkts = 1;
 741         /*
 742          * Check device status before going any further.
 743          */
 744         if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) {
 745                 if (stat & BS_STAT_DMAERR) {
 746                         ubsec_totalreset(sc);
 747                         ubsecstats.hst_dmaerr++;
 748                 } else
 749                         ubsecstats.hst_mcr1full++;
 750                 return;
 751         }
 752         if (sc->sc_nqueue > ubsecstats.hst_maxqueue)
 753                 ubsecstats.hst_maxqueue = sc->sc_nqueue;
 754         if (npkts > UBS_MAX_AGGR)
 755                 npkts = UBS_MAX_AGGR;
 756         if (npkts < 2)                          /* special case 1 op */
 757                 goto feed1;
 758
 759         ubsecstats.hst_totbatch += npkts-1;
 760 #ifdef UBSEC_DEBUG
 761         if (ubsec_debug)
 762                 printf("merging %d records\n", npkts);
 763 #endif /* UBSEC_DEBUG */
 764
 765         q = SIMPLEQ_FIRST(&sc->sc_queue);
 766         SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next);
 767         --sc->sc_nqueue;
 768
 769         bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE);
 770         if (q->q_dst_map != NULL)
 771                 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD);
 772
 773         q->q_nstacked_mcrs = npkts - 1;         /* Number of packets stacked */
 774
 775         for (i = 0; i < q->q_nstacked_mcrs; i++) {
 776                 q2 = SIMPLEQ_FIRST(&sc->sc_queue);
 777                 bus_dmamap_sync(sc->sc_dmat, q2->q_src_map,
 778                     BUS_DMASYNC_PREWRITE);
 779                 if (q2->q_dst_map != NULL)
 780                         bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map,
 781                             BUS_DMASYNC_PREREAD);
 782                 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next);
 783                 --sc->sc_nqueue;
 784
 785                 v = (void*)(((char *)&q2->q_dma->d_dma->d_mcr) + sizeof(struct ubsec_mcr) -
 786                     sizeof(struct ubsec_mcr_add));
 787                 bcopy(v, &q->q_dma->d_dma->d_mcradd[i], sizeof(struct ubsec_mcr_add));
 788                 q->q_stacked_mcr[i] = q2;
 789         }
 790         q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts);
 791         SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
 792         sc->sc_nqchip += npkts;
 793         if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
 794                 ubsecstats.hst_maxqchip = sc->sc_nqchip;
 795         ubsec_dma_sync(&q->q_dma->d_alloc,
 796             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 797         WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
 798             offsetof(struct ubsec_dmachunk, d_mcr));
 799         return;
 800 feed1:
 801         q = SIMPLEQ_FIRST(&sc->sc_queue);
 802
 803         bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE);
 804         if (q->q_dst_map != NULL)
 805                 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD);
 806         ubsec_dma_sync(&q->q_dma->d_alloc,
 807             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 808
 809         WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
 810             offsetof(struct ubsec_dmachunk, d_mcr));
 811 #ifdef UBSEC_DEBUG
 812         if (ubsec_debug)
 813                 printf("feed1: q->chip %p %08x stat %08x\n",
 814                       q, (u_int32_t)vtophys(&q->q_dma->d_dma->d_mcr),
 815                       stat);
 816 #endif /* UBSEC_DEBUG */
 817         SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next);
 818         --sc->sc_nqueue;
 819         SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
 820         sc->sc_nqchip++;
 821         if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
 822                 ubsecstats.hst_maxqchip = sc->sc_nqchip;
 823         return;
 824 }
 825
 826 static void
 827 ubsec_setup_enckey(struct ubsec_session *ses, int algo, caddr_t key)
 828 {
 829
 830         /* Go ahead and compute key in ubsec's byte order */
 831         if (algo == CRYPTO_DES_CBC) {
 832                 bcopy(key, &ses->ses_deskey[0], 8);
 833                 bcopy(key, &ses->ses_deskey[2], 8);
 834                 bcopy(key, &ses->ses_deskey[4], 8);
 835         } else
 836                 bcopy(key, ses->ses_deskey, 24);
 837
 838         SWAP32(ses->ses_deskey[0]);
 839         SWAP32(ses->ses_deskey[1]);
 840         SWAP32(ses->ses_deskey[2]);
 841         SWAP32(ses->ses_deskey[3]);
 842         SWAP32(ses->ses_deskey[4]);
 843         SWAP32(ses->ses_deskey[5]);
 844 }
 845
 846 static void
 847 ubsec_setup_mackey(struct ubsec_session *ses, int algo, caddr_t key, int klen)
 848 {
 849         MD5_CTX md5ctx;
 850         SHA1_CTX sha1ctx;
 851         int i;
 852
 853         for (i = 0; i < klen; i++)
 854                 key[i] ^= HMAC_IPAD_VAL;
 855
 856         if (algo == CRYPTO_MD5_HMAC) {
 857                 MD5Init(&md5ctx);
 858                 MD5Update(&md5ctx, key, klen);
 859                 MD5Update(&md5ctx, hmac_ipad_buffer, MD5_HMAC_BLOCK_LEN - klen);
 860                 bcopy(md5ctx.state, ses->ses_hminner, sizeof(md5ctx.state));
 861         } else {
 862                 SHA1Init(&sha1ctx);
 863                 SHA1Update(&sha1ctx, key, klen);
 864                 SHA1Update(&sha1ctx, hmac_ipad_buffer,
 865                     SHA1_HMAC_BLOCK_LEN - klen);
 866                 bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32));
 867         }
 868
 869         for (i = 0; i < klen; i++)
 870                 key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
 871
 872         if (algo == CRYPTO_MD5_HMAC) {
 873                 MD5Init(&md5ctx);
 874                 MD5Update(&md5ctx, key, klen);
 875                 MD5Update(&md5ctx, hmac_opad_buffer, MD5_HMAC_BLOCK_LEN - klen);
 876                 bcopy(md5ctx.state, ses->ses_hmouter, sizeof(md5ctx.state));
 877         } else {
 878                 SHA1Init(&sha1ctx);
 879                 SHA1Update(&sha1ctx, key, klen);
 880                 SHA1Update(&sha1ctx, hmac_opad_buffer,
 881                     SHA1_HMAC_BLOCK_LEN - klen);
 882                 bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32));
 883         }
 884
 885         for (i = 0; i < klen; i++)
 886                 key[i] ^= HMAC_OPAD_VAL;
 887 }
 888
 889 /*
 890  * Allocate a new 'session' and return an encoded session id.  'sidp'
 891  * contains our registration id, and should contain an encoded session
 892  * id on successful allocation.
 893  */
 894 static int
 895 ubsec_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri)
 896 {
 897         struct ubsec_softc *sc = device_get_softc(dev);
 898         struct cryptoini *c, *encini = NULL, *macini = NULL;
 899         struct ubsec_session *ses = NULL;
 900         int sesn;
 901
 902         if (sidp == NULL || cri == NULL || sc == NULL)
 903                 return (EINVAL);
 904
 905         for (c = cri; c != NULL; c = c->cri_next) {
 906                 if (c->cri_alg == CRYPTO_MD5_HMAC ||
 907                     c->cri_alg == CRYPTO_SHA1_HMAC) {
 908                         if (macini)
 909                                 return (EINVAL);
 910                         macini = c;
 911                 } else if (c->cri_alg == CRYPTO_DES_CBC ||
 912                     c->cri_alg == CRYPTO_3DES_CBC) {
 913                         if (encini)
 914                                 return (EINVAL);
 915                         encini = c;
 916                 } else
 917                         return (EINVAL);
 918         }
 919         if (encini == NULL && macini == NULL)
 920                 return (EINVAL);
 921
 922         if (sc->sc_sessions == NULL) {
 923                 ses = sc->sc_sessions = (struct ubsec_session *)malloc(
 924                     sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT);
 925                 if (ses == NULL)
 926                         return (ENOMEM);
 927                 sesn = 0;
 928                 sc->sc_nsessions = 1;
 929         } else {
 930                 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
 931                         if (sc->sc_sessions[sesn].ses_used == 0) {
 932                                 ses = &sc->sc_sessions[sesn];
 933                                 break;
 934                         }
 935                 }
 936
 937                 if (ses == NULL) {
 938                         sesn = sc->sc_nsessions;
 939                         ses = (struct ubsec_session *)malloc((sesn + 1) *
 940                             sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT);
 941                         if (ses == NULL)
 942                                 return (ENOMEM);
 943                         bcopy(sc->sc_sessions, ses, sesn *
 944                             sizeof(struct ubsec_session));
 945                         bzero(sc->sc_sessions, sesn *
 946                             sizeof(struct ubsec_session));
 947                         free(sc->sc_sessions, M_DEVBUF);
 948                         sc->sc_sessions = ses;
 949                         ses = &sc->sc_sessions[sesn];
 950                         sc->sc_nsessions++;
 951                 }
 952         }
 953         bzero(ses, sizeof(struct ubsec_session));
 954         ses->ses_used = 1;
 955
 956         if (encini) {
 957                 /* get an IV, network byte order */
 958                 /* XXX may read fewer than requested */
 959                 read_random(ses->ses_iv, sizeof(ses->ses_iv));
 960
 961                 if (encini->cri_key != NULL) {
 962                         ubsec_setup_enckey(ses, encini->cri_alg,
 963                             encini->cri_key);
 964                 }
 965         }
 966
 967         if (macini) {
 968                 ses->ses_mlen = macini->cri_mlen;
 969                 if (ses->ses_mlen == 0) {
 970                         if (macini->cri_alg == CRYPTO_MD5_HMAC)
 971                                 ses->ses_mlen = MD5_HASH_LEN;
 972                         else
 973                                 ses->ses_mlen = SHA1_HASH_LEN;
 974                 }
 975
 976                 if (macini->cri_key != NULL) {
 977                         ubsec_setup_mackey(ses, macini->cri_alg,
 978                             macini->cri_key, macini->cri_klen / 8);
 979                 }
 980         }
 981
 982         *sidp = UBSEC_SID(device_get_unit(sc->sc_dev), sesn);
 983         return (0);
 984 }
 985
 986 /*
 987  * Deallocate a session.
 988  */
 989 static int
 990 ubsec_freesession(device_t dev, u_int64_t tid)
 991 {
 992         struct ubsec_softc *sc = device_get_softc(dev);
 993         int session, ret;
 994         u_int32_t sid = CRYPTO_SESID2LID(tid);
 995
 996         if (sc == NULL)
 997                 return (EINVAL);
 998
 999         session = UBSEC_SESSION(sid);
1000         if (session < sc->sc_nsessions) {
1001                 bzero(&sc->sc_sessions[session],
1002                         sizeof(sc->sc_sessions[session]));
1003                 ret = 0;
1004         } else
1005                 ret = EINVAL;
1006
1007         return (ret);
1008 }
1009
1010 static void
1011 ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error)
1012 {
1013         struct ubsec_operand *op = arg;
1014
1015         KASSERT(nsegs <= UBS_MAX_SCATTER,
1016                 ("Too many DMA segments returned when mapping operand"));
1017 #ifdef UBSEC_DEBUG
1018         if (ubsec_debug)
1019                 printf("ubsec_op_cb: mapsize %u nsegs %d error %d\n",
1020                         (u_int) mapsize, nsegs, error);
1021 #endif
1022         if (error != 0)
1023                 return;
1024         op->mapsize = mapsize;
1025         op->nsegs = nsegs;
1026         bcopy(seg, op->segs, nsegs * sizeof (seg[0]));
1027 }
1028
1029 static int
1030 ubsec_process(device_t dev, struct cryptop *crp, int hint)
1031 {
1032         struct ubsec_softc *sc = device_get_softc(dev);
1033         struct ubsec_q *q = NULL;
1034         int err = 0, i, j, nicealign;
1035         struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
1036         int encoffset = 0, macoffset = 0, cpskip, cpoffset;
1037         int sskip, dskip, stheend, dtheend;
1038         int16_t coffset;
1039         struct ubsec_session *ses;
1040         struct ubsec_pktctx ctx;
1041         struct ubsec_dma *dmap = NULL;
1042
1043         if (crp == NULL || crp->crp_callback == NULL || sc == NULL) {
1044                 ubsecstats.hst_invalid++;
1045                 return (EINVAL);
1046         }
1047         if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) {
1048                 ubsecstats.hst_badsession++;
1049                 return (EINVAL);
1050         }
1051
1052         mtx_lock(&sc->sc_freeqlock);
1053         if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
1054                 ubsecstats.hst_queuefull++;
1055                 sc->sc_needwakeup |= CRYPTO_SYMQ;
1056                 mtx_unlock(&sc->sc_freeqlock);
1057                 return (ERESTART);
1058         }
1059         q = SIMPLEQ_FIRST(&sc->sc_freequeue);
1060         SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q_next);
1061         mtx_unlock(&sc->sc_freeqlock);
1062
1063         dmap = q->q_dma; /* Save dma pointer */
1064         bzero(q, sizeof(struct ubsec_q));
1065         bzero(&ctx, sizeof(ctx));
1066
1067         q->q_sesn = UBSEC_SESSION(crp->crp_sid);
1068         q->q_dma = dmap;
1069         ses = &sc->sc_sessions[q->q_sesn];
1070
1071         if (crp->crp_flags & CRYPTO_F_IMBUF) {
1072                 q->q_src_m = (struct mbuf *)crp->crp_buf;
1073                 q->q_dst_m = (struct mbuf *)crp->crp_buf;
1074         } else if (crp->crp_flags & CRYPTO_F_IOV) {
1075                 q->q_src_io = (struct uio *)crp->crp_buf;
1076                 q->q_dst_io = (struct uio *)crp->crp_buf;
1077         } else {
1078                 ubsecstats.hst_badflags++;
1079                 err = EINVAL;
1080                 goto errout;    /* XXX we don't handle contiguous blocks! */
1081         }
1082
1083         bzero(&dmap->d_dma->d_mcr, sizeof(struct ubsec_mcr));
1084
1085         dmap->d_dma->d_mcr.mcr_pkts = htole16(1);
1086         dmap->d_dma->d_mcr.mcr_flags = 0;
1087         q->q_crp = crp;
1088
1089         crd1 = crp->crp_desc;
1090         if (crd1 == NULL) {
1091                 ubsecstats.hst_nodesc++;
1092                 err = EINVAL;
1093                 goto errout;
1094         }
1095         crd2 = crd1->crd_next;
1096
1097         if (crd2 == NULL) {
1098                 if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
1099                     crd1->crd_alg == CRYPTO_SHA1_HMAC) {
1100                         maccrd = crd1;
1101                         enccrd = NULL;
1102                 } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
1103                     crd1->crd_alg == CRYPTO_3DES_CBC) {
1104                         maccrd = NULL;
1105                         enccrd = crd1;
1106                 } else {
1107                         ubsecstats.hst_badalg++;
1108                         err = EINVAL;
1109                         goto errout;
1110                 }
1111         } else {
1112                 if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
1113                     crd1->crd_alg == CRYPTO_SHA1_HMAC) &&
1114                     (crd2->crd_alg == CRYPTO_DES_CBC ||
1115                         crd2->crd_alg == CRYPTO_3DES_CBC) &&
1116                     ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
1117                         maccrd = crd1;
1118                         enccrd = crd2;
1119                 } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
1120                     crd1->crd_alg == CRYPTO_3DES_CBC) &&
1121                     (crd2->crd_alg == CRYPTO_MD5_HMAC ||
1122                         crd2->crd_alg == CRYPTO_SHA1_HMAC) &&
1123                     (crd1->crd_flags & CRD_F_ENCRYPT)) {
1124                         enccrd = crd1;
1125                         maccrd = crd2;
1126                 } else {
1127                         /*
1128                          * We cannot order the ubsec as requested
1129                          */
1130                         ubsecstats.hst_badalg++;
1131                         err = EINVAL;
1132                         goto errout;
1133                 }
1134         }
1135
1136         if (enccrd) {
1137                 if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
1138                         ubsec_setup_enckey(ses, enccrd->crd_alg,
1139                             enccrd->crd_key);
1140                 }
1141
1142                 encoffset = enccrd->crd_skip;
1143                 ctx.pc_flags |= htole16(UBS_PKTCTX_ENC_3DES);
1144
1145                 if (enccrd->crd_flags & CRD_F_ENCRYPT) {
1146                         q->q_flags |= UBSEC_QFLAGS_COPYOUTIV;
1147
1148                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
1149                                 bcopy(enccrd->crd_iv, ctx.pc_iv, 8);
1150                         else {
1151                                 ctx.pc_iv[0] = ses->ses_iv[0];
1152                                 ctx.pc_iv[1] = ses->ses_iv[1];
1153                         }
1154
1155                         if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
1156                                 crypto_copyback(crp->crp_flags, crp->crp_buf,
1157                                     enccrd->crd_inject, 8, (caddr_t)ctx.pc_iv);
1158                         }
1159                 } else {
1160                         ctx.pc_flags |= htole16(UBS_PKTCTX_INBOUND);
1161
1162                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
1163                                 bcopy(enccrd->crd_iv, ctx.pc_iv, 8);
1164                         else {
1165                                 crypto_copydata(crp->crp_flags, crp->crp_buf,
1166                                     enccrd->crd_inject, 8, (caddr_t)ctx.pc_iv);
1167                         }
1168                 }
1169
1170                 ctx.pc_deskey[0] = ses->ses_deskey[0];
1171                 ctx.pc_deskey[1] = ses->ses_deskey[1];
1172                 ctx.pc_deskey[2] = ses->ses_deskey[2];
1173                 ctx.pc_deskey[3] = ses->ses_deskey[3];
1174                 ctx.pc_deskey[4] = ses->ses_deskey[4];
1175                 ctx.pc_deskey[5] = ses->ses_deskey[5];
1176                 SWAP32(ctx.pc_iv[0]);
1177                 SWAP32(ctx.pc_iv[1]);
1178         }
1179
1180         if (maccrd) {
1181                 if (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
1182                         ubsec_setup_mackey(ses, maccrd->crd_alg,
1183                             maccrd->crd_key, maccrd->crd_klen / 8);
1184                 }
1185
1186                 macoffset = maccrd->crd_skip;
1187
1188                 if (maccrd->crd_alg == CRYPTO_MD5_HMAC)
1189                         ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_MD5);
1190                 else
1191                         ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_SHA1);
1192
1193                 for (i = 0; i < 5; i++) {
1194                         ctx.pc_hminner[i] = ses->ses_hminner[i];
1195                         ctx.pc_hmouter[i] = ses->ses_hmouter[i];
1196
1197                         HTOLE32(ctx.pc_hminner[i]);
1198                         HTOLE32(ctx.pc_hmouter[i]);
1199                 }
1200         }
1201
1202         if (enccrd && maccrd) {
1203                 /*
1204                  * ubsec cannot handle packets where the end of encryption
1205                  * and authentication are not the same, or where the
1206                  * encrypted part begins before the authenticated part.
1207                  */
1208                 if ((encoffset + enccrd->crd_len) !=
1209                     (macoffset + maccrd->crd_len)) {
1210                         ubsecstats.hst_lenmismatch++;
1211                         err = EINVAL;
1212                         goto errout;
1213                 }
1214                 if (enccrd->crd_skip < maccrd->crd_skip) {
1215                         ubsecstats.hst_skipmismatch++;
1216                         err = EINVAL;
1217                         goto errout;
1218                 }
1219                 sskip = maccrd->crd_skip;
1220                 cpskip = dskip = enccrd->crd_skip;
1221                 stheend = maccrd->crd_len;
1222                 dtheend = enccrd->crd_len;
1223                 coffset = enccrd->crd_skip - maccrd->crd_skip;
1224                 cpoffset = cpskip + dtheend;
1225 #ifdef UBSEC_DEBUG
1226                 if (ubsec_debug) {
1227                         printf("mac: skip %d, len %d, inject %d\n",
1228                             maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject);
1229                         printf("enc: skip %d, len %d, inject %d\n",
1230                             enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject);
1231                         printf("src: skip %d, len %d\n", sskip, stheend);
1232                         printf("dst: skip %d, len %d\n", dskip, dtheend);
1233                         printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n",
1234                             coffset, stheend, cpskip, cpoffset);
1235                 }
1236 #endif
1237         } else {
1238                 cpskip = dskip = sskip = macoffset + encoffset;
1239                 dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len;
1240                 cpoffset = cpskip + dtheend;
1241                 coffset = 0;
1242         }
1243         ctx.pc_offset = htole16(coffset >> 2);
1244
1245         if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &q->q_src_map)) {
1246                 ubsecstats.hst_nomap++;
1247                 err = ENOMEM;
1248                 goto errout;
1249         }
1250         if (crp->crp_flags & CRYPTO_F_IMBUF) {
1251                 if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map,
1252                     q->q_src_m, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) {
1253                         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1254                         q->q_src_map = NULL;
1255                         ubsecstats.hst_noload++;
1256                         err = ENOMEM;
1257                         goto errout;
1258                 }
1259         } else if (crp->crp_flags & CRYPTO_F_IOV) {
1260                 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map,
1261                     q->q_src_io, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) {
1262                         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1263                         q->q_src_map = NULL;
1264                         ubsecstats.hst_noload++;
1265                         err = ENOMEM;
1266                         goto errout;
1267                 }
1268         }
1269         nicealign = ubsec_dmamap_aligned(&q->q_src);
1270
1271         dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend);
1272
1273 #ifdef UBSEC_DEBUG
1274         if (ubsec_debug)
1275                 printf("src skip: %d nicealign: %u\n", sskip, nicealign);
1276 #endif
1277         for (i = j = 0; i < q->q_src_nsegs; i++) {
1278                 struct ubsec_pktbuf *pb;
1279                 bus_size_t packl = q->q_src_segs[i].ds_len;
1280                 bus_addr_t packp = q->q_src_segs[i].ds_addr;
1281
1282                 if (sskip >= packl) {
1283                         sskip -= packl;
1284                         continue;
1285                 }
1286
1287                 packl -= sskip;
1288                 packp += sskip;
1289                 sskip = 0;
1290
1291                 if (packl > 0xfffc) {
1292                         err = EIO;
1293                         goto errout;
1294                 }
1295
1296                 if (j == 0)
1297                         pb = &dmap->d_dma->d_mcr.mcr_ipktbuf;
1298                 else
1299                         pb = &dmap->d_dma->d_sbuf[j - 1];
1300
1301                 pb->pb_addr = htole32(packp);
1302
1303                 if (stheend) {
1304                         if (packl > stheend) {
1305                                 pb->pb_len = htole32(stheend);
1306                                 stheend = 0;
1307                         } else {
1308                                 pb->pb_len = htole32(packl);
1309                                 stheend -= packl;
1310                         }
1311                 } else
1312                         pb->pb_len = htole32(packl);
1313
1314                 if ((i + 1) == q->q_src_nsegs)
1315                         pb->pb_next = 0;
1316                 else
1317                         pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1318                             offsetof(struct ubsec_dmachunk, d_sbuf[j]));
1319                 j++;
1320         }
1321
1322         if (enccrd == NULL && maccrd != NULL) {
1323                 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0;
1324                 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0;
1325                 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr +
1326                     offsetof(struct ubsec_dmachunk, d_macbuf[0]));
1327 #ifdef UBSEC_DEBUG
1328                 if (ubsec_debug)
1329                         printf("opkt: %x %x %x\n",
1330                             dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr,
1331                             dmap->d_dma->d_mcr.mcr_opktbuf.pb_len,
1332                             dmap->d_dma->d_mcr.mcr_opktbuf.pb_next);
1333 #endif
1334         } else {
1335                 if (crp->crp_flags & CRYPTO_F_IOV) {
1336                         if (!nicealign) {
1337                                 ubsecstats.hst_iovmisaligned++;
1338                                 err = EINVAL;
1339                                 goto errout;
1340                         }
1341                         if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT,
1342                              &q->q_dst_map)) {
1343                                 ubsecstats.hst_nomap++;
1344                                 err = ENOMEM;
1345                                 goto errout;
1346                         }
1347                         if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map,
1348                             q->q_dst_io, ubsec_op_cb, &q->q_dst, BUS_DMA_NOWAIT) != 0) {
1349                                 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
1350                                 q->q_dst_map = NULL;
1351                                 ubsecstats.hst_noload++;
1352                                 err = ENOMEM;
1353                                 goto errout;
1354                         }
1355                 } else if (crp->crp_flags & CRYPTO_F_IMBUF) {
1356                         if (nicealign) {
1357                                 q->q_dst = q->q_src;
1358                         } else {
1359                                 int totlen, len;
1360                                 struct mbuf *m, *top, **mp;
1361
1362                                 ubsecstats.hst_unaligned++;
1363                                 totlen = q->q_src_mapsize;
1364                                 if (totlen >= MINCLSIZE) {
1365                                         m = m_getcl(M_NOWAIT, MT_DATA,
1366                                             q->q_src_m->m_flags & M_PKTHDR);
1367                                         len = MCLBYTES;
1368                                 } else if (q->q_src_m->m_flags & M_PKTHDR) {
1369                                         m = m_gethdr(M_NOWAIT, MT_DATA);
1370                                         len = MHLEN;
1371                                 } else {
1372                                         m = m_get(M_NOWAIT, MT_DATA);
1373                                         len = MLEN;
1374                                 }
1375                                 if (m && q->q_src_m->m_flags & M_PKTHDR &&
1376                                     !m_dup_pkthdr(m, q->q_src_m, M_NOWAIT)) {
1377                                         m_free(m);
1378                                         m = NULL;
1379                                 }
1380                                 if (m == NULL) {
1381                                         ubsecstats.hst_nombuf++;
1382                                         err = sc->sc_nqueue ? ERESTART : ENOMEM;
1383                                         goto errout;
1384                                 }
1385                                 m->m_len = len = min(totlen, len);
1386                                 totlen -= len;
1387                                 top = m;
1388                                 mp = &top;
1389
1390                                 while (totlen > 0) {
1391                                         if (totlen >= MINCLSIZE) {
1392                                                 m = m_getcl(M_NOWAIT,
1393                                                     MT_DATA, 0);
1394                                                 len = MCLBYTES;
1395                                         } else {
1396                                                 m = m_get(M_NOWAIT, MT_DATA);
1397                                                 len = MLEN;
1398                                         }
1399                                         if (m == NULL) {
1400                                                 m_freem(top);
1401                                                 ubsecstats.hst_nombuf++;
1402                                                 err = sc->sc_nqueue ? ERESTART : ENOMEM;
1403                                                 goto errout;
1404                                         }
1405                                         m->m_len = len = min(totlen, len);
1406                                         totlen -= len;
1407                                         *mp = m;
1408                                         mp = &m->m_next;
1409                                 }
1410                                 q->q_dst_m = top;
1411                                 ubsec_mcopy(q->q_src_m, q->q_dst_m,
1412                                     cpskip, cpoffset);
1413                                 if (bus_dmamap_create(sc->sc_dmat,
1414                                     BUS_DMA_NOWAIT, &q->q_dst_map) != 0) {
1415                                         ubsecstats.hst_nomap++;
1416                                         err = ENOMEM;
1417                                         goto errout;
1418                                 }
1419                                 if (bus_dmamap_load_mbuf(sc->sc_dmat,
1420                                     q->q_dst_map, q->q_dst_m,
1421                                     ubsec_op_cb, &q->q_dst,
1422                                     BUS_DMA_NOWAIT) != 0) {
1423                                         bus_dmamap_destroy(sc->sc_dmat,
1424                                         q->q_dst_map);
1425                                         q->q_dst_map = NULL;
1426                                         ubsecstats.hst_noload++;
1427                                         err = ENOMEM;
1428                                         goto errout;
1429                                 }
1430                         }
1431                 } else {
1432                         ubsecstats.hst_badflags++;
1433                         err = EINVAL;
1434                         goto errout;
1435                 }
1436
1437 #ifdef UBSEC_DEBUG
1438                 if (ubsec_debug)
1439                         printf("dst skip: %d\n", dskip);
1440 #endif
1441                 for (i = j = 0; i < q->q_dst_nsegs; i++) {
1442                         struct ubsec_pktbuf *pb;
1443                         bus_size_t packl = q->q_dst_segs[i].ds_len;
1444                         bus_addr_t packp = q->q_dst_segs[i].ds_addr;
1445
1446                         if (dskip >= packl) {
1447                                 dskip -= packl;
1448                                 continue;
1449                         }
1450
1451                         packl -= dskip;
1452                         packp += dskip;
1453                         dskip = 0;
1454
1455                         if (packl > 0xfffc) {
1456                                 err = EIO;
1457                                 goto errout;
1458                         }
1459
1460                         if (j == 0)
1461                                 pb = &dmap->d_dma->d_mcr.mcr_opktbuf;
1462                         else
1463                                 pb = &dmap->d_dma->d_dbuf[j - 1];
1464
1465                         pb->pb_addr = htole32(packp);
1466
1467                         if (dtheend) {
1468                                 if (packl > dtheend) {
1469                                         pb->pb_len = htole32(dtheend);
1470                                         dtheend = 0;
1471                                 } else {
1472                                         pb->pb_len = htole32(packl);
1473                                         dtheend -= packl;
1474                                 }
1475                         } else
1476                                 pb->pb_len = htole32(packl);
1477
1478                         if ((i + 1) == q->q_dst_nsegs) {
1479                                 if (maccrd)
1480                                         pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1481                                             offsetof(struct ubsec_dmachunk, d_macbuf[0]));
1482                                 else
1483                                         pb->pb_next = 0;
1484                         } else
1485                                 pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1486                                     offsetof(struct ubsec_dmachunk, d_dbuf[j]));
1487                         j++;
1488                 }
1489         }
1490
1491         dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr +
1492             offsetof(struct ubsec_dmachunk, d_ctx));
1493
1494         if (sc->sc_flags & UBS_FLAGS_LONGCTX) {
1495                 struct ubsec_pktctx_long *ctxl;
1496
1497                 ctxl = (struct ubsec_pktctx_long *)(dmap->d_alloc.dma_vaddr +
1498                     offsetof(struct ubsec_dmachunk, d_ctx));
1499
1500                 /* transform small context into long context */
1501                 ctxl->pc_len = htole16(sizeof(struct ubsec_pktctx_long));
1502                 ctxl->pc_type = htole16(UBS_PKTCTX_TYPE_IPSEC);
1503                 ctxl->pc_flags = ctx.pc_flags;
1504                 ctxl->pc_offset = ctx.pc_offset;
1505                 for (i = 0; i < 6; i++)
1506                         ctxl->pc_deskey[i] = ctx.pc_deskey[i];
1507                 for (i = 0; i < 5; i++)
1508                         ctxl->pc_hminner[i] = ctx.pc_hminner[i];
1509                 for (i = 0; i < 5; i++)
1510                         ctxl->pc_hmouter[i] = ctx.pc_hmouter[i];
1511                 ctxl->pc_iv[0] = ctx.pc_iv[0];
1512                 ctxl->pc_iv[1] = ctx.pc_iv[1];
1513         } else
1514                 bcopy(&ctx, dmap->d_alloc.dma_vaddr +
1515                     offsetof(struct ubsec_dmachunk, d_ctx),
1516                     sizeof(struct ubsec_pktctx));
1517
1518         mtx_lock(&sc->sc_mcr1lock);
1519         SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next);
1520         sc->sc_nqueue++;
1521         ubsecstats.hst_ipackets++;
1522         ubsecstats.hst_ibytes += dmap->d_alloc.dma_size;
1523         if ((hint & CRYPTO_HINT_MORE) == 0 || sc->sc_nqueue >= UBS_MAX_AGGR)
1524                 ubsec_feed(sc);
1525         mtx_unlock(&sc->sc_mcr1lock);
1526         return (0);
1527
1528 errout:
1529         if (q != NULL) {
1530                 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
1531                         m_freem(q->q_dst_m);
1532
1533                 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
1534                         bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
1535                         bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
1536                 }
1537                 if (q->q_src_map != NULL) {
1538                         bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
1539                         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1540                 }
1541         }
1542         if (q != NULL || err == ERESTART) {
1543                 mtx_lock(&sc->sc_freeqlock);
1544                 if (q != NULL)
1545                         SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
1546                 if (err == ERESTART)
1547                         sc->sc_needwakeup |= CRYPTO_SYMQ;
1548                 mtx_unlock(&sc->sc_freeqlock);
1549         }
1550         if (err != ERESTART) {
1551                 crp->crp_etype = err;
1552                 crypto_done(crp);
1553         }
1554         return (err);
1555 }
1556
1557 static void
1558 ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q)
1559 {
1560         struct cryptop *crp = (struct cryptop *)q->q_crp;
1561         struct cryptodesc *crd;
1562         struct ubsec_dma *dmap = q->q_dma;
1563
1564         ubsecstats.hst_opackets++;
1565         ubsecstats.hst_obytes += dmap->d_alloc.dma_size;
1566
1567         ubsec_dma_sync(&dmap->d_alloc,
1568             BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
1569         if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
1570                 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map,
1571                     BUS_DMASYNC_POSTREAD);
1572                 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
1573                 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
1574         }
1575         bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_POSTWRITE);
1576         bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
1577         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1578
1579         if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) {
1580                 m_freem(q->q_src_m);
1581                 crp->crp_buf = (caddr_t)q->q_dst_m;
1582         }
1583
1584         /* copy out IV for future use */
1585         if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) {
1586                 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1587                         if (crd->crd_alg != CRYPTO_DES_CBC &&
1588                             crd->crd_alg != CRYPTO_3DES_CBC)
1589                                 continue;
1590                         crypto_copydata(crp->crp_flags, crp->crp_buf,
1591                             crd->crd_skip + crd->crd_len - 8, 8,
1592                             (caddr_t)sc->sc_sessions[q->q_sesn].ses_iv);
1593                         break;
1594                 }
1595         }
1596
1597         for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1598                 if (crd->crd_alg != CRYPTO_MD5_HMAC &&
1599                     crd->crd_alg != CRYPTO_SHA1_HMAC)
1600                         continue;
1601                 crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject,
1602                     sc->sc_sessions[q->q_sesn].ses_mlen,
1603                     (caddr_t)dmap->d_dma->d_macbuf);
1604                 break;
1605         }
1606         mtx_lock(&sc->sc_freeqlock);
1607         SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
1608         mtx_unlock(&sc->sc_freeqlock);
1609         crypto_done(crp);
1610 }
1611
1612 static void
1613 ubsec_mcopy(struct mbuf *srcm, struct mbuf *dstm, int hoffset, int toffset)
1614 {
1615         int i, j, dlen, slen;
1616         caddr_t dptr, sptr;
1617
1618         j = 0;
1619         sptr = srcm->m_data;
1620         slen = srcm->m_len;
1621         dptr = dstm->m_data;
1622         dlen = dstm->m_len;
1623
1624         while (1) {
1625                 for (i = 0; i < min(slen, dlen); i++) {
1626                         if (j < hoffset || j >= toffset)
1627                                 *dptr++ = *sptr++;
1628                         slen--;
1629                         dlen--;
1630                         j++;
1631                 }
1632                 if (slen == 0) {
1633                         srcm = srcm->m_next;
1634                         if (srcm == NULL)
1635                                 return;
1636                         sptr = srcm->m_data;
1637                         slen = srcm->m_len;
1638                 }
1639                 if (dlen == 0) {
1640                         dstm = dstm->m_next;
1641                         if (dstm == NULL)
1642                                 return;
1643                         dptr = dstm->m_data;
1644                         dlen = dstm->m_len;
1645                 }
1646         }
1647 }
1648
1649 /*
1650  * feed the key generator, must be called at splimp() or higher.
1651  */
1652 static int
1653 ubsec_feed2(struct ubsec_softc *sc)
1654 {
1655         struct ubsec_q2 *q;
1656
1657         while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) {
1658                 if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL)
1659                         break;
1660                 q = SIMPLEQ_FIRST(&sc->sc_queue2);
1661
1662                 ubsec_dma_sync(&q->q_mcr,
1663                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1664                 ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_PREWRITE);
1665
1666                 WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr);
1667                 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, q_next);
1668                 --sc->sc_nqueue2;
1669                 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next);
1670         }
1671         return (0);
1672 }
1673
1674 /*
1675  * Callback for handling random numbers
1676  */
1677 static void
1678 ubsec_callback2(struct ubsec_softc *sc, struct ubsec_q2 *q)
1679 {
1680         struct cryptkop *krp;
1681         struct ubsec_ctx_keyop *ctx;
1682
1683         ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr;
1684         ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_POSTWRITE);
1685
1686         switch (q->q_type) {
1687 #ifndef UBSEC_NO_RNG
1688         case UBS_CTXOP_RNGBYPASS: {
1689                 struct ubsec_q2_rng *rng = (struct ubsec_q2_rng *)q;
1690
1691                 ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_POSTREAD);
1692                 (*sc->sc_harvest)(sc->sc_rndtest,
1693                         rng->rng_buf.dma_vaddr,
1694                         UBSEC_RNG_BUFSIZ*sizeof (u_int32_t));
1695                 rng->rng_used = 0;
1696                 callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
1697                 break;
1698         }
1699 #endif
1700         case UBS_CTXOP_MODEXP: {
1701                 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q;
1702                 u_int rlen, clen;
1703
1704                 krp = me->me_krp;
1705                 rlen = (me->me_modbits + 7) / 8;
1706                 clen = (krp->krp_param[krp->krp_iparams].crp_nbits + 7) / 8;
1707
1708                 ubsec_dma_sync(&me->me_M, BUS_DMASYNC_POSTWRITE);
1709                 ubsec_dma_sync(&me->me_E, BUS_DMASYNC_POSTWRITE);
1710                 ubsec_dma_sync(&me->me_C, BUS_DMASYNC_POSTREAD);
1711                 ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_POSTWRITE);
1712
1713                 if (clen < rlen)
1714                         krp->krp_status = E2BIG;
1715                 else {
1716                         if (sc->sc_flags & UBS_FLAGS_HWNORM) {
1717                                 bzero(krp->krp_param[krp->krp_iparams].crp_p,
1718                                     (krp->krp_param[krp->krp_iparams].crp_nbits
1719                                         + 7) / 8);
1720                                 bcopy(me->me_C.dma_vaddr,
1721                                     krp->krp_param[krp->krp_iparams].crp_p,
1722                                     (me->me_modbits + 7) / 8);
1723                         } else
1724                                 ubsec_kshift_l(me->me_shiftbits,
1725                                     me->me_C.dma_vaddr, me->me_normbits,
1726                                     krp->krp_param[krp->krp_iparams].crp_p,
1727                                     krp->krp_param[krp->krp_iparams].crp_nbits);
1728                 }
1729
1730                 crypto_kdone(krp);
1731
1732                 /* bzero all potentially sensitive data */
1733                 bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
1734                 bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
1735                 bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
1736                 bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
1737
1738                 /* Can't free here, so put us on the free list. */
1739                 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next);
1740                 break;
1741         }
1742         case UBS_CTXOP_RSAPRIV: {
1743                 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q;
1744                 u_int len;
1745
1746                 krp = rp->rpr_krp;
1747                 ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_POSTWRITE);
1748                 ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_POSTREAD);
1749
1750                 len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + 7) / 8;
1751                 bcopy(rp->rpr_msgout.dma_vaddr,
1752                     krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len);
1753
1754                 crypto_kdone(krp);
1755
1756                 bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size);
1757                 bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size);
1758                 bzero(rp->rpr_q.q_ctx.dma_vaddr, rp->rpr_q.q_ctx.dma_size);
1759
1760                 /* Can't free here, so put us on the free list. */
1761                 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next);
1762                 break;
1763         }
1764         default:
1765                 device_printf(sc->sc_dev, "unknown ctx op: %x\n",
1766                     letoh16(ctx->ctx_op));
1767                 break;
1768         }
1769 }
1770
1771 #ifndef UBSEC_NO_RNG
1772 static void
1773 ubsec_rng(void *vsc)
1774 {
1775         struct ubsec_softc *sc = vsc;
1776         struct ubsec_q2_rng *rng = &sc->sc_rng;
1777         struct ubsec_mcr *mcr;
1778         struct ubsec_ctx_rngbypass *ctx;
1779
1780         mtx_lock(&sc->sc_mcr2lock);
1781         if (rng->rng_used) {
1782                 mtx_unlock(&sc->sc_mcr2lock);
1783                 return;
1784         }
1785         sc->sc_nqueue2++;
1786         if (sc->sc_nqueue2 >= UBS_MAX_NQUEUE)
1787                 goto out;
1788
1789         mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr;
1790         ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr;
1791
1792         mcr->mcr_pkts = htole16(1);
1793         mcr->mcr_flags = 0;
1794         mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr);
1795         mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = 0;
1796         mcr->mcr_ipktbuf.pb_len = 0;
1797         mcr->mcr_reserved = mcr->mcr_pktlen = 0;
1798         mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr);
1799         mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) &
1800             UBS_PKTBUF_LEN);
1801         mcr->mcr_opktbuf.pb_next = 0;
1802
1803         ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass));
1804         ctx->rbp_op = htole16(UBS_CTXOP_RNGBYPASS);
1805         rng->rng_q.q_type = UBS_CTXOP_RNGBYPASS;
1806
1807         ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_PREREAD);
1808
1809         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next);
1810         rng->rng_used = 1;
1811         ubsec_feed2(sc);
1812         ubsecstats.hst_rng++;
1813         mtx_unlock(&sc->sc_mcr2lock);
1814
1815         return;
1816
1817 out:
1818         /*
1819          * Something weird happened, generate our own call back.
1820          */
1821         sc->sc_nqueue2--;
1822         mtx_unlock(&sc->sc_mcr2lock);
1823         callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
1824 }
1825 #endif /* UBSEC_NO_RNG */
1826
1827 static void
1828 ubsec_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
1829 {
1830         bus_addr_t *paddr = (bus_addr_t*) arg;
1831         *paddr = segs->ds_addr;
1832 }
1833
1834 static int
1835 ubsec_dma_malloc(
1836         struct ubsec_softc *sc,
1837         bus_size_t size,
1838         struct ubsec_dma_alloc *dma,
1839         int mapflags
1840 )
1841 {
1842         int r;
1843
1844         /* XXX could specify sc_dmat as parent but that just adds overhead */
1845         r = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev),     /* parent */
1846                                1, 0,                    /* alignment, bounds */
1847                                BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
1848                                BUS_SPACE_MAXADDR,       /* highaddr */
1849                                NULL, NULL,              /* filter, filterarg */
1850                                size,                    /* maxsize */
1851                                1,                       /* nsegments */
1852                                size,                    /* maxsegsize */
1853                                BUS_DMA_ALLOCNOW,        /* flags */
1854                                NULL, NULL,              /* lockfunc, lockarg */
1855                                &dma->dma_tag);
1856         if (r != 0) {
1857                 device_printf(sc->sc_dev, "ubsec_dma_malloc: "
1858                         "bus_dma_tag_create failed; error %u\n", r);
1859                 goto fail_1;
1860         }
1861
1862         r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
1863                              BUS_DMA_NOWAIT, &dma->dma_map);
1864         if (r != 0) {
1865                 device_printf(sc->sc_dev, "ubsec_dma_malloc: "
1866                         "bus_dmammem_alloc failed; size %ju, error %u\n",
1867                         (intmax_t)size, r);
1868                 goto fail_2;
1869         }
1870
1871         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
1872                             size,
1873                             ubsec_dmamap_cb,
1874                             &dma->dma_paddr,
1875                             mapflags | BUS_DMA_NOWAIT);
1876         if (r != 0) {
1877                 device_printf(sc->sc_dev, "ubsec_dma_malloc: "
1878                         "bus_dmamap_load failed; error %u\n", r);
1879                 goto fail_3;
1880         }
1881
1882         dma->dma_size = size;
1883         return (0);
1884
1885 fail_3:
1886         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
1887 fail_2:
1888         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1889 fail_1:
1890         bus_dma_tag_destroy(dma->dma_tag);
1891         dma->dma_tag = NULL;
1892         return (r);
1893 }
1894
1895 static void
1896 ubsec_dma_free(struct ubsec_softc *sc, struct ubsec_dma_alloc *dma)
1897 {
1898         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
1899         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1900         bus_dma_tag_destroy(dma->dma_tag);
1901 }
1902
1903 /*
1904  * Resets the board.  Values in the regesters are left as is
1905  * from the reset (i.e. initial values are assigned elsewhere).
1906  */
1907 static void
1908 ubsec_reset_board(struct ubsec_softc *sc)
1909 {
1910     volatile u_int32_t ctrl;
1911
1912     ctrl = READ_REG(sc, BS_CTRL);
1913     ctrl |= BS_CTRL_RESET;
1914     WRITE_REG(sc, BS_CTRL, ctrl);
1915
1916     /*
1917      * Wait aprox. 30 PCI clocks = 900 ns = 0.9 us
1918      */
1919     DELAY(10);
1920 }
1921
1922 /*
1923  * Init Broadcom registers
1924  */
1925 static void
1926 ubsec_init_board(struct ubsec_softc *sc)
1927 {
1928         u_int32_t ctrl;
1929
1930         ctrl = READ_REG(sc, BS_CTRL);
1931         ctrl &= ~(BS_CTRL_BE32 | BS_CTRL_BE64);
1932         ctrl |= BS_CTRL_LITTLE_ENDIAN | BS_CTRL_MCR1INT;
1933
1934         if (sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG))
1935                 ctrl |= BS_CTRL_MCR2INT;
1936         else
1937                 ctrl &= ~BS_CTRL_MCR2INT;
1938
1939         if (sc->sc_flags & UBS_FLAGS_HWNORM)
1940                 ctrl &= ~BS_CTRL_SWNORM;
1941
1942         WRITE_REG(sc, BS_CTRL, ctrl);
1943 }
1944
1945 /*
1946  * Init Broadcom PCI registers
1947  */
1948 static void
1949 ubsec_init_pciregs(device_t dev)
1950 {
1951 #if 0
1952         u_int32_t misc;
1953
1954         misc = pci_conf_read(pc, pa->pa_tag, BS_RTY_TOUT);
1955         misc = (misc & ~(UBS_PCI_RTY_MASK << UBS_PCI_RTY_SHIFT))
1956             | ((UBS_DEF_RTY & 0xff) << UBS_PCI_RTY_SHIFT);
1957         misc = (misc & ~(UBS_PCI_TOUT_MASK << UBS_PCI_TOUT_SHIFT))
1958             | ((UBS_DEF_TOUT & 0xff) << UBS_PCI_TOUT_SHIFT);
1959         pci_conf_write(pc, pa->pa_tag, BS_RTY_TOUT, misc);
1960 #endif
1961
1962         /*
1963          * This will set the cache line size to 1, this will
1964          * force the BCM58xx chip just to do burst read/writes.
1965          * Cache line read/writes are to slow
1966          */
1967         pci_write_config(dev, PCIR_CACHELNSZ, UBS_DEF_CACHELINE, 1);
1968 }
1969
1970 /*
1971  * Clean up after a chip crash.
1972  * It is assumed that the caller in splimp()
1973  */
1974 static void
1975 ubsec_cleanchip(struct ubsec_softc *sc)
1976 {
1977         struct ubsec_q *q;
1978
1979         while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
1980                 q = SIMPLEQ_FIRST(&sc->sc_qchip);
1981                 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q_next);
1982                 ubsec_free_q(sc, q);
1983         }
1984         sc->sc_nqchip = 0;
1985 }
1986
1987 /*
1988  * free a ubsec_q
1989  * It is assumed that the caller is within splimp().
1990  */
1991 static int
1992 ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q)
1993 {
1994         struct ubsec_q *q2;
1995         struct cryptop *crp;
1996         int npkts;
1997         int i;
1998
1999         npkts = q->q_nstacked_mcrs;
2000
2001         for (i = 0; i < npkts; i++) {
2002                 if(q->q_stacked_mcr[i]) {
2003                         q2 = q->q_stacked_mcr[i];
2004
2005                         if ((q2->q_dst_m != NULL) && (q2->q_src_m != q2->q_dst_m))
2006                                 m_freem(q2->q_dst_m);
2007
2008                         crp = (struct cryptop *)q2->q_crp;
2009
2010                         SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next);
2011
2012                         crp->crp_etype = EFAULT;
2013                         crypto_done(crp);
2014                 } else {
2015                         break;
2016                 }
2017         }
2018
2019         /*
2020          * Free header MCR
2021          */
2022         if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
2023                 m_freem(q->q_dst_m);
2024
2025         crp = (struct cryptop *)q->q_crp;
2026
2027         SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
2028
2029         crp->crp_etype = EFAULT;
2030         crypto_done(crp);
2031         return(0);
2032 }
2033
2034 /*
2035  * Routine to reset the chip and clean up.
2036  * It is assumed that the caller is in splimp()
2037  */
2038 static void
2039 ubsec_totalreset(struct ubsec_softc *sc)
2040 {
2041         ubsec_reset_board(sc);
2042         ubsec_init_board(sc);
2043         ubsec_cleanchip(sc);
2044 }
2045
2046 static int
2047 ubsec_dmamap_aligned(struct ubsec_operand *op)
2048 {
2049         int i;
2050
2051         for (i = 0; i < op->nsegs; i++) {
2052                 if (op->segs[i].ds_addr & 3)
2053                         return (0);
2054                 if ((i != (op->nsegs - 1)) &&
2055                     (op->segs[i].ds_len & 3))
2056                         return (0);
2057         }
2058         return (1);
2059 }
2060
2061 static void
2062 ubsec_kfree(struct ubsec_softc *sc, struct ubsec_q2 *q)
2063 {
2064         switch (q->q_type) {
2065         case UBS_CTXOP_MODEXP: {
2066                 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q;
2067
2068                 ubsec_dma_free(sc, &me->me_q.q_mcr);
2069                 ubsec_dma_free(sc, &me->me_q.q_ctx);
2070                 ubsec_dma_free(sc, &me->me_M);
2071                 ubsec_dma_free(sc, &me->me_E);
2072                 ubsec_dma_free(sc, &me->me_C);
2073                 ubsec_dma_free(sc, &me->me_epb);
2074                 free(me, M_DEVBUF);
2075                 break;
2076         }
2077         case UBS_CTXOP_RSAPRIV: {
2078                 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q;
2079
2080                 ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
2081                 ubsec_dma_free(sc, &rp->rpr_q.q_ctx);
2082                 ubsec_dma_free(sc, &rp->rpr_msgin);
2083                 ubsec_dma_free(sc, &rp->rpr_msgout);
2084                 free(rp, M_DEVBUF);
2085                 break;
2086         }
2087         default:
2088                 device_printf(sc->sc_dev, "invalid kfree 0x%x\n", q->q_type);
2089                 break;
2090         }
2091 }
2092
2093 static int
2094 ubsec_kprocess(device_t dev, struct cryptkop *krp, int hint)
2095 {
2096         struct ubsec_softc *sc = device_get_softc(dev);
2097         int r;
2098
2099         if (krp == NULL || krp->krp_callback == NULL)
2100                 return (EINVAL);
2101
2102         while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) {
2103                 struct ubsec_q2 *q;
2104
2105                 q = SIMPLEQ_FIRST(&sc->sc_q2free);
2106                 SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, q_next);
2107                 ubsec_kfree(sc, q);
2108         }
2109
2110         switch (krp->krp_op) {
2111         case CRK_MOD_EXP:
2112                 if (sc->sc_flags & UBS_FLAGS_HWNORM)
2113                         r = ubsec_kprocess_modexp_hw(sc, krp, hint);
2114                 else
2115                         r = ubsec_kprocess_modexp_sw(sc, krp, hint);
2116                 break;
2117         case CRK_MOD_EXP_CRT:
2118                 return (ubsec_kprocess_rsapriv(sc, krp, hint));
2119         default:
2120                 device_printf(sc->sc_dev, "kprocess: invalid op 0x%x\n",
2121                     krp->krp_op);
2122                 krp->krp_status = EOPNOTSUPP;
2123                 crypto_kdone(krp);
2124                 return (0);
2125         }
2126         return (0);                     /* silence compiler */
2127 }
2128
2129 /*
2130  * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization)
2131  */
2132 static int
2133 ubsec_kprocess_modexp_sw(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
2134 {
2135         struct ubsec_q2_modexp *me;
2136         struct ubsec_mcr *mcr;
2137         struct ubsec_ctx_modexp *ctx;
2138         struct ubsec_pktbuf *epb;
2139         int err = 0;
2140         u_int nbits, normbits, mbits, shiftbits, ebits;
2141
2142         me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT);
2143         if (me == NULL) {
2144                 err = ENOMEM;
2145                 goto errout;
2146         }
2147         bzero(me, sizeof *me);
2148         me->me_krp = krp;
2149         me->me_q.q_type = UBS_CTXOP_MODEXP;
2150
2151         nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
2152         if (nbits <= 512)
2153                 normbits = 512;
2154         else if (nbits <= 768)
2155                 normbits = 768;
2156         else if (nbits <= 1024)
2157                 normbits = 1024;
2158         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536)
2159                 normbits = 1536;
2160         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048)
2161                 normbits = 2048;
2162         else {
2163                 err = E2BIG;
2164                 goto errout;
2165         }
2166
2167         shiftbits = normbits - nbits;
2168
2169         me->me_modbits = nbits;
2170         me->me_shiftbits = shiftbits;
2171         me->me_normbits = normbits;
2172
2173         /* Sanity check: result bits must be >= true modulus bits. */
2174         if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
2175                 err = ERANGE;
2176                 goto errout;
2177         }
2178
2179         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2180             &me->me_q.q_mcr, 0)) {
2181                 err = ENOMEM;
2182                 goto errout;
2183         }
2184         mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
2185
2186         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
2187             &me->me_q.q_ctx, 0)) {
2188                 err = ENOMEM;
2189                 goto errout;
2190         }
2191
2192         mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
2193         if (mbits > nbits) {
2194                 err = E2BIG;
2195                 goto errout;
2196         }
2197         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) {
2198                 err = ENOMEM;
2199                 goto errout;
2200         }
2201         ubsec_kshift_r(shiftbits,
2202             krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits,
2203             me->me_M.dma_vaddr, normbits);
2204
2205         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) {
2206                 err = ENOMEM;
2207                 goto errout;
2208         }
2209         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
2210
2211         ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
2212         if (ebits > nbits) {
2213                 err = E2BIG;
2214                 goto errout;
2215         }
2216         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) {
2217                 err = ENOMEM;
2218                 goto errout;
2219         }
2220         ubsec_kshift_r(shiftbits,
2221             krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits,
2222             me->me_E.dma_vaddr, normbits);
2223
2224         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
2225             &me->me_epb, 0)) {
2226                 err = ENOMEM;
2227                 goto errout;
2228         }
2229         epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
2230         epb->pb_addr = htole32(me->me_E.dma_paddr);
2231         epb->pb_next = 0;
2232         epb->pb_len = htole32(normbits / 8);
2233
2234 #ifdef UBSEC_DEBUG
2235         if (ubsec_debug) {
2236                 printf("Epb ");
2237                 ubsec_dump_pb(epb);
2238         }
2239 #endif
2240
2241         mcr->mcr_pkts = htole16(1);
2242         mcr->mcr_flags = 0;
2243         mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
2244         mcr->mcr_reserved = 0;
2245         mcr->mcr_pktlen = 0;
2246
2247         mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
2248         mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8);
2249         mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
2250
2251         mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
2252         mcr->mcr_opktbuf.pb_next = 0;
2253         mcr->mcr_opktbuf.pb_len = htole32(normbits / 8);
2254
2255 #ifdef DIAGNOSTIC
2256         /* Misaligned output buffer will hang the chip. */
2257         if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0)
2258                 panic("%s: modexp invalid addr 0x%x\n",
2259                     device_get_nameunit(sc->sc_dev),
2260                     letoh32(mcr->mcr_opktbuf.pb_addr));
2261         if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0)
2262                 panic("%s: modexp invalid len 0x%x\n",
2263                     device_get_nameunit(sc->sc_dev),
2264                     letoh32(mcr->mcr_opktbuf.pb_len));
2265 #endif
2266
2267         ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
2268         bzero(ctx, sizeof(*ctx));
2269         ubsec_kshift_r(shiftbits,
2270             krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits,
2271             ctx->me_N, normbits);
2272         ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t)));
2273         ctx->me_op = htole16(UBS_CTXOP_MODEXP);
2274         ctx->me_E_len = htole16(nbits);
2275         ctx->me_N_len = htole16(nbits);
2276
2277 #ifdef UBSEC_DEBUG
2278         if (ubsec_debug) {
2279                 ubsec_dump_mcr(mcr);
2280                 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
2281         }
2282 #endif
2283
2284         /*
2285          * ubsec_feed2 will sync mcr and ctx, we just need to sync
2286          * everything else.
2287          */
2288         ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE);
2289         ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE);
2290         ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD);
2291         ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE);
2292
2293         /* Enqueue and we're done... */
2294         mtx_lock(&sc->sc_mcr2lock);
2295         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
2296         ubsec_feed2(sc);
2297         ubsecstats.hst_modexp++;
2298         mtx_unlock(&sc->sc_mcr2lock);
2299
2300         return (0);
2301
2302 errout:
2303         if (me != NULL) {
2304                 if (me->me_q.q_mcr.dma_tag != NULL)
2305                         ubsec_dma_free(sc, &me->me_q.q_mcr);
2306                 if (me->me_q.q_ctx.dma_tag != NULL) {
2307                         bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
2308                         ubsec_dma_free(sc, &me->me_q.q_ctx);
2309                 }
2310                 if (me->me_M.dma_tag != NULL) {
2311                         bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
2312                         ubsec_dma_free(sc, &me->me_M);
2313                 }
2314                 if (me->me_E.dma_tag != NULL) {
2315                         bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
2316                         ubsec_dma_free(sc, &me->me_E);
2317                 }
2318                 if (me->me_C.dma_tag != NULL) {
2319                         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
2320                         ubsec_dma_free(sc, &me->me_C);
2321                 }
2322                 if (me->me_epb.dma_tag != NULL)
2323                         ubsec_dma_free(sc, &me->me_epb);
2324                 free(me, M_DEVBUF);
2325         }
2326         krp->krp_status = err;
2327         crypto_kdone(krp);
2328         return (0);
2329 }
2330
2331 /*
2332  * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization)
2333  */
2334 static int
2335 ubsec_kprocess_modexp_hw(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
2336 {
2337         struct ubsec_q2_modexp *me;
2338         struct ubsec_mcr *mcr;
2339         struct ubsec_ctx_modexp *ctx;
2340         struct ubsec_pktbuf *epb;
2341         int err = 0;
2342         u_int nbits, normbits, mbits, shiftbits, ebits;
2343
2344         me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT);
2345         if (me == NULL) {
2346                 err = ENOMEM;
2347                 goto errout;
2348         }
2349         bzero(me, sizeof *me);
2350         me->me_krp = krp;
2351         me->me_q.q_type = UBS_CTXOP_MODEXP;
2352
2353         nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
2354         if (nbits <= 512)
2355                 normbits = 512;
2356         else if (nbits <= 768)
2357                 normbits = 768;
2358         else if (nbits <= 1024)
2359                 normbits = 1024;
2360         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536)
2361                 normbits = 1536;
2362         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048)
2363                 normbits = 2048;
2364         else {
2365                 err = E2BIG;
2366                 goto errout;
2367         }
2368
2369         shiftbits = normbits - nbits;
2370
2371         /* XXX ??? */
2372         me->me_modbits = nbits;
2373         me->me_shiftbits = shiftbits;
2374         me->me_normbits = normbits;
2375
2376         /* Sanity check: result bits must be >= true modulus bits. */
2377         if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
2378                 err = ERANGE;
2379                 goto errout;
2380         }
2381
2382         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2383             &me->me_q.q_mcr, 0)) {
2384                 err = ENOMEM;
2385                 goto errout;
2386         }
2387         mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
2388
2389         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
2390             &me->me_q.q_ctx, 0)) {
2391                 err = ENOMEM;
2392                 goto errout;
2393         }
2394
2395         mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
2396         if (mbits > nbits) {
2397                 err = E2BIG;
2398                 goto errout;
2399         }
2400         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) {
2401                 err = ENOMEM;
2402                 goto errout;
2403         }
2404         bzero(me->me_M.dma_vaddr, normbits / 8);
2405         bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p,
2406             me->me_M.dma_vaddr, (mbits + 7) / 8);
2407
2408         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) {
2409                 err = ENOMEM;
2410                 goto errout;
2411         }
2412         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
2413
2414         ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
2415         if (ebits > nbits) {
2416                 err = E2BIG;
2417                 goto errout;
2418         }
2419         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) {
2420                 err = ENOMEM;
2421                 goto errout;
2422         }
2423         bzero(me->me_E.dma_vaddr, normbits / 8);
2424         bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p,
2425             me->me_E.dma_vaddr, (ebits + 7) / 8);
2426
2427         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
2428             &me->me_epb, 0)) {
2429                 err = ENOMEM;
2430                 goto errout;
2431         }
2432         epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
2433         epb->pb_addr = htole32(me->me_E.dma_paddr);
2434         epb->pb_next = 0;
2435         epb->pb_len = htole32((ebits + 7) / 8);
2436
2437 #ifdef UBSEC_DEBUG
2438         if (ubsec_debug) {
2439                 printf("Epb ");
2440                 ubsec_dump_pb(epb);
2441         }
2442 #endif
2443
2444         mcr->mcr_pkts = htole16(1);
2445         mcr->mcr_flags = 0;
2446         mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
2447         mcr->mcr_reserved = 0;
2448         mcr->mcr_pktlen = 0;
2449
2450         mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
2451         mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8);
2452         mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
2453
2454         mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
2455         mcr->mcr_opktbuf.pb_next = 0;
2456         mcr->mcr_opktbuf.pb_len = htole32(normbits / 8);
2457
2458 #ifdef DIAGNOSTIC
2459         /* Misaligned output buffer will hang the chip. */
2460         if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0)
2461                 panic("%s: modexp invalid addr 0x%x\n",
2462                     device_get_nameunit(sc->sc_dev),
2463                     letoh32(mcr->mcr_opktbuf.pb_addr));
2464         if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0)
2465                 panic("%s: modexp invalid len 0x%x\n",
2466                     device_get_nameunit(sc->sc_dev),
2467                     letoh32(mcr->mcr_opktbuf.pb_len));
2468 #endif
2469
2470         ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
2471         bzero(ctx, sizeof(*ctx));
2472         bcopy(krp->krp_param[UBS_MODEXP_PAR_N].crp_p, ctx->me_N,
2473             (nbits + 7) / 8);
2474         ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t)));
2475         ctx->me_op = htole16(UBS_CTXOP_MODEXP);
2476         ctx->me_E_len = htole16(ebits);
2477         ctx->me_N_len = htole16(nbits);
2478
2479 #ifdef UBSEC_DEBUG
2480         if (ubsec_debug) {
2481                 ubsec_dump_mcr(mcr);
2482                 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
2483         }
2484 #endif
2485
2486         /*
2487          * ubsec_feed2 will sync mcr and ctx, we just need to sync
2488          * everything else.
2489          */
2490         ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE);
2491         ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE);
2492         ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD);
2493         ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE);
2494
2495         /* Enqueue and we're done... */
2496         mtx_lock(&sc->sc_mcr2lock);
2497         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
2498         ubsec_feed2(sc);
2499         mtx_unlock(&sc->sc_mcr2lock);
2500
2501         return (0);
2502
2503 errout:
2504         if (me != NULL) {
2505                 if (me->me_q.q_mcr.dma_tag != NULL)
2506                         ubsec_dma_free(sc, &me->me_q.q_mcr);
2507                 if (me->me_q.q_ctx.dma_tag != NULL) {
2508                         bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
2509                         ubsec_dma_free(sc, &me->me_q.q_ctx);
2510                 }
2511                 if (me->me_M.dma_tag != NULL) {
2512                         bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
2513                         ubsec_dma_free(sc, &me->me_M);
2514                 }
2515                 if (me->me_E.dma_tag != NULL) {
2516                         bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
2517                         ubsec_dma_free(sc, &me->me_E);
2518                 }
2519                 if (me->me_C.dma_tag != NULL) {
2520                         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
2521                         ubsec_dma_free(sc, &me->me_C);
2522                 }
2523                 if (me->me_epb.dma_tag != NULL)
2524                         ubsec_dma_free(sc, &me->me_epb);
2525                 free(me, M_DEVBUF);
2526         }
2527         krp->krp_status = err;
2528         crypto_kdone(krp);
2529         return (0);
2530 }
2531
2532 static int
2533 ubsec_kprocess_rsapriv(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
2534 {
2535         struct ubsec_q2_rsapriv *rp = NULL;
2536         struct ubsec_mcr *mcr;
2537         struct ubsec_ctx_rsapriv *ctx;
2538         int err = 0;
2539         u_int padlen, msglen;
2540
2541         msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]);
2542         padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]);
2543         if (msglen > padlen)
2544                 padlen = msglen;
2545
2546         if (padlen <= 256)
2547                 padlen = 256;
2548         else if (padlen <= 384)
2549                 padlen = 384;
2550         else if (padlen <= 512)
2551                 padlen = 512;
2552         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 768)
2553                 padlen = 768;
2554         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 1024)
2555                 padlen = 1024;
2556         else {
2557                 err = E2BIG;
2558                 goto errout;
2559         }
2560
2561         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) {
2562                 err = E2BIG;
2563                 goto errout;
2564         }
2565
2566         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) {
2567                 err = E2BIG;
2568                 goto errout;
2569         }
2570
2571         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) {
2572                 err = E2BIG;
2573                 goto errout;
2574         }
2575
2576         rp = (struct ubsec_q2_rsapriv *)malloc(sizeof *rp, M_DEVBUF, M_NOWAIT);
2577         if (rp == NULL)
2578                 return (ENOMEM);
2579         bzero(rp, sizeof *rp);
2580         rp->rpr_krp = krp;
2581         rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV;
2582
2583         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2584             &rp->rpr_q.q_mcr, 0)) {
2585                 err = ENOMEM;
2586                 goto errout;
2587         }
2588         mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr;
2589
2590         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv),
2591             &rp->rpr_q.q_ctx, 0)) {
2592                 err = ENOMEM;
2593                 goto errout;
2594         }
2595         ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr;
2596         bzero(ctx, sizeof *ctx);
2597
2598         /* Copy in p */
2599         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p,
2600             &ctx->rpr_buf[0 * (padlen / 8)],
2601             (krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + 7) / 8);
2602
2603         /* Copy in q */
2604         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p,
2605             &ctx->rpr_buf[1 * (padlen / 8)],
2606             (krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + 7) / 8);
2607
2608         /* Copy in dp */
2609         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p,
2610             &ctx->rpr_buf[2 * (padlen / 8)],
2611             (krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + 7) / 8);
2612
2613         /* Copy in dq */
2614         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p,
2615             &ctx->rpr_buf[3 * (padlen / 8)],
2616             (krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + 7) / 8);
2617
2618         /* Copy in pinv */
2619         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p,
2620             &ctx->rpr_buf[4 * (padlen / 8)],
2621             (krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + 7) / 8);
2622
2623         msglen = padlen * 2;
2624
2625         /* Copy in input message (aligned buffer/length). */
2626         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) {
2627                 /* Is this likely? */
2628                 err = E2BIG;
2629                 goto errout;
2630         }
2631         if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgin, 0)) {
2632                 err = ENOMEM;
2633                 goto errout;
2634         }
2635         bzero(rp->rpr_msgin.dma_vaddr, (msglen + 7) / 8);
2636         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p,
2637             rp->rpr_msgin.dma_vaddr,
2638             (krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + 7) / 8);
2639
2640         /* Prepare space for output message (aligned buffer/length). */
2641         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) {
2642                 /* Is this likely? */
2643                 err = E2BIG;
2644                 goto errout;
2645         }
2646         if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgout, 0)) {
2647                 err = ENOMEM;
2648                 goto errout;
2649         }
2650         bzero(rp->rpr_msgout.dma_vaddr, (msglen + 7) / 8);
2651
2652         mcr->mcr_pkts = htole16(1);
2653         mcr->mcr_flags = 0;
2654         mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr);
2655         mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr);
2656         mcr->mcr_ipktbuf.pb_next = 0;
2657         mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size);
2658         mcr->mcr_reserved = 0;
2659         mcr->mcr_pktlen = htole16(msglen);
2660         mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr);
2661         mcr->mcr_opktbuf.pb_next = 0;
2662         mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size);
2663
2664 #ifdef DIAGNOSTIC
2665         if (rp->rpr_msgin.dma_paddr & 3 || rp->rpr_msgin.dma_size & 3) {
2666                 panic("%s: rsapriv: invalid msgin %x(0x%jx)",
2667                     device_get_nameunit(sc->sc_dev),
2668                     rp->rpr_msgin.dma_paddr, (uintmax_t)rp->rpr_msgin.dma_size);
2669         }
2670         if (rp->rpr_msgout.dma_paddr & 3 || rp->rpr_msgout.dma_size & 3) {
2671                 panic("%s: rsapriv: invalid msgout %x(0x%jx)",
2672                     device_get_nameunit(sc->sc_dev),
2673                     rp->rpr_msgout.dma_paddr, (uintmax_t)rp->rpr_msgout.dma_size);
2674         }
2675 #endif
2676
2677         ctx->rpr_len = (sizeof(u_int16_t) * 4) + (5 * (padlen / 8));
2678         ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV);
2679         ctx->rpr_q_len = htole16(padlen);
2680         ctx->rpr_p_len = htole16(padlen);
2681
2682         /*
2683          * ubsec_feed2 will sync mcr and ctx, we just need to sync
2684          * everything else.
2685          */
2686         ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_PREWRITE);
2687         ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_PREREAD);
2688
2689         /* Enqueue and we're done... */
2690         mtx_lock(&sc->sc_mcr2lock);
2691         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next);
2692         ubsec_feed2(sc);
2693         ubsecstats.hst_modexpcrt++;
2694         mtx_unlock(&sc->sc_mcr2lock);
2695         return (0);
2696
2697 errout:
2698         if (rp != NULL) {
2699                 if (rp->rpr_q.q_mcr.dma_tag != NULL)
2700                         ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
2701                 if (rp->rpr_msgin.dma_tag != NULL) {
2702                         bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size);
2703                         ubsec_dma_free(sc, &rp->rpr_msgin);
2704                 }
2705                 if (rp->rpr_msgout.dma_tag != NULL) {
2706                         bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size);
2707                         ubsec_dma_free(sc, &rp->rpr_msgout);
2708                 }
2709                 free(rp, M_DEVBUF);
2710         }
2711         krp->krp_status = err;
2712         crypto_kdone(krp);
2713         return (0);
2714 }
2715
2716 #ifdef UBSEC_DEBUG
2717 static void
2718 ubsec_dump_pb(volatile struct ubsec_pktbuf *pb)
2719 {
2720         printf("addr 0x%x (0x%x) next 0x%x\n",
2721             pb->pb_addr, pb->pb_len, pb->pb_next);
2722 }
2723
2724 static void
2725 ubsec_dump_ctx2(struct ubsec_ctx_keyop *c)
2726 {
2727         printf("CTX (0x%x):\n", c->ctx_len);
2728         switch (letoh16(c->ctx_op)) {
2729         case UBS_CTXOP_RNGBYPASS:
2730         case UBS_CTXOP_RNGSHA1:
2731                 break;
2732         case UBS_CTXOP_MODEXP:
2733         {
2734                 struct ubsec_ctx_modexp *cx = (void *)c;
2735                 int i, len;
2736
2737                 printf(" Elen %u, Nlen %u\n",
2738                     letoh16(cx->me_E_len), letoh16(cx->me_N_len));
2739                 len = (cx->me_N_len + 7)/8;
2740                 for (i = 0; i < len; i++)
2741                         printf("%s%02x", (i == 0) ? " N: " : ":", cx->me_N[i]);
2742                 printf("\n");
2743                 break;
2744         }
2745         default:
2746                 printf("unknown context: %x\n", c->ctx_op);
2747         }
2748         printf("END CTX\n");
2749 }
2750
2751 static void
2752 ubsec_dump_mcr(struct ubsec_mcr *mcr)
2753 {
2754         volatile struct ubsec_mcr_add *ma;
2755         int i;
2756
2757         printf("MCR:\n");
2758         printf(" pkts: %u, flags 0x%x\n",
2759             letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags));
2760         ma = (volatile struct ubsec_mcr_add *)&mcr->mcr_cmdctxp;
2761         for (i = 0; i < letoh16(mcr->mcr_pkts); i++) {
2762                 printf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i,
2763                     letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen),
2764                     letoh16(ma->mcr_reserved));
2765                 printf(" %d: ipkt ", i);
2766                 ubsec_dump_pb(&ma->mcr_ipktbuf);
2767                 printf(" %d: opkt ", i);
2768                 ubsec_dump_pb(&ma->mcr_opktbuf);
2769                 ma++;
2770         }
2771         printf("END MCR\n");
2772 }
2773 #endif /* UBSEC_DEBUG */
2774
2775 /*
2776  * Return the number of significant bits of a big number.
2777  */
2778 static int
2779 ubsec_ksigbits(struct crparam *cr)
2780 {
2781         u_int plen = (cr->crp_nbits + 7) / 8;
2782         int i, sig = plen * 8;
2783         u_int8_t c, *p = cr->crp_p;
2784
2785         for (i = plen - 1; i >= 0; i--) {
2786                 c = p[i];
2787                 if (c != 0) {
2788                         while ((c & 0x80) == 0) {
2789                                 sig--;
2790                                 c <<= 1;
2791                         }
2792                         break;
2793                 }
2794                 sig -= 8;
2795         }
2796         return (sig);
2797 }
2798
2799 static void
2800 ubsec_kshift_r(
2801         u_int shiftbits,
2802         u_int8_t *src, u_int srcbits,
2803         u_int8_t *dst, u_int dstbits)
2804 {
2805         u_int slen, dlen;
2806         int i, si, di, n;
2807
2808         slen = (srcbits + 7) / 8;
2809         dlen = (dstbits + 7) / 8;
2810
2811         for (i = 0; i < slen; i++)
2812                 dst[i] = src[i];
2813         for (i = 0; i < dlen - slen; i++)
2814                 dst[slen + i] = 0;
2815
2816         n = shiftbits / 8;
2817         if (n != 0) {
2818                 si = dlen - n - 1;
2819                 di = dlen - 1;
2820                 while (si >= 0)
2821                         dst[di--] = dst[si--];
2822                 while (di >= 0)
2823                         dst[di--] = 0;
2824         }
2825
2826         n = shiftbits % 8;
2827         if (n != 0) {
2828                 for (i = dlen - 1; i > 0; i--)
2829                         dst[i] = (dst[i] << n) |
2830                             (dst[i - 1] >> (8 - n));
2831                 dst[0] = dst[0] << n;
2832         }
2833 }
2834
2835 static void
2836 ubsec_kshift_l(
2837         u_int shiftbits,
2838         u_int8_t *src, u_int srcbits,
2839         u_int8_t *dst, u_int dstbits)
2840 {
2841         int slen, dlen, i, n;
2842
2843         slen = (srcbits + 7) / 8;
2844         dlen = (dstbits + 7) / 8;
2845
2846         n = shiftbits / 8;
2847         for (i = 0; i < slen; i++)
2848                 dst[i] = src[i + n];
2849         for (i = 0; i < dlen - slen; i++)
2850                 dst[slen + i] = 0;
2851
2852         n = shiftbits % 8;
2853         if (n != 0) {
2854                 for (i = 0; i < (dlen - 1); i++)
2855                         dst[i] = (dst[i] >> n) | (dst[i + 1] << (8 - n));
2856                 dst[dlen - 1] = dst[dlen - 1] >> n;
2857         }
2858 }