sys/dev/ubsec/ubsec.c

   1 /*      $OpenBSD: ubsec.c,v 1.115 2002/09/24 18:33:26 jason Exp $       */
   2
   3 /*-
   4  * Copyright (c) 2000 Jason L. Wright (jason@thought.net)
   5  * Copyright (c) 2000 Theo de Raadt (deraadt@openbsd.org)
   6  * Copyright (c) 2001 Patrik Lindergren (patrik@ipunplugged.com)
   7  *
   8  * All rights reserved.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  * 3. All advertising materials mentioning features or use of this software
  19  *    must display the following acknowledgement:
  20  *      This product includes software developed by Jason L. Wright
  21  * 4. The name of the author may not be used to endorse or promote products
  22  *    derived from this software without specific prior written permission.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  25  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  26  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  27  * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
  28  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  29  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  30  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  31  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  32  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  33  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  34  * POSSIBILITY OF SUCH DAMAGE.
  35  *
  36  * Effort sponsored in part by the Defense Advanced Research Projects
  37  * Agency (DARPA) and Air Force Research Laboratory, Air Force
  38  * Materiel Command, USAF, under agreement number F30602-01-2-0537.
  39  */
  40
  41 #include <sys/cdefs.h>
  42 __FBSDID("$FreeBSD$");
  43
  44 /*
  45  * uBsec 5[56]01, 58xx hardware crypto accelerator
  46  */
  47
  48 #include "opt_ubsec.h"
  49
  50 #include <sys/param.h>
  51 #include <sys/systm.h>
  52 #include <sys/proc.h>
  53 #include <sys/errno.h>
  54 #include <sys/malloc.h>
  55 #include <sys/kernel.h>
  56 #include <sys/module.h>
  57 #include <sys/mbuf.h>
  58 #include <sys/lock.h>
  59 #include <sys/mutex.h>
  60 #include <sys/sysctl.h>
  61 #include <sys/endian.h>
  62
  63 #include <vm/vm.h>
  64 #include <vm/pmap.h>
  65
  66 #include <machine/bus.h>
  67 #include <machine/resource.h>
  68 #include <sys/bus.h>
  69 #include <sys/rman.h>
  70
  71 #include <crypto/sha1.h>
  72 #include <opencrypto/cryptodev.h>
  73 #include <opencrypto/cryptosoft.h>
  74 #include <sys/md5.h>
  75 #include <sys/random.h>
  76
  77 #include <dev/pci/pcivar.h>
  78 #include <dev/pci/pcireg.h>
  79
  80 /* grr, #defines for gratuitous incompatibility in queue.h */
  81 #define SIMPLEQ_HEAD            STAILQ_HEAD
  82 #define SIMPLEQ_ENTRY           STAILQ_ENTRY
  83 #define SIMPLEQ_INIT            STAILQ_INIT
  84 #define SIMPLEQ_INSERT_TAIL     STAILQ_INSERT_TAIL
  85 #define SIMPLEQ_EMPTY           STAILQ_EMPTY
  86 #define SIMPLEQ_FIRST           STAILQ_FIRST
  87 #define SIMPLEQ_REMOVE_HEAD     STAILQ_REMOVE_HEAD_UNTIL
  88 #define SIMPLEQ_FOREACH         STAILQ_FOREACH
  89 /* ditto for endian.h */
  90 #define letoh16(x)              le16toh(x)
  91 #define letoh32(x)              le32toh(x)
  92
  93 #ifdef UBSEC_RNDTEST
  94 #include <dev/rndtest/rndtest.h>
  95 #endif
  96 #include <dev/ubsec/ubsecreg.h>
  97 #include <dev/ubsec/ubsecvar.h>
  98
  99 /*
 100  * Prototypes and count for the pci_device structure
 101  */
 102 static  int ubsec_probe(device_t);
 103 static  int ubsec_attach(device_t);
 104 static  int ubsec_detach(device_t);
 105 static  int ubsec_suspend(device_t);
 106 static  int ubsec_resume(device_t);
 107 static  void ubsec_shutdown(device_t);
 108
 109 static device_method_t ubsec_methods[] = {
 110         /* Device interface */
 111         DEVMETHOD(device_probe,         ubsec_probe),
 112         DEVMETHOD(device_attach,        ubsec_attach),
 113         DEVMETHOD(device_detach,        ubsec_detach),
 114         DEVMETHOD(device_suspend,       ubsec_suspend),
 115         DEVMETHOD(device_resume,        ubsec_resume),
 116         DEVMETHOD(device_shutdown,      ubsec_shutdown),
 117
 118         /* bus interface */
 119         DEVMETHOD(bus_print_child,      bus_generic_print_child),
 120         DEVMETHOD(bus_driver_added,     bus_generic_driver_added),
 121
 122         { 0, 0 }
 123 };
 124 static driver_t ubsec_driver = {
 125         "ubsec",
 126         ubsec_methods,
 127         sizeof (struct ubsec_softc)
 128 };
 129 static devclass_t ubsec_devclass;
 130
 131 DRIVER_MODULE(ubsec, pci, ubsec_driver, ubsec_devclass, 0, 0);
 132 MODULE_DEPEND(ubsec, crypto, 1, 1, 1);
 133 #ifdef UBSEC_RNDTEST
 134 MODULE_DEPEND(ubsec, rndtest, 1, 1, 1);
 135 #endif
 136
 137 static  void ubsec_intr(void *);
 138 static  int ubsec_newsession(void *, u_int32_t *, struct cryptoini *);
 139 static  int ubsec_freesession(void *, u_int64_t);
 140 static  int ubsec_process(void *, struct cryptop *, int);
 141 static  void ubsec_callback(struct ubsec_softc *, struct ubsec_q *);
 142 static  void ubsec_feed(struct ubsec_softc *);
 143 static  void ubsec_mcopy(struct mbuf *, struct mbuf *, int, int);
 144 static  void ubsec_callback2(struct ubsec_softc *, struct ubsec_q2 *);
 145 static  int ubsec_feed2(struct ubsec_softc *);
 146 static  void ubsec_rng(void *);
 147 static  int ubsec_dma_malloc(struct ubsec_softc *, bus_size_t,
 148                              struct ubsec_dma_alloc *, int);
 149 #define ubsec_dma_sync(_dma, _flags) \
 150         bus_dmamap_sync((_dma)->dma_tag, (_dma)->dma_map, (_flags))
 151 static  void ubsec_dma_free(struct ubsec_softc *, struct ubsec_dma_alloc *);
 152 static  int ubsec_dmamap_aligned(struct ubsec_operand *op);
 153
 154 static  void ubsec_reset_board(struct ubsec_softc *sc);
 155 static  void ubsec_init_board(struct ubsec_softc *sc);
 156 static  void ubsec_init_pciregs(device_t dev);
 157 static  void ubsec_totalreset(struct ubsec_softc *sc);
 158
 159 static  int ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q);
 160
 161 static  int ubsec_kprocess(void*, struct cryptkop *, int);
 162 static  int ubsec_kprocess_modexp_hw(struct ubsec_softc *, struct cryptkop *, int);
 163 static  int ubsec_kprocess_modexp_sw(struct ubsec_softc *, struct cryptkop *, int);
 164 static  int ubsec_kprocess_rsapriv(struct ubsec_softc *, struct cryptkop *, int);
 165 static  void ubsec_kfree(struct ubsec_softc *, struct ubsec_q2 *);
 166 static  int ubsec_ksigbits(struct crparam *);
 167 static  void ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
 168 static  void ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
 169
 170 SYSCTL_NODE(_hw, OID_AUTO, ubsec, CTLFLAG_RD, 0, "Broadcom driver parameters");
 171
 172 #ifdef UBSEC_DEBUG
 173 static  void ubsec_dump_pb(volatile struct ubsec_pktbuf *);
 174 static  void ubsec_dump_mcr(struct ubsec_mcr *);
 175 static  void ubsec_dump_ctx2(struct ubsec_ctx_keyop *);
 176
 177 static  int ubsec_debug = 0;
 178 SYSCTL_INT(_hw_ubsec, OID_AUTO, debug, CTLFLAG_RW, &ubsec_debug,
 179             0, "control debugging msgs");
 180 #endif
 181
 182 #define READ_REG(sc,r) \
 183         bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r))
 184
 185 #define WRITE_REG(sc,reg,val) \
 186         bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val)
 187
 188 #define SWAP32(x) (x) = htole32(ntohl((x)))
 189 #define HTOLE32(x) (x) = htole32(x)
 190
 191 struct ubsec_stats ubsecstats;
 192 SYSCTL_STRUCT(_hw_ubsec, OID_AUTO, stats, CTLFLAG_RD, &ubsecstats,
 193             ubsec_stats, "driver statistics");
 194
 195 static int
 196 ubsec_probe(device_t dev)
 197 {
 198         if (pci_get_vendor(dev) == PCI_VENDOR_SUN &&
 199             (pci_get_device(dev) == PCI_PRODUCT_SUN_5821 ||
 200              pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K))
 201                 return (BUS_PROBE_DEFAULT);
 202         if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL &&
 203             (pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5501 ||
 204              pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601))
 205                 return (BUS_PROBE_DEFAULT);
 206         if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 207             (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5801 ||
 208              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 ||
 209              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805 ||
 210              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820 ||
 211              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 ||
 212              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 ||
 213              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823
 214              ))
 215                 return (BUS_PROBE_DEFAULT);
 216         return (ENXIO);
 217 }
 218
 219 static const char*
 220 ubsec_partname(struct ubsec_softc *sc)
 221 {
 222         /* XXX sprintf numbers when not decoded */
 223         switch (pci_get_vendor(sc->sc_dev)) {
 224         case PCI_VENDOR_BROADCOM:
 225                 switch (pci_get_device(sc->sc_dev)) {
 226                 case PCI_PRODUCT_BROADCOM_5801: return "Broadcom 5801";
 227                 case PCI_PRODUCT_BROADCOM_5802: return "Broadcom 5802";
 228                 case PCI_PRODUCT_BROADCOM_5805: return "Broadcom 5805";
 229                 case PCI_PRODUCT_BROADCOM_5820: return "Broadcom 5820";
 230                 case PCI_PRODUCT_BROADCOM_5821: return "Broadcom 5821";
 231                 case PCI_PRODUCT_BROADCOM_5822: return "Broadcom 5822";
 232                 case PCI_PRODUCT_BROADCOM_5823: return "Broadcom 5823";
 233                 }
 234                 return "Broadcom unknown-part";
 235         case PCI_VENDOR_BLUESTEEL:
 236                 switch (pci_get_device(sc->sc_dev)) {
 237                 case PCI_PRODUCT_BLUESTEEL_5601: return "Bluesteel 5601";
 238                 }
 239                 return "Bluesteel unknown-part";
 240         case PCI_VENDOR_SUN:
 241                 switch (pci_get_device(sc->sc_dev)) {
 242                 case PCI_PRODUCT_SUN_5821: return "Sun Crypto 5821";
 243                 case PCI_PRODUCT_SUN_SCA1K: return "Sun Crypto 1K";
 244                 }
 245                 return "Sun unknown-part";
 246         }
 247         return "Unknown-vendor unknown-part";
 248 }
 249
 250 static void
 251 default_harvest(struct rndtest_state *rsp, void *buf, u_int count)
 252 {
 253         random_harvest(buf, count, count*NBBY, 0, RANDOM_PURE);
 254 }
 255
 256 static int
 257 ubsec_attach(device_t dev)
 258 {
 259         struct ubsec_softc *sc = device_get_softc(dev);
 260         struct ubsec_dma *dmap;
 261         u_int32_t cmd, i;
 262         int rid;
 263
 264         bzero(sc, sizeof (*sc));
 265         sc->sc_dev = dev;
 266
 267         SIMPLEQ_INIT(&sc->sc_queue);
 268         SIMPLEQ_INIT(&sc->sc_qchip);
 269         SIMPLEQ_INIT(&sc->sc_queue2);
 270         SIMPLEQ_INIT(&sc->sc_qchip2);
 271         SIMPLEQ_INIT(&sc->sc_q2free);
 272
 273         /* XXX handle power management */
 274
 275         sc->sc_statmask = BS_STAT_MCR1_DONE | BS_STAT_DMAERR;
 276
 277         if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL &&
 278             pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601)
 279                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG;
 280
 281         if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 282             (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 ||
 283              pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805))
 284                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG;
 285
 286         if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 287             pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820)
 288                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG |
 289                     UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY;
 290
 291         if ((pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 292              (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 ||
 293               pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 ||
 294               pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823)) ||
 295             (pci_get_vendor(dev) == PCI_VENDOR_SUN &&
 296              (pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K ||
 297               pci_get_device(dev) == PCI_PRODUCT_SUN_5821))) {
 298                 /* NB: the 5821/5822 defines some additional status bits */
 299                 sc->sc_statmask |= BS_STAT_MCR1_ALLEMPTY |
 300                     BS_STAT_MCR2_ALLEMPTY;
 301                 sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG |
 302                     UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY;
 303         }
 304
 305         cmd = pci_read_config(dev, PCIR_COMMAND, 4);
 306         cmd |= PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN;
 307         pci_write_config(dev, PCIR_COMMAND, cmd, 4);
 308         cmd = pci_read_config(dev, PCIR_COMMAND, 4);
 309
 310         if (!(cmd & PCIM_CMD_MEMEN)) {
 311                 device_printf(dev, "failed to enable memory mapping\n");
 312                 goto bad;
 313         }
 314
 315         if (!(cmd & PCIM_CMD_BUSMASTEREN)) {
 316                 device_printf(dev, "failed to enable bus mastering\n");
 317                 goto bad;
 318         }
 319
 320         /*
 321          * Setup memory-mapping of PCI registers.
 322          */
 323         rid = BS_BAR;
 324         sc->sc_sr = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 325                                            RF_ACTIVE);
 326         if (sc->sc_sr == NULL) {
 327                 device_printf(dev, "cannot map register space\n");
 328                 goto bad;
 329         }
 330         sc->sc_st = rman_get_bustag(sc->sc_sr);
 331         sc->sc_sh = rman_get_bushandle(sc->sc_sr);
 332
 333         /*
 334          * Arrange interrupt line.
 335          */
 336         rid = 0;
 337         sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 338                                             RF_SHAREABLE|RF_ACTIVE);
 339         if (sc->sc_irq == NULL) {
 340                 device_printf(dev, "could not map interrupt\n");
 341                 goto bad1;
 342         }
 343         /*
 344          * NB: Network code assumes we are blocked with splimp()
 345          *     so make sure the IRQ is mapped appropriately.
 346          */
 347         if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE,
 348                            ubsec_intr, sc, &sc->sc_ih)) {
 349                 device_printf(dev, "could not establish interrupt\n");
 350                 goto bad2;
 351         }
 352
 353         sc->sc_cid = crypto_get_driverid(0);
 354         if (sc->sc_cid < 0) {
 355                 device_printf(dev, "could not get crypto driver id\n");
 356                 goto bad3;
 357         }
 358
 359         /*
 360          * Setup DMA descriptor area.
 361          */
 362         if (bus_dma_tag_create(NULL,                    /* parent */
 363                                1, 0,                    /* alignment, bounds */
 364                                BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
 365                                BUS_SPACE_MAXADDR,       /* highaddr */
 366                                NULL, NULL,              /* filter, filterarg */
 367                                0x3ffff,                 /* maxsize */
 368                                UBS_MAX_SCATTER,         /* nsegments */
 369                                0xffff,                  /* maxsegsize */
 370                                BUS_DMA_ALLOCNOW,        /* flags */
 371                                NULL, NULL,              /* lockfunc, lockarg */
 372                                &sc->sc_dmat)) {
 373                 device_printf(dev, "cannot allocate DMA tag\n");
 374                 goto bad4;
 375         }
 376         SIMPLEQ_INIT(&sc->sc_freequeue);
 377         dmap = sc->sc_dmaa;
 378         for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) {
 379                 struct ubsec_q *q;
 380
 381                 q = (struct ubsec_q *)malloc(sizeof(struct ubsec_q),
 382                     M_DEVBUF, M_NOWAIT);
 383                 if (q == NULL) {
 384                         device_printf(dev, "cannot allocate queue buffers\n");
 385                         break;
 386                 }
 387
 388                 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk),
 389                     &dmap->d_alloc, 0)) {
 390                         device_printf(dev, "cannot allocate dma buffers\n");
 391                         free(q, M_DEVBUF);
 392                         break;
 393                 }
 394                 dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr;
 395
 396                 q->q_dma = dmap;
 397                 sc->sc_queuea[i] = q;
 398
 399                 SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
 400         }
 401         mtx_init(&sc->sc_mcr1lock, device_get_nameunit(dev),
 402                 "mcr1 operations", MTX_DEF);
 403         mtx_init(&sc->sc_freeqlock, device_get_nameunit(dev),
 404                 "mcr1 free q", MTX_DEF);
 405
 406         device_printf(sc->sc_dev, "%s\n", ubsec_partname(sc));
 407
 408         crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0,
 409             ubsec_newsession, ubsec_freesession, ubsec_process, sc);
 410         crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0,
 411              ubsec_newsession, ubsec_freesession, ubsec_process, sc);
 412         crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0,
 413              ubsec_newsession, ubsec_freesession, ubsec_process, sc);
 414         crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0,
 415              ubsec_newsession, ubsec_freesession, ubsec_process, sc);
 416
 417         /*
 418          * Reset Broadcom chip
 419          */
 420         ubsec_reset_board(sc);
 421
 422         /*
 423          * Init Broadcom specific PCI settings
 424          */
 425         ubsec_init_pciregs(dev);
 426
 427         /*
 428          * Init Broadcom chip
 429          */
 430         ubsec_init_board(sc);
 431
 432 #ifndef UBSEC_NO_RNG
 433         if (sc->sc_flags & UBS_FLAGS_RNG) {
 434                 sc->sc_statmask |= BS_STAT_MCR2_DONE;
 435 #ifdef UBSEC_RNDTEST
 436                 sc->sc_rndtest = rndtest_attach(dev);
 437                 if (sc->sc_rndtest)
 438                         sc->sc_harvest = rndtest_harvest;
 439                 else
 440                         sc->sc_harvest = default_harvest;
 441 #else
 442                 sc->sc_harvest = default_harvest;
 443 #endif
 444
 445                 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
 446                     &sc->sc_rng.rng_q.q_mcr, 0))
 447                         goto skip_rng;
 448
 449                 if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass),
 450                     &sc->sc_rng.rng_q.q_ctx, 0)) {
 451                         ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
 452                         goto skip_rng;
 453                 }
 454
 455                 if (ubsec_dma_malloc(sc, sizeof(u_int32_t) *
 456                     UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, 0)) {
 457                         ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx);
 458                         ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
 459                         goto skip_rng;
 460                 }
 461
 462                 if (hz >= 100)
 463                         sc->sc_rnghz = hz / 100;
 464                 else
 465                         sc->sc_rnghz = 1;
 466                 callout_init(&sc->sc_rngto, CALLOUT_MPSAFE);
 467                 callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
 468 skip_rng:
 469         ;
 470         }
 471 #endif /* UBSEC_NO_RNG */
 472         mtx_init(&sc->sc_mcr2lock, device_get_nameunit(dev),
 473                 "mcr2 operations", MTX_DEF);
 474
 475         if (sc->sc_flags & UBS_FLAGS_KEY) {
 476                 sc->sc_statmask |= BS_STAT_MCR2_DONE;
 477
 478                 crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0,
 479                         ubsec_kprocess, sc);
 480 #if 0
 481                 crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0,
 482                         ubsec_kprocess, sc);
 483 #endif
 484         }
 485         return (0);
 486 bad4:
 487         crypto_unregister_all(sc->sc_cid);
 488 bad3:
 489         bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
 490 bad2:
 491         bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
 492 bad1:
 493         bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
 494 bad:
 495         return (ENXIO);
 496 }
 497
 498 /*
 499  * Detach a device that successfully probed.
 500  */
 501 static int
 502 ubsec_detach(device_t dev)
 503 {
 504         struct ubsec_softc *sc = device_get_softc(dev);
 505
 506         /* XXX wait/abort active ops */
 507
 508         /* disable interrupts */
 509         WRITE_REG(sc, BS_CTRL, READ_REG(sc, BS_CTRL) &~
 510                 (BS_CTRL_MCR2INT | BS_CTRL_MCR1INT | BS_CTRL_DMAERR));
 511
 512         callout_stop(&sc->sc_rngto);
 513
 514         crypto_unregister_all(sc->sc_cid);
 515
 516 #ifdef UBSEC_RNDTEST
 517         if (sc->sc_rndtest)
 518                 rndtest_detach(sc->sc_rndtest);
 519 #endif
 520
 521         while (!SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
 522                 struct ubsec_q *q;
 523
 524                 q = SIMPLEQ_FIRST(&sc->sc_freequeue);
 525                 SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q, q_next);
 526                 ubsec_dma_free(sc, &q->q_dma->d_alloc);
 527                 free(q, M_DEVBUF);
 528         }
 529         mtx_destroy(&sc->sc_mcr1lock);
 530 #ifndef UBSEC_NO_RNG
 531         if (sc->sc_flags & UBS_FLAGS_RNG) {
 532                 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
 533                 ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx);
 534                 ubsec_dma_free(sc, &sc->sc_rng.rng_buf);
 535         }
 536 #endif /* UBSEC_NO_RNG */
 537         mtx_destroy(&sc->sc_mcr2lock);
 538
 539         bus_generic_detach(dev);
 540         bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
 541         bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
 542
 543         bus_dma_tag_destroy(sc->sc_dmat);
 544         bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
 545
 546         return (0);
 547 }
 548
 549 /*
 550  * Stop all chip i/o so that the kernel's probe routines don't
 551  * get confused by errant DMAs when rebooting.
 552  */
 553 static void
 554 ubsec_shutdown(device_t dev)
 555 {
 556 #ifdef notyet
 557         ubsec_stop(device_get_softc(dev));
 558 #endif
 559 }
 560
 561 /*
 562  * Device suspend routine.
 563  */
 564 static int
 565 ubsec_suspend(device_t dev)
 566 {
 567         struct ubsec_softc *sc = device_get_softc(dev);
 568
 569 #ifdef notyet
 570         /* XXX stop the device and save PCI settings */
 571 #endif
 572         sc->sc_suspended = 1;
 573
 574         return (0);
 575 }
 576
 577 static int
 578 ubsec_resume(device_t dev)
 579 {
 580         struct ubsec_softc *sc = device_get_softc(dev);
 581
 582 #ifdef notyet
 583         /* XXX retore PCI settings and start the device */
 584 #endif
 585         sc->sc_suspended = 0;
 586         return (0);
 587 }
 588
 589 /*
 590  * UBSEC Interrupt routine
 591  */
 592 static void
 593 ubsec_intr(void *arg)
 594 {
 595         struct ubsec_softc *sc = arg;
 596         volatile u_int32_t stat;
 597         struct ubsec_q *q;
 598         struct ubsec_dma *dmap;
 599         int npkts = 0, i;
 600
 601         stat = READ_REG(sc, BS_STAT);
 602         stat &= sc->sc_statmask;
 603         if (stat == 0)
 604                 return;
 605
 606         WRITE_REG(sc, BS_STAT, stat);           /* IACK */
 607
 608         /*
 609          * Check to see if we have any packets waiting for us
 610          */
 611         if ((stat & BS_STAT_MCR1_DONE)) {
 612                 mtx_lock(&sc->sc_mcr1lock);
 613                 while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
 614                         q = SIMPLEQ_FIRST(&sc->sc_qchip);
 615                         dmap = q->q_dma;
 616
 617                         if ((dmap->d_dma->d_mcr.mcr_flags & htole16(UBS_MCR_DONE)) == 0)
 618                                 break;
 619
 620                         SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q, q_next);
 621
 622                         npkts = q->q_nstacked_mcrs;
 623                         sc->sc_nqchip -= 1+npkts;
 624                         /*
 625                          * search for further sc_qchip ubsec_q's that share
 626                          * the same MCR, and complete them too, they must be
 627                          * at the top.
 628                          */
 629                         for (i = 0; i < npkts; i++) {
 630                                 if(q->q_stacked_mcr[i]) {
 631                                         ubsec_callback(sc, q->q_stacked_mcr[i]);
 632                                 } else {
 633                                         break;
 634                                 }
 635                         }
 636                         ubsec_callback(sc, q);
 637                 }
 638                 /*
 639                  * Don't send any more packet to chip if there has been
 640                  * a DMAERR.
 641                  */
 642                 if (!(stat & BS_STAT_DMAERR))
 643                         ubsec_feed(sc);
 644                 mtx_unlock(&sc->sc_mcr1lock);
 645         }
 646
 647         /*
 648          * Check to see if we have any key setups/rng's waiting for us
 649          */
 650         if ((sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) &&
 651             (stat & BS_STAT_MCR2_DONE)) {
 652                 struct ubsec_q2 *q2;
 653                 struct ubsec_mcr *mcr;
 654
 655                 mtx_lock(&sc->sc_mcr2lock);
 656                 while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) {
 657                         q2 = SIMPLEQ_FIRST(&sc->sc_qchip2);
 658
 659                         ubsec_dma_sync(&q2->q_mcr,
 660                             BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
 661
 662                         mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr;
 663                         if ((mcr->mcr_flags & htole16(UBS_MCR_DONE)) == 0) {
 664                                 ubsec_dma_sync(&q2->q_mcr,
 665                                     BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
 666                                 break;
 667                         }
 668                         SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, q2, q_next);
 669                         ubsec_callback2(sc, q2);
 670                         /*
 671                          * Don't send any more packet to chip if there has been
 672                          * a DMAERR.
 673                          */
 674                         if (!(stat & BS_STAT_DMAERR))
 675                                 ubsec_feed2(sc);
 676                 }
 677                 mtx_unlock(&sc->sc_mcr2lock);
 678         }
 679
 680         /*
 681          * Check to see if we got any DMA Error
 682          */
 683         if (stat & BS_STAT_DMAERR) {
 684 #ifdef UBSEC_DEBUG
 685                 if (ubsec_debug) {
 686                         volatile u_int32_t a = READ_REG(sc, BS_ERR);
 687
 688                         printf("dmaerr %s@%08x\n",
 689                             (a & BS_ERR_READ) ? "read" : "write",
 690                             a & BS_ERR_ADDR);
 691                 }
 692 #endif /* UBSEC_DEBUG */
 693                 ubsecstats.hst_dmaerr++;
 694                 mtx_lock(&sc->sc_mcr1lock);
 695                 ubsec_totalreset(sc);
 696                 ubsec_feed(sc);
 697                 mtx_unlock(&sc->sc_mcr1lock);
 698         }
 699
 700         if (sc->sc_needwakeup) {                /* XXX check high watermark */
 701                 int wakeup;
 702
 703                 mtx_lock(&sc->sc_freeqlock);
 704                 wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ);
 705 #ifdef UBSEC_DEBUG
 706                 if (ubsec_debug)
 707                         device_printf(sc->sc_dev, "wakeup crypto (%x)\n",
 708                                 sc->sc_needwakeup);
 709 #endif /* UBSEC_DEBUG */
 710                 sc->sc_needwakeup &= ~wakeup;
 711                 mtx_unlock(&sc->sc_freeqlock);
 712                 crypto_unblock(sc->sc_cid, wakeup);
 713         }
 714 }
 715
 716 /*
 717  * ubsec_feed() - aggregate and post requests to chip
 718  */
 719 static void
 720 ubsec_feed(struct ubsec_softc *sc)
 721 {
 722         struct ubsec_q *q, *q2;
 723         int npkts, i;
 724         void *v;
 725         u_int32_t stat;
 726
 727         /*
 728          * Decide how many ops to combine in a single MCR.  We cannot
 729          * aggregate more than UBS_MAX_AGGR because this is the number
 730          * of slots defined in the data structure.  Note that
 731          * aggregation only happens if ops are marked batch'able.
 732          * Aggregating ops reduces the number of interrupts to the host
 733          * but also (potentially) increases the latency for processing
 734          * completed ops as we only get an interrupt when all aggregated
 735          * ops have completed.
 736          */
 737         if (sc->sc_nqueue == 0)
 738                 return;
 739         if (sc->sc_nqueue > 1) {
 740                 npkts = 0;
 741                 SIMPLEQ_FOREACH(q, &sc->sc_queue, q_next) {
 742                         npkts++;
 743                         if ((q->q_crp->crp_flags & CRYPTO_F_BATCH) == 0)
 744                                 break;
 745                 }
 746         } else
 747                 npkts = 1;
 748         /*
 749          * Check device status before going any further.
 750          */
 751         if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) {
 752                 if (stat & BS_STAT_DMAERR) {
 753                         ubsec_totalreset(sc);
 754                         ubsecstats.hst_dmaerr++;
 755                 } else
 756                         ubsecstats.hst_mcr1full++;
 757                 return;
 758         }
 759         if (sc->sc_nqueue > ubsecstats.hst_maxqueue)
 760                 ubsecstats.hst_maxqueue = sc->sc_nqueue;
 761         if (npkts > UBS_MAX_AGGR)
 762                 npkts = UBS_MAX_AGGR;
 763         if (npkts < 2)                          /* special case 1 op */
 764                 goto feed1;
 765
 766         ubsecstats.hst_totbatch += npkts-1;
 767 #ifdef UBSEC_DEBUG
 768         if (ubsec_debug)
 769                 printf("merging %d records\n", npkts);
 770 #endif /* UBSEC_DEBUG */
 771
 772         q = SIMPLEQ_FIRST(&sc->sc_queue);
 773         SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q, q_next);
 774         --sc->sc_nqueue;
 775
 776         bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE);
 777         if (q->q_dst_map != NULL)
 778                 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD);
 779
 780         q->q_nstacked_mcrs = npkts - 1;         /* Number of packets stacked */
 781
 782         for (i = 0; i < q->q_nstacked_mcrs; i++) {
 783                 q2 = SIMPLEQ_FIRST(&sc->sc_queue);
 784                 bus_dmamap_sync(sc->sc_dmat, q2->q_src_map,
 785                     BUS_DMASYNC_PREWRITE);
 786                 if (q2->q_dst_map != NULL)
 787                         bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map,
 788                             BUS_DMASYNC_PREREAD);
 789                 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q2, q_next);
 790                 --sc->sc_nqueue;
 791
 792                 v = (void*)(((char *)&q2->q_dma->d_dma->d_mcr) + sizeof(struct ubsec_mcr) -
 793                     sizeof(struct ubsec_mcr_add));
 794                 bcopy(v, &q->q_dma->d_dma->d_mcradd[i], sizeof(struct ubsec_mcr_add));
 795                 q->q_stacked_mcr[i] = q2;
 796         }
 797         q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts);
 798         SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
 799         sc->sc_nqchip += npkts;
 800         if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
 801                 ubsecstats.hst_maxqchip = sc->sc_nqchip;
 802         ubsec_dma_sync(&q->q_dma->d_alloc,
 803             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 804         WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
 805             offsetof(struct ubsec_dmachunk, d_mcr));
 806         return;
 807 feed1:
 808         q = SIMPLEQ_FIRST(&sc->sc_queue);
 809
 810         bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE);
 811         if (q->q_dst_map != NULL)
 812                 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD);
 813         ubsec_dma_sync(&q->q_dma->d_alloc,
 814             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 815
 816         WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
 817             offsetof(struct ubsec_dmachunk, d_mcr));
 818 #ifdef UBSEC_DEBUG
 819         if (ubsec_debug)
 820                 printf("feed1: q->chip %p %08x stat %08x\n",
 821                       q, (u_int32_t)vtophys(&q->q_dma->d_dma->d_mcr),
 822                       stat);
 823 #endif /* UBSEC_DEBUG */
 824         SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q, q_next);
 825         --sc->sc_nqueue;
 826         SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
 827         sc->sc_nqchip++;
 828         if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
 829                 ubsecstats.hst_maxqchip = sc->sc_nqchip;
 830         return;
 831 }
 832
 833 /*
 834  * Allocate a new 'session' and return an encoded session id.  'sidp'
 835  * contains our registration id, and should contain an encoded session
 836  * id on successful allocation.
 837  */
 838 static int
 839 ubsec_newsession(void *arg, u_int32_t *sidp, struct cryptoini *cri)
 840 {
 841         struct cryptoini *c, *encini = NULL, *macini = NULL;
 842         struct ubsec_softc *sc = arg;
 843         struct ubsec_session *ses = NULL;
 844         MD5_CTX md5ctx;
 845         SHA1_CTX sha1ctx;
 846         int i, sesn;
 847
 848         if (sidp == NULL || cri == NULL || sc == NULL)
 849                 return (EINVAL);
 850
 851         for (c = cri; c != NULL; c = c->cri_next) {
 852                 if (c->cri_alg == CRYPTO_MD5_HMAC ||
 853                     c->cri_alg == CRYPTO_SHA1_HMAC) {
 854                         if (macini)
 855                                 return (EINVAL);
 856                         macini = c;
 857                 } else if (c->cri_alg == CRYPTO_DES_CBC ||
 858                     c->cri_alg == CRYPTO_3DES_CBC) {
 859                         if (encini)
 860                                 return (EINVAL);
 861                         encini = c;
 862                 } else
 863                         return (EINVAL);
 864         }
 865         if (encini == NULL && macini == NULL)
 866                 return (EINVAL);
 867
 868         if (sc->sc_sessions == NULL) {
 869                 ses = sc->sc_sessions = (struct ubsec_session *)malloc(
 870                     sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT);
 871                 if (ses == NULL)
 872                         return (ENOMEM);
 873                 sesn = 0;
 874                 sc->sc_nsessions = 1;
 875         } else {
 876                 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
 877                         if (sc->sc_sessions[sesn].ses_used == 0) {
 878                                 ses = &sc->sc_sessions[sesn];
 879                                 break;
 880                         }
 881                 }
 882
 883                 if (ses == NULL) {
 884                         sesn = sc->sc_nsessions;
 885                         ses = (struct ubsec_session *)malloc((sesn + 1) *
 886                             sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT);
 887                         if (ses == NULL)
 888                                 return (ENOMEM);
 889                         bcopy(sc->sc_sessions, ses, sesn *
 890                             sizeof(struct ubsec_session));
 891                         bzero(sc->sc_sessions, sesn *
 892                             sizeof(struct ubsec_session));
 893                         free(sc->sc_sessions, M_DEVBUF);
 894                         sc->sc_sessions = ses;
 895                         ses = &sc->sc_sessions[sesn];
 896                         sc->sc_nsessions++;
 897                 }
 898         }
 899         bzero(ses, sizeof(struct ubsec_session));
 900         ses->ses_used = 1;
 901
 902         if (encini) {
 903                 /* get an IV, network byte order */
 904                 /* XXX may read fewer than requested */
 905                 read_random(ses->ses_iv, sizeof(ses->ses_iv));
 906
 907                 /* Go ahead and compute key in ubsec's byte order */
 908                 if (encini->cri_alg == CRYPTO_DES_CBC) {
 909                         bcopy(encini->cri_key, &ses->ses_deskey[0], 8);
 910                         bcopy(encini->cri_key, &ses->ses_deskey[2], 8);
 911                         bcopy(encini->cri_key, &ses->ses_deskey[4], 8);
 912                 } else
 913                         bcopy(encini->cri_key, ses->ses_deskey, 24);
 914
 915                 SWAP32(ses->ses_deskey[0]);
 916                 SWAP32(ses->ses_deskey[1]);
 917                 SWAP32(ses->ses_deskey[2]);
 918                 SWAP32(ses->ses_deskey[3]);
 919                 SWAP32(ses->ses_deskey[4]);
 920                 SWAP32(ses->ses_deskey[5]);
 921         }
 922
 923         if (macini) {
 924                 ses->ses_mlen = macini->cri_mlen;
 925                 if (ses->ses_mlen == 0) {
 926                         if (macini->cri_alg == CRYPTO_MD5_HMAC)
 927                                 ses->ses_mlen = MD5_DIGEST_LENGTH;
 928                         else
 929                                 ses->ses_mlen = SHA1_RESULTLEN;
 930                 }
 931
 932                 for (i = 0; i < macini->cri_klen / 8; i++)
 933                         macini->cri_key[i] ^= HMAC_IPAD_VAL;
 934
 935                 if (macini->cri_alg == CRYPTO_MD5_HMAC) {
 936                         MD5Init(&md5ctx);
 937                         MD5Update(&md5ctx, macini->cri_key,
 938                             macini->cri_klen / 8);
 939                         MD5Update(&md5ctx, hmac_ipad_buffer,
 940                             HMAC_BLOCK_LEN - (macini->cri_klen / 8));
 941                         bcopy(md5ctx.state, ses->ses_hminner,
 942                             sizeof(md5ctx.state));
 943                 } else {
 944                         SHA1Init(&sha1ctx);
 945                         SHA1Update(&sha1ctx, macini->cri_key,
 946                             macini->cri_klen / 8);
 947                         SHA1Update(&sha1ctx, hmac_ipad_buffer,
 948                             HMAC_BLOCK_LEN - (macini->cri_klen / 8));
 949                         bcopy(sha1ctx.h.b32, ses->ses_hminner,
 950                             sizeof(sha1ctx.h.b32));
 951                 }
 952
 953                 for (i = 0; i < macini->cri_klen / 8; i++)
 954                         macini->cri_key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
 955
 956                 if (macini->cri_alg == CRYPTO_MD5_HMAC) {
 957                         MD5Init(&md5ctx);
 958                         MD5Update(&md5ctx, macini->cri_key,
 959                             macini->cri_klen / 8);
 960                         MD5Update(&md5ctx, hmac_opad_buffer,
 961                             HMAC_BLOCK_LEN - (macini->cri_klen / 8));
 962                         bcopy(md5ctx.state, ses->ses_hmouter,
 963                             sizeof(md5ctx.state));
 964                 } else {
 965                         SHA1Init(&sha1ctx);
 966                         SHA1Update(&sha1ctx, macini->cri_key,
 967                             macini->cri_klen / 8);
 968                         SHA1Update(&sha1ctx, hmac_opad_buffer,
 969                             HMAC_BLOCK_LEN - (macini->cri_klen / 8));
 970                         bcopy(sha1ctx.h.b32, ses->ses_hmouter,
 971                             sizeof(sha1ctx.h.b32));
 972                 }
 973
 974                 for (i = 0; i < macini->cri_klen / 8; i++)
 975                         macini->cri_key[i] ^= HMAC_OPAD_VAL;
 976         }
 977
 978         *sidp = UBSEC_SID(device_get_unit(sc->sc_dev), sesn);
 979         return (0);
 980 }
 981
 982 /*
 983  * Deallocate a session.
 984  */
 985 static int
 986 ubsec_freesession(void *arg, u_int64_t tid)
 987 {
 988         struct ubsec_softc *sc = arg;
 989         int session, ret;
 990         u_int32_t sid = CRYPTO_SESID2LID(tid);
 991
 992         if (sc == NULL)
 993                 return (EINVAL);
 994
 995         session = UBSEC_SESSION(sid);
 996         if (session < sc->sc_nsessions) {
 997                 bzero(&sc->sc_sessions[session],
 998                         sizeof(sc->sc_sessions[session]));
 999                 ret = 0;
1000         } else
1001                 ret = EINVAL;
1002
1003         return (ret);
1004 }
1005
1006 static void
1007 ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error)
1008 {
1009         struct ubsec_operand *op = arg;
1010
1011         KASSERT(nsegs <= UBS_MAX_SCATTER,
1012                 ("Too many DMA segments returned when mapping operand"));
1013 #ifdef UBSEC_DEBUG
1014         if (ubsec_debug)
1015                 printf("ubsec_op_cb: mapsize %u nsegs %d\n",
1016                         (u_int) mapsize, nsegs);
1017 #endif
1018         op->mapsize = mapsize;
1019         op->nsegs = nsegs;
1020         bcopy(seg, op->segs, nsegs * sizeof (seg[0]));
1021 }
1022
1023 static int
1024 ubsec_process(void *arg, struct cryptop *crp, int hint)
1025 {
1026         struct ubsec_q *q = NULL;
1027         int err = 0, i, j, nicealign;
1028         struct ubsec_softc *sc = arg;
1029         struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
1030         int encoffset = 0, macoffset = 0, cpskip, cpoffset;
1031         int sskip, dskip, stheend, dtheend;
1032         int16_t coffset;
1033         struct ubsec_session *ses;
1034         struct ubsec_pktctx ctx;
1035         struct ubsec_dma *dmap = NULL;
1036
1037         if (crp == NULL || crp->crp_callback == NULL || sc == NULL) {
1038                 ubsecstats.hst_invalid++;
1039                 return (EINVAL);
1040         }
1041         if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) {
1042                 ubsecstats.hst_badsession++;
1043                 return (EINVAL);
1044         }
1045
1046         mtx_lock(&sc->sc_freeqlock);
1047         if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
1048                 ubsecstats.hst_queuefull++;
1049                 sc->sc_needwakeup |= CRYPTO_SYMQ;
1050                 mtx_unlock(&sc->sc_freeqlock);
1051                 return (ERESTART);
1052         }
1053         q = SIMPLEQ_FIRST(&sc->sc_freequeue);
1054         SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q, q_next);
1055         mtx_unlock(&sc->sc_freeqlock);
1056
1057         dmap = q->q_dma; /* Save dma pointer */
1058         bzero(q, sizeof(struct ubsec_q));
1059         bzero(&ctx, sizeof(ctx));
1060
1061         q->q_sesn = UBSEC_SESSION(crp->crp_sid);
1062         q->q_dma = dmap;
1063         ses = &sc->sc_sessions[q->q_sesn];
1064
1065         if (crp->crp_flags & CRYPTO_F_IMBUF) {
1066                 q->q_src_m = (struct mbuf *)crp->crp_buf;
1067                 q->q_dst_m = (struct mbuf *)crp->crp_buf;
1068         } else if (crp->crp_flags & CRYPTO_F_IOV) {
1069                 q->q_src_io = (struct uio *)crp->crp_buf;
1070                 q->q_dst_io = (struct uio *)crp->crp_buf;
1071         } else {
1072                 ubsecstats.hst_badflags++;
1073                 err = EINVAL;
1074                 goto errout;    /* XXX we don't handle contiguous blocks! */
1075         }
1076
1077         bzero(&dmap->d_dma->d_mcr, sizeof(struct ubsec_mcr));
1078
1079         dmap->d_dma->d_mcr.mcr_pkts = htole16(1);
1080         dmap->d_dma->d_mcr.mcr_flags = 0;
1081         q->q_crp = crp;
1082
1083         crd1 = crp->crp_desc;
1084         if (crd1 == NULL) {
1085                 ubsecstats.hst_nodesc++;
1086                 err = EINVAL;
1087                 goto errout;
1088         }
1089         crd2 = crd1->crd_next;
1090
1091         if ((crd1->crd_flags & CRD_F_KEY_EXPLICIT) ||
1092             (crd2 != NULL && (crd2->crd_flags & CRD_F_KEY_EXPLICIT))) {
1093                 ubsecstats.hst_badflags++;
1094                 err = EINVAL;
1095                 goto errout;
1096         }
1097
1098         if (crd2 == NULL) {
1099                 if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
1100                     crd1->crd_alg == CRYPTO_SHA1_HMAC) {
1101                         maccrd = crd1;
1102                         enccrd = NULL;
1103                 } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
1104                     crd1->crd_alg == CRYPTO_3DES_CBC) {
1105                         maccrd = NULL;
1106                         enccrd = crd1;
1107                 } else {
1108                         ubsecstats.hst_badalg++;
1109                         err = EINVAL;
1110                         goto errout;
1111                 }
1112         } else {
1113                 if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
1114                     crd1->crd_alg == CRYPTO_SHA1_HMAC) &&
1115                     (crd2->crd_alg == CRYPTO_DES_CBC ||
1116                         crd2->crd_alg == CRYPTO_3DES_CBC) &&
1117                     ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
1118                         maccrd = crd1;
1119                         enccrd = crd2;
1120                 } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
1121                     crd1->crd_alg == CRYPTO_3DES_CBC) &&
1122                     (crd2->crd_alg == CRYPTO_MD5_HMAC ||
1123                         crd2->crd_alg == CRYPTO_SHA1_HMAC) &&
1124                     (crd1->crd_flags & CRD_F_ENCRYPT)) {
1125                         enccrd = crd1;
1126                         maccrd = crd2;
1127                 } else {
1128                         /*
1129                          * We cannot order the ubsec as requested
1130                          */
1131                         ubsecstats.hst_badalg++;
1132                         err = EINVAL;
1133                         goto errout;
1134                 }
1135         }
1136
1137         if (enccrd) {
1138                 encoffset = enccrd->crd_skip;
1139                 ctx.pc_flags |= htole16(UBS_PKTCTX_ENC_3DES);
1140
1141                 if (enccrd->crd_flags & CRD_F_ENCRYPT) {
1142                         q->q_flags |= UBSEC_QFLAGS_COPYOUTIV;
1143
1144                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
1145                                 bcopy(enccrd->crd_iv, ctx.pc_iv, 8);
1146                         else {
1147                                 ctx.pc_iv[0] = ses->ses_iv[0];
1148                                 ctx.pc_iv[1] = ses->ses_iv[1];
1149                         }
1150
1151                         if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
1152                                 if (crp->crp_flags & CRYPTO_F_IMBUF)
1153                                         m_copyback(q->q_src_m,
1154                                             enccrd->crd_inject,
1155                                             8, (caddr_t)ctx.pc_iv);
1156                                 else if (crp->crp_flags & CRYPTO_F_IOV)
1157                                         cuio_copyback(q->q_src_io,
1158                                             enccrd->crd_inject,
1159                                             8, (caddr_t)ctx.pc_iv);
1160                         }
1161                 } else {
1162                         ctx.pc_flags |= htole16(UBS_PKTCTX_INBOUND);
1163
1164                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
1165                                 bcopy(enccrd->crd_iv, ctx.pc_iv, 8);
1166                         else if (crp->crp_flags & CRYPTO_F_IMBUF)
1167                                 m_copydata(q->q_src_m, enccrd->crd_inject,
1168                                     8, (caddr_t)ctx.pc_iv);
1169                         else if (crp->crp_flags & CRYPTO_F_IOV)
1170                                 cuio_copydata(q->q_src_io,
1171                                     enccrd->crd_inject, 8,
1172                                     (caddr_t)ctx.pc_iv);
1173                 }
1174
1175                 ctx.pc_deskey[0] = ses->ses_deskey[0];
1176                 ctx.pc_deskey[1] = ses->ses_deskey[1];
1177                 ctx.pc_deskey[2] = ses->ses_deskey[2];
1178                 ctx.pc_deskey[3] = ses->ses_deskey[3];
1179                 ctx.pc_deskey[4] = ses->ses_deskey[4];
1180                 ctx.pc_deskey[5] = ses->ses_deskey[5];
1181                 SWAP32(ctx.pc_iv[0]);
1182                 SWAP32(ctx.pc_iv[1]);
1183         }
1184
1185         if (maccrd) {
1186                 macoffset = maccrd->crd_skip;
1187
1188                 if (maccrd->crd_alg == CRYPTO_MD5_HMAC)
1189                         ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_MD5);
1190                 else
1191                         ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_SHA1);
1192
1193                 for (i = 0; i < 5; i++) {
1194                         ctx.pc_hminner[i] = ses->ses_hminner[i];
1195                         ctx.pc_hmouter[i] = ses->ses_hmouter[i];
1196
1197                         HTOLE32(ctx.pc_hminner[i]);
1198                         HTOLE32(ctx.pc_hmouter[i]);
1199                 }
1200         }
1201
1202         if (enccrd && maccrd) {
1203                 /*
1204                  * ubsec cannot handle packets where the end of encryption
1205                  * and authentication are not the same, or where the
1206                  * encrypted part begins before the authenticated part.
1207                  */
1208                 if ((encoffset + enccrd->crd_len) !=
1209                     (macoffset + maccrd->crd_len)) {
1210                         ubsecstats.hst_lenmismatch++;
1211                         err = EINVAL;
1212                         goto errout;
1213                 }
1214                 if (enccrd->crd_skip < maccrd->crd_skip) {
1215                         ubsecstats.hst_skipmismatch++;
1216                         err = EINVAL;
1217                         goto errout;
1218                 }
1219                 sskip = maccrd->crd_skip;
1220                 cpskip = dskip = enccrd->crd_skip;
1221                 stheend = maccrd->crd_len;
1222                 dtheend = enccrd->crd_len;
1223                 coffset = enccrd->crd_skip - maccrd->crd_skip;
1224                 cpoffset = cpskip + dtheend;
1225 #ifdef UBSEC_DEBUG
1226                 if (ubsec_debug) {
1227                         printf("mac: skip %d, len %d, inject %d\n",
1228                             maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject);
1229                         printf("enc: skip %d, len %d, inject %d\n",
1230                             enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject);
1231                         printf("src: skip %d, len %d\n", sskip, stheend);
1232                         printf("dst: skip %d, len %d\n", dskip, dtheend);
1233                         printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n",
1234                             coffset, stheend, cpskip, cpoffset);
1235                 }
1236 #endif
1237         } else {
1238                 cpskip = dskip = sskip = macoffset + encoffset;
1239                 dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len;
1240                 cpoffset = cpskip + dtheend;
1241                 coffset = 0;
1242         }
1243         ctx.pc_offset = htole16(coffset >> 2);
1244
1245         if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &q->q_src_map)) {
1246                 ubsecstats.hst_nomap++;
1247                 err = ENOMEM;
1248                 goto errout;
1249         }
1250         if (crp->crp_flags & CRYPTO_F_IMBUF) {
1251                 if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map,
1252                     q->q_src_m, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) {
1253                         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1254                         q->q_src_map = NULL;
1255                         ubsecstats.hst_noload++;
1256                         err = ENOMEM;
1257                         goto errout;
1258                 }
1259         } else if (crp->crp_flags & CRYPTO_F_IOV) {
1260                 if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map,
1261                     q->q_src_io, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) {
1262                         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1263                         q->q_src_map = NULL;
1264                         ubsecstats.hst_noload++;
1265                         err = ENOMEM;
1266                         goto errout;
1267                 }
1268         }
1269         nicealign = ubsec_dmamap_aligned(&q->q_src);
1270
1271         dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend);
1272
1273 #ifdef UBSEC_DEBUG
1274         if (ubsec_debug)
1275                 printf("src skip: %d nicealign: %u\n", sskip, nicealign);
1276 #endif
1277         for (i = j = 0; i < q->q_src_nsegs; i++) {
1278                 struct ubsec_pktbuf *pb;
1279                 bus_size_t packl = q->q_src_segs[i].ds_len;
1280                 bus_addr_t packp = q->q_src_segs[i].ds_addr;
1281
1282                 if (sskip >= packl) {
1283                         sskip -= packl;
1284                         continue;
1285                 }
1286
1287                 packl -= sskip;
1288                 packp += sskip;
1289                 sskip = 0;
1290
1291                 if (packl > 0xfffc) {
1292                         err = EIO;
1293                         goto errout;
1294                 }
1295
1296                 if (j == 0)
1297                         pb = &dmap->d_dma->d_mcr.mcr_ipktbuf;
1298                 else
1299                         pb = &dmap->d_dma->d_sbuf[j - 1];
1300
1301                 pb->pb_addr = htole32(packp);
1302
1303                 if (stheend) {
1304                         if (packl > stheend) {
1305                                 pb->pb_len = htole32(stheend);
1306                                 stheend = 0;
1307                         } else {
1308                                 pb->pb_len = htole32(packl);
1309                                 stheend -= packl;
1310                         }
1311                 } else
1312                         pb->pb_len = htole32(packl);
1313
1314                 if ((i + 1) == q->q_src_nsegs)
1315                         pb->pb_next = 0;
1316                 else
1317                         pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1318                             offsetof(struct ubsec_dmachunk, d_sbuf[j]));
1319                 j++;
1320         }
1321
1322         if (enccrd == NULL && maccrd != NULL) {
1323                 dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0;
1324                 dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0;
1325                 dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr +
1326                     offsetof(struct ubsec_dmachunk, d_macbuf[0]));
1327 #ifdef UBSEC_DEBUG
1328                 if (ubsec_debug)
1329                         printf("opkt: %x %x %x\n",
1330                             dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr,
1331                             dmap->d_dma->d_mcr.mcr_opktbuf.pb_len,
1332                             dmap->d_dma->d_mcr.mcr_opktbuf.pb_next);
1333 #endif
1334         } else {
1335                 if (crp->crp_flags & CRYPTO_F_IOV) {
1336                         if (!nicealign) {
1337                                 ubsecstats.hst_iovmisaligned++;
1338                                 err = EINVAL;
1339                                 goto errout;
1340                         }
1341                         if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT,
1342                              &q->q_dst_map)) {
1343                                 ubsecstats.hst_nomap++;
1344                                 err = ENOMEM;
1345                                 goto errout;
1346                         }
1347                         if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map,
1348                             q->q_dst_io, ubsec_op_cb, &q->q_dst, BUS_DMA_NOWAIT) != 0) {
1349                                 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
1350                                 q->q_dst_map = NULL;
1351                                 ubsecstats.hst_noload++;
1352                                 err = ENOMEM;
1353                                 goto errout;
1354                         }
1355                 } else if (crp->crp_flags & CRYPTO_F_IMBUF) {
1356                         if (nicealign) {
1357                                 q->q_dst = q->q_src;
1358                         } else {
1359                                 int totlen, len;
1360                                 struct mbuf *m, *top, **mp;
1361
1362                                 ubsecstats.hst_unaligned++;
1363                                 totlen = q->q_src_mapsize;
1364                                 if (q->q_src_m->m_flags & M_PKTHDR) {
1365                                         len = MHLEN;
1366                                         MGETHDR(m, M_DONTWAIT, MT_DATA);
1367                                         if (m && !m_dup_pkthdr(m, q->q_src_m, M_DONTWAIT)) {
1368                                                 m_free(m);
1369                                                 m = NULL;
1370                                         }
1371                                 } else {
1372                                         len = MLEN;
1373                                         MGET(m, M_DONTWAIT, MT_DATA);
1374                                 }
1375                                 if (m == NULL) {
1376                                         ubsecstats.hst_nombuf++;
1377                                         err = sc->sc_nqueue ? ERESTART : ENOMEM;
1378                                         goto errout;
1379                                 }
1380                                 if (totlen >= MINCLSIZE) {
1381                                         MCLGET(m, M_DONTWAIT);
1382                                         if ((m->m_flags & M_EXT) == 0) {
1383                                                 m_free(m);
1384                                                 ubsecstats.hst_nomcl++;
1385                                                 err = sc->sc_nqueue ? ERESTART : ENOMEM;
1386                                                 goto errout;
1387                                         }
1388                                         len = MCLBYTES;
1389                                 }
1390                                 m->m_len = len;
1391                                 top = NULL;
1392                                 mp = &top;
1393
1394                                 while (totlen > 0) {
1395                                         if (top) {
1396                                                 MGET(m, M_DONTWAIT, MT_DATA);
1397                                                 if (m == NULL) {
1398                                                         m_freem(top);
1399                                                         ubsecstats.hst_nombuf++;
1400                                                         err = sc->sc_nqueue ? ERESTART : ENOMEM;
1401                                                         goto errout;
1402                                                 }
1403                                                 len = MLEN;
1404                                         }
1405                                         if (top && totlen >= MINCLSIZE) {
1406                                                 MCLGET(m, M_DONTWAIT);
1407                                                 if ((m->m_flags & M_EXT) == 0) {
1408                                                         *mp = m;
1409                                                         m_freem(top);
1410                                                         ubsecstats.hst_nomcl++;
1411                                                         err = sc->sc_nqueue ? ERESTART : ENOMEM;
1412                                                         goto errout;
1413                                                 }
1414                                                 len = MCLBYTES;
1415                                         }
1416                                         m->m_len = len = min(totlen, len);
1417                                         totlen -= len;
1418                                         *mp = m;
1419                                         mp = &m->m_next;
1420                                 }
1421                                 q->q_dst_m = top;
1422                                 ubsec_mcopy(q->q_src_m, q->q_dst_m,
1423                                     cpskip, cpoffset);
1424                                 if (bus_dmamap_create(sc->sc_dmat,
1425                                     BUS_DMA_NOWAIT, &q->q_dst_map) != 0) {
1426                                         ubsecstats.hst_nomap++;
1427                                         err = ENOMEM;
1428                                         goto errout;
1429                                 }
1430                                 if (bus_dmamap_load_mbuf(sc->sc_dmat,
1431                                     q->q_dst_map, q->q_dst_m,
1432                                     ubsec_op_cb, &q->q_dst,
1433                                     BUS_DMA_NOWAIT) != 0) {
1434                                         bus_dmamap_destroy(sc->sc_dmat,
1435                                         q->q_dst_map);
1436                                         q->q_dst_map = NULL;
1437                                         ubsecstats.hst_noload++;
1438                                         err = ENOMEM;
1439                                         goto errout;
1440                                 }
1441                         }
1442                 } else {
1443                         ubsecstats.hst_badflags++;
1444                         err = EINVAL;
1445                         goto errout;
1446                 }
1447
1448 #ifdef UBSEC_DEBUG
1449                 if (ubsec_debug)
1450                         printf("dst skip: %d\n", dskip);
1451 #endif
1452                 for (i = j = 0; i < q->q_dst_nsegs; i++) {
1453                         struct ubsec_pktbuf *pb;
1454                         bus_size_t packl = q->q_dst_segs[i].ds_len;
1455                         bus_addr_t packp = q->q_dst_segs[i].ds_addr;
1456
1457                         if (dskip >= packl) {
1458                                 dskip -= packl;
1459                                 continue;
1460                         }
1461
1462                         packl -= dskip;
1463                         packp += dskip;
1464                         dskip = 0;
1465
1466                         if (packl > 0xfffc) {
1467                                 err = EIO;
1468                                 goto errout;
1469                         }
1470
1471                         if (j == 0)
1472                                 pb = &dmap->d_dma->d_mcr.mcr_opktbuf;
1473                         else
1474                                 pb = &dmap->d_dma->d_dbuf[j - 1];
1475
1476                         pb->pb_addr = htole32(packp);
1477
1478                         if (dtheend) {
1479                                 if (packl > dtheend) {
1480                                         pb->pb_len = htole32(dtheend);
1481                                         dtheend = 0;
1482                                 } else {
1483                                         pb->pb_len = htole32(packl);
1484                                         dtheend -= packl;
1485                                 }
1486                         } else
1487                                 pb->pb_len = htole32(packl);
1488
1489                         if ((i + 1) == q->q_dst_nsegs) {
1490                                 if (maccrd)
1491                                         pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1492                                             offsetof(struct ubsec_dmachunk, d_macbuf[0]));
1493                                 else
1494                                         pb->pb_next = 0;
1495                         } else
1496                                 pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
1497                                     offsetof(struct ubsec_dmachunk, d_dbuf[j]));
1498                         j++;
1499                 }
1500         }
1501
1502         dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr +
1503             offsetof(struct ubsec_dmachunk, d_ctx));
1504
1505         if (sc->sc_flags & UBS_FLAGS_LONGCTX) {
1506                 struct ubsec_pktctx_long *ctxl;
1507
1508                 ctxl = (struct ubsec_pktctx_long *)(dmap->d_alloc.dma_vaddr +
1509                     offsetof(struct ubsec_dmachunk, d_ctx));
1510
1511                 /* transform small context into long context */
1512                 ctxl->pc_len = htole16(sizeof(struct ubsec_pktctx_long));
1513                 ctxl->pc_type = htole16(UBS_PKTCTX_TYPE_IPSEC);
1514                 ctxl->pc_flags = ctx.pc_flags;
1515                 ctxl->pc_offset = ctx.pc_offset;
1516                 for (i = 0; i < 6; i++)
1517                         ctxl->pc_deskey[i] = ctx.pc_deskey[i];
1518                 for (i = 0; i < 5; i++)
1519                         ctxl->pc_hminner[i] = ctx.pc_hminner[i];
1520                 for (i = 0; i < 5; i++)
1521                         ctxl->pc_hmouter[i] = ctx.pc_hmouter[i];
1522                 ctxl->pc_iv[0] = ctx.pc_iv[0];
1523                 ctxl->pc_iv[1] = ctx.pc_iv[1];
1524         } else
1525                 bcopy(&ctx, dmap->d_alloc.dma_vaddr +
1526                     offsetof(struct ubsec_dmachunk, d_ctx),
1527                     sizeof(struct ubsec_pktctx));
1528
1529         mtx_lock(&sc->sc_mcr1lock);
1530         SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next);
1531         sc->sc_nqueue++;
1532         ubsecstats.hst_ipackets++;
1533         ubsecstats.hst_ibytes += dmap->d_alloc.dma_size;
1534         if ((hint & CRYPTO_HINT_MORE) == 0 || sc->sc_nqueue >= UBS_MAX_AGGR)
1535                 ubsec_feed(sc);
1536         mtx_unlock(&sc->sc_mcr1lock);
1537         return (0);
1538
1539 errout:
1540         if (q != NULL) {
1541                 if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
1542                         m_freem(q->q_dst_m);
1543
1544                 if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
1545                         bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
1546                         bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
1547                 }
1548                 if (q->q_src_map != NULL) {
1549                         bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
1550                         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1551                 }
1552         }
1553         if (q != NULL || err == ERESTART) {
1554                 mtx_lock(&sc->sc_freeqlock);
1555                 if (q != NULL)
1556                         SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
1557                 if (err == ERESTART)
1558                         sc->sc_needwakeup |= CRYPTO_SYMQ;
1559                 mtx_unlock(&sc->sc_freeqlock);
1560         }
1561         if (err != ERESTART) {
1562                 crp->crp_etype = err;
1563                 crypto_done(crp);
1564         }
1565         return (err);
1566 }
1567
1568 static void
1569 ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q)
1570 {
1571         struct cryptop *crp = (struct cryptop *)q->q_crp;
1572         struct cryptodesc *crd;
1573         struct ubsec_dma *dmap = q->q_dma;
1574
1575         ubsecstats.hst_opackets++;
1576         ubsecstats.hst_obytes += dmap->d_alloc.dma_size;
1577
1578         ubsec_dma_sync(&dmap->d_alloc,
1579             BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
1580         if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
1581                 bus_dmamap_sync(sc->sc_dmat, q->q_dst_map,
1582                     BUS_DMASYNC_POSTREAD);
1583                 bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
1584                 bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
1585         }
1586         bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_POSTWRITE);
1587         bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
1588         bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
1589
1590         if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) {
1591                 m_freem(q->q_src_m);
1592                 crp->crp_buf = (caddr_t)q->q_dst_m;
1593         }
1594         ubsecstats.hst_obytes += ((struct mbuf *)crp->crp_buf)->m_len;
1595
1596         /* copy out IV for future use */
1597         if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) {
1598                 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1599                         if (crd->crd_alg != CRYPTO_DES_CBC &&
1600                             crd->crd_alg != CRYPTO_3DES_CBC)
1601                                 continue;
1602                         if (crp->crp_flags & CRYPTO_F_IMBUF)
1603                                 m_copydata((struct mbuf *)crp->crp_buf,
1604                                     crd->crd_skip + crd->crd_len - 8, 8,
1605                                     (caddr_t)sc->sc_sessions[q->q_sesn].ses_iv);
1606                         else if (crp->crp_flags & CRYPTO_F_IOV) {
1607                                 cuio_copydata((struct uio *)crp->crp_buf,
1608                                     crd->crd_skip + crd->crd_len - 8, 8,
1609                                     (caddr_t)sc->sc_sessions[q->q_sesn].ses_iv);
1610                         }
1611                         break;
1612                 }
1613         }
1614
1615         for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1616                 if (crd->crd_alg != CRYPTO_MD5_HMAC &&
1617                     crd->crd_alg != CRYPTO_SHA1_HMAC)
1618                         continue;
1619                 if (crp->crp_flags & CRYPTO_F_IMBUF)
1620                         m_copyback((struct mbuf *)crp->crp_buf,
1621                             crd->crd_inject,
1622                             sc->sc_sessions[q->q_sesn].ses_mlen,
1623                             (caddr_t)dmap->d_dma->d_macbuf);
1624                 else if (crp->crp_flags & CRYPTO_F_IOV)
1625                         cuio_copyback((struct uio *)crp->crp_buf,
1626                             crd->crd_inject,
1627                             sc->sc_sessions[q->q_sesn].ses_mlen,
1628                             (caddr_t)dmap->d_dma->d_macbuf);
1629                 break;
1630         }
1631         mtx_lock(&sc->sc_freeqlock);
1632         SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
1633         mtx_unlock(&sc->sc_freeqlock);
1634         crypto_done(crp);
1635 }
1636
1637 static void
1638 ubsec_mcopy(struct mbuf *srcm, struct mbuf *dstm, int hoffset, int toffset)
1639 {
1640         int i, j, dlen, slen;
1641         caddr_t dptr, sptr;
1642
1643         j = 0;
1644         sptr = srcm->m_data;
1645         slen = srcm->m_len;
1646         dptr = dstm->m_data;
1647         dlen = dstm->m_len;
1648
1649         while (1) {
1650                 for (i = 0; i < min(slen, dlen); i++) {
1651                         if (j < hoffset || j >= toffset)
1652                                 *dptr++ = *sptr++;
1653                         slen--;
1654                         dlen--;
1655                         j++;
1656                 }
1657                 if (slen == 0) {
1658                         srcm = srcm->m_next;
1659                         if (srcm == NULL)
1660                                 return;
1661                         sptr = srcm->m_data;
1662                         slen = srcm->m_len;
1663                 }
1664                 if (dlen == 0) {
1665                         dstm = dstm->m_next;
1666                         if (dstm == NULL)
1667                                 return;
1668                         dptr = dstm->m_data;
1669                         dlen = dstm->m_len;
1670                 }
1671         }
1672 }
1673
1674 /*
1675  * feed the key generator, must be called at splimp() or higher.
1676  */
1677 static int
1678 ubsec_feed2(struct ubsec_softc *sc)
1679 {
1680         struct ubsec_q2 *q;
1681
1682         while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) {
1683                 if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL)
1684                         break;
1685                 q = SIMPLEQ_FIRST(&sc->sc_queue2);
1686
1687                 ubsec_dma_sync(&q->q_mcr,
1688                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1689                 ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_PREWRITE);
1690
1691                 WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr);
1692                 SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, q, q_next);
1693                 --sc->sc_nqueue2;
1694                 SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next);
1695         }
1696         return (0);
1697 }
1698
1699 /*
1700  * Callback for handling random numbers
1701  */
1702 static void
1703 ubsec_callback2(struct ubsec_softc *sc, struct ubsec_q2 *q)
1704 {
1705         struct cryptkop *krp;
1706         struct ubsec_ctx_keyop *ctx;
1707
1708         ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr;
1709         ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_POSTWRITE);
1710
1711         switch (q->q_type) {
1712 #ifndef UBSEC_NO_RNG
1713         case UBS_CTXOP_RNGBYPASS: {
1714                 struct ubsec_q2_rng *rng = (struct ubsec_q2_rng *)q;
1715
1716                 ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_POSTREAD);
1717                 (*sc->sc_harvest)(sc->sc_rndtest,
1718                         rng->rng_buf.dma_vaddr,
1719                         UBSEC_RNG_BUFSIZ*sizeof (u_int32_t));
1720                 rng->rng_used = 0;
1721                 callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
1722                 break;
1723         }
1724 #endif
1725         case UBS_CTXOP_MODEXP: {
1726                 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q;
1727                 u_int rlen, clen;
1728
1729                 krp = me->me_krp;
1730                 rlen = (me->me_modbits + 7) / 8;
1731                 clen = (krp->krp_param[krp->krp_iparams].crp_nbits + 7) / 8;
1732
1733                 ubsec_dma_sync(&me->me_M, BUS_DMASYNC_POSTWRITE);
1734                 ubsec_dma_sync(&me->me_E, BUS_DMASYNC_POSTWRITE);
1735                 ubsec_dma_sync(&me->me_C, BUS_DMASYNC_POSTREAD);
1736                 ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_POSTWRITE);
1737
1738                 if (clen < rlen)
1739                         krp->krp_status = E2BIG;
1740                 else {
1741                         if (sc->sc_flags & UBS_FLAGS_HWNORM) {
1742                                 bzero(krp->krp_param[krp->krp_iparams].crp_p,
1743                                     (krp->krp_param[krp->krp_iparams].crp_nbits
1744                                         + 7) / 8);
1745                                 bcopy(me->me_C.dma_vaddr,
1746                                     krp->krp_param[krp->krp_iparams].crp_p,
1747                                     (me->me_modbits + 7) / 8);
1748                         } else
1749                                 ubsec_kshift_l(me->me_shiftbits,
1750                                     me->me_C.dma_vaddr, me->me_normbits,
1751                                     krp->krp_param[krp->krp_iparams].crp_p,
1752                                     krp->krp_param[krp->krp_iparams].crp_nbits);
1753                 }
1754
1755                 crypto_kdone(krp);
1756
1757                 /* bzero all potentially sensitive data */
1758                 bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
1759                 bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
1760                 bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
1761                 bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
1762
1763                 /* Can't free here, so put us on the free list. */
1764                 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next);
1765                 break;
1766         }
1767         case UBS_CTXOP_RSAPRIV: {
1768                 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q;
1769                 u_int len;
1770
1771                 krp = rp->rpr_krp;
1772                 ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_POSTWRITE);
1773                 ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_POSTREAD);
1774
1775                 len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + 7) / 8;
1776                 bcopy(rp->rpr_msgout.dma_vaddr,
1777                     krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len);
1778
1779                 crypto_kdone(krp);
1780
1781                 bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size);
1782                 bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size);
1783                 bzero(rp->rpr_q.q_ctx.dma_vaddr, rp->rpr_q.q_ctx.dma_size);
1784
1785                 /* Can't free here, so put us on the free list. */
1786                 SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next);
1787                 break;
1788         }
1789         default:
1790                 device_printf(sc->sc_dev, "unknown ctx op: %x\n",
1791                     letoh16(ctx->ctx_op));
1792                 break;
1793         }
1794 }
1795
1796 #ifndef UBSEC_NO_RNG
1797 static void
1798 ubsec_rng(void *vsc)
1799 {
1800         struct ubsec_softc *sc = vsc;
1801         struct ubsec_q2_rng *rng = &sc->sc_rng;
1802         struct ubsec_mcr *mcr;
1803         struct ubsec_ctx_rngbypass *ctx;
1804
1805         mtx_lock(&sc->sc_mcr2lock);
1806         if (rng->rng_used) {
1807                 mtx_unlock(&sc->sc_mcr2lock);
1808                 return;
1809         }
1810         sc->sc_nqueue2++;
1811         if (sc->sc_nqueue2 >= UBS_MAX_NQUEUE)
1812                 goto out;
1813
1814         mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr;
1815         ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr;
1816
1817         mcr->mcr_pkts = htole16(1);
1818         mcr->mcr_flags = 0;
1819         mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr);
1820         mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = 0;
1821         mcr->mcr_ipktbuf.pb_len = 0;
1822         mcr->mcr_reserved = mcr->mcr_pktlen = 0;
1823         mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr);
1824         mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) &
1825             UBS_PKTBUF_LEN);
1826         mcr->mcr_opktbuf.pb_next = 0;
1827
1828         ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass));
1829         ctx->rbp_op = htole16(UBS_CTXOP_RNGBYPASS);
1830         rng->rng_q.q_type = UBS_CTXOP_RNGBYPASS;
1831
1832         ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_PREREAD);
1833
1834         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next);
1835         rng->rng_used = 1;
1836         ubsec_feed2(sc);
1837         ubsecstats.hst_rng++;
1838         mtx_unlock(&sc->sc_mcr2lock);
1839
1840         return;
1841
1842 out:
1843         /*
1844          * Something weird happened, generate our own call back.
1845          */
1846         sc->sc_nqueue2--;
1847         mtx_unlock(&sc->sc_mcr2lock);
1848         callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
1849 }
1850 #endif /* UBSEC_NO_RNG */
1851
1852 static void
1853 ubsec_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
1854 {
1855         bus_addr_t *paddr = (bus_addr_t*) arg;
1856         *paddr = segs->ds_addr;
1857 }
1858
1859 static int
1860 ubsec_dma_malloc(
1861         struct ubsec_softc *sc,
1862         bus_size_t size,
1863         struct ubsec_dma_alloc *dma,
1864         int mapflags
1865 )
1866 {
1867         int r;
1868
1869         /* XXX could specify sc_dmat as parent but that just adds overhead */
1870         r = bus_dma_tag_create(NULL,                    /* parent */
1871                                1, 0,                    /* alignment, bounds */
1872                                BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
1873                                BUS_SPACE_MAXADDR,       /* highaddr */
1874                                NULL, NULL,              /* filter, filterarg */
1875                                size,                    /* maxsize */
1876                                1,                       /* nsegments */
1877                                size,                    /* maxsegsize */
1878                                BUS_DMA_ALLOCNOW,        /* flags */
1879                                NULL, NULL,              /* lockfunc, lockarg */
1880                                &dma->dma_tag);
1881         if (r != 0) {
1882                 device_printf(sc->sc_dev, "ubsec_dma_malloc: "
1883                         "bus_dma_tag_create failed; error %u\n", r);
1884                 goto fail_0;
1885         }
1886
1887         r = bus_dmamap_create(dma->dma_tag, BUS_DMA_NOWAIT, &dma->dma_map);
1888         if (r != 0) {
1889                 device_printf(sc->sc_dev, "ubsec_dma_malloc: "
1890                         "bus_dmamap_create failed; error %u\n", r);
1891                 goto fail_1;
1892         }
1893
1894         r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
1895                              BUS_DMA_NOWAIT, &dma->dma_map);
1896         if (r != 0) {
1897                 device_printf(sc->sc_dev, "ubsec_dma_malloc: "
1898                         "bus_dmammem_alloc failed; size %zu, error %u\n",
1899                         size, r);
1900                 goto fail_2;
1901         }
1902
1903         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
1904                             size,
1905                             ubsec_dmamap_cb,
1906                             &dma->dma_paddr,
1907                             mapflags | BUS_DMA_NOWAIT);
1908         if (r != 0) {
1909                 device_printf(sc->sc_dev, "ubsec_dma_malloc: "
1910                         "bus_dmamap_load failed; error %u\n", r);
1911                 goto fail_3;
1912         }
1913
1914         dma->dma_size = size;
1915         return (0);
1916
1917 fail_3:
1918         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
1919 fail_2:
1920         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1921 fail_1:
1922         bus_dmamap_destroy(dma->dma_tag, dma->dma_map);
1923         bus_dma_tag_destroy(dma->dma_tag);
1924 fail_0:
1925         dma->dma_map = NULL;
1926         dma->dma_tag = NULL;
1927         return (r);
1928 }
1929
1930 static void
1931 ubsec_dma_free(struct ubsec_softc *sc, struct ubsec_dma_alloc *dma)
1932 {
1933         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
1934         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1935         bus_dmamap_destroy(dma->dma_tag, dma->dma_map);
1936         bus_dma_tag_destroy(dma->dma_tag);
1937 }
1938
1939 /*
1940  * Resets the board.  Values in the regesters are left as is
1941  * from the reset (i.e. initial values are assigned elsewhere).
1942  */
1943 static void
1944 ubsec_reset_board(struct ubsec_softc *sc)
1945 {
1946     volatile u_int32_t ctrl;
1947
1948     ctrl = READ_REG(sc, BS_CTRL);
1949     ctrl |= BS_CTRL_RESET;
1950     WRITE_REG(sc, BS_CTRL, ctrl);
1951
1952     /*
1953      * Wait aprox. 30 PCI clocks = 900 ns = 0.9 us
1954      */
1955     DELAY(10);
1956 }
1957
1958 /*
1959  * Init Broadcom registers
1960  */
1961 static void
1962 ubsec_init_board(struct ubsec_softc *sc)
1963 {
1964         u_int32_t ctrl;
1965
1966         ctrl = READ_REG(sc, BS_CTRL);
1967         ctrl &= ~(BS_CTRL_BE32 | BS_CTRL_BE64);
1968         ctrl |= BS_CTRL_LITTLE_ENDIAN | BS_CTRL_MCR1INT;
1969
1970         if (sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG))
1971                 ctrl |= BS_CTRL_MCR2INT;
1972         else
1973                 ctrl &= ~BS_CTRL_MCR2INT;
1974
1975         if (sc->sc_flags & UBS_FLAGS_HWNORM)
1976                 ctrl &= ~BS_CTRL_SWNORM;
1977
1978         WRITE_REG(sc, BS_CTRL, ctrl);
1979 }
1980
1981 /*
1982  * Init Broadcom PCI registers
1983  */
1984 static void
1985 ubsec_init_pciregs(device_t dev)
1986 {
1987 #if 0
1988         u_int32_t misc;
1989
1990         misc = pci_conf_read(pc, pa->pa_tag, BS_RTY_TOUT);
1991         misc = (misc & ~(UBS_PCI_RTY_MASK << UBS_PCI_RTY_SHIFT))
1992             | ((UBS_DEF_RTY & 0xff) << UBS_PCI_RTY_SHIFT);
1993         misc = (misc & ~(UBS_PCI_TOUT_MASK << UBS_PCI_TOUT_SHIFT))
1994             | ((UBS_DEF_TOUT & 0xff) << UBS_PCI_TOUT_SHIFT);
1995         pci_conf_write(pc, pa->pa_tag, BS_RTY_TOUT, misc);
1996 #endif
1997
1998         /*
1999          * This will set the cache line size to 1, this will
2000          * force the BCM58xx chip just to do burst read/writes.
2001          * Cache line read/writes are to slow
2002          */
2003         pci_write_config(dev, PCIR_CACHELNSZ, UBS_DEF_CACHELINE, 1);
2004 }
2005
2006 /*
2007  * Clean up after a chip crash.
2008  * It is assumed that the caller in splimp()
2009  */
2010 static void
2011 ubsec_cleanchip(struct ubsec_softc *sc)
2012 {
2013         struct ubsec_q *q;
2014
2015         while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
2016                 q = SIMPLEQ_FIRST(&sc->sc_qchip);
2017                 SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q, q_next);
2018                 ubsec_free_q(sc, q);
2019         }
2020         sc->sc_nqchip = 0;
2021 }
2022
2023 /*
2024  * free a ubsec_q
2025  * It is assumed that the caller is within splimp().
2026  */
2027 static int
2028 ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q)
2029 {
2030         struct ubsec_q *q2;
2031         struct cryptop *crp;
2032         int npkts;
2033         int i;
2034
2035         npkts = q->q_nstacked_mcrs;
2036
2037         for (i = 0; i < npkts; i++) {
2038                 if(q->q_stacked_mcr[i]) {
2039                         q2 = q->q_stacked_mcr[i];
2040
2041                         if ((q2->q_dst_m != NULL) && (q2->q_src_m != q2->q_dst_m))
2042                                 m_freem(q2->q_dst_m);
2043
2044                         crp = (struct cryptop *)q2->q_crp;
2045
2046                         SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next);
2047
2048                         crp->crp_etype = EFAULT;
2049                         crypto_done(crp);
2050                 } else {
2051                         break;
2052                 }
2053         }
2054
2055         /*
2056          * Free header MCR
2057          */
2058         if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
2059                 m_freem(q->q_dst_m);
2060
2061         crp = (struct cryptop *)q->q_crp;
2062
2063         SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
2064
2065         crp->crp_etype = EFAULT;
2066         crypto_done(crp);
2067         return(0);
2068 }
2069
2070 /*
2071  * Routine to reset the chip and clean up.
2072  * It is assumed that the caller is in splimp()
2073  */
2074 static void
2075 ubsec_totalreset(struct ubsec_softc *sc)
2076 {
2077         ubsec_reset_board(sc);
2078         ubsec_init_board(sc);
2079         ubsec_cleanchip(sc);
2080 }
2081
2082 static int
2083 ubsec_dmamap_aligned(struct ubsec_operand *op)
2084 {
2085         int i;
2086
2087         for (i = 0; i < op->nsegs; i++) {
2088                 if (op->segs[i].ds_addr & 3)
2089                         return (0);
2090                 if ((i != (op->nsegs - 1)) &&
2091                     (op->segs[i].ds_len & 3))
2092                         return (0);
2093         }
2094         return (1);
2095 }
2096
2097 static void
2098 ubsec_kfree(struct ubsec_softc *sc, struct ubsec_q2 *q)
2099 {
2100         switch (q->q_type) {
2101         case UBS_CTXOP_MODEXP: {
2102                 struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q;
2103
2104                 ubsec_dma_free(sc, &me->me_q.q_mcr);
2105                 ubsec_dma_free(sc, &me->me_q.q_ctx);
2106                 ubsec_dma_free(sc, &me->me_M);
2107                 ubsec_dma_free(sc, &me->me_E);
2108                 ubsec_dma_free(sc, &me->me_C);
2109                 ubsec_dma_free(sc, &me->me_epb);
2110                 free(me, M_DEVBUF);
2111                 break;
2112         }
2113         case UBS_CTXOP_RSAPRIV: {
2114                 struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q;
2115
2116                 ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
2117                 ubsec_dma_free(sc, &rp->rpr_q.q_ctx);
2118                 ubsec_dma_free(sc, &rp->rpr_msgin);
2119                 ubsec_dma_free(sc, &rp->rpr_msgout);
2120                 free(rp, M_DEVBUF);
2121                 break;
2122         }
2123         default:
2124                 device_printf(sc->sc_dev, "invalid kfree 0x%x\n", q->q_type);
2125                 break;
2126         }
2127 }
2128
2129 static int
2130 ubsec_kprocess(void *arg, struct cryptkop *krp, int hint)
2131 {
2132         struct ubsec_softc *sc = arg;
2133         int r;
2134
2135         if (krp == NULL || krp->krp_callback == NULL)
2136                 return (EINVAL);
2137
2138         while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) {
2139                 struct ubsec_q2 *q;
2140
2141                 q = SIMPLEQ_FIRST(&sc->sc_q2free);
2142                 SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, q, q_next);
2143                 ubsec_kfree(sc, q);
2144         }
2145
2146         switch (krp->krp_op) {
2147         case CRK_MOD_EXP:
2148                 if (sc->sc_flags & UBS_FLAGS_HWNORM)
2149                         r = ubsec_kprocess_modexp_hw(sc, krp, hint);
2150                 else
2151                         r = ubsec_kprocess_modexp_sw(sc, krp, hint);
2152                 break;
2153         case CRK_MOD_EXP_CRT:
2154                 return (ubsec_kprocess_rsapriv(sc, krp, hint));
2155         default:
2156                 device_printf(sc->sc_dev, "kprocess: invalid op 0x%x\n",
2157                     krp->krp_op);
2158                 krp->krp_status = EOPNOTSUPP;
2159                 crypto_kdone(krp);
2160                 return (0);
2161         }
2162         return (0);                     /* silence compiler */
2163 }
2164
2165 /*
2166  * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization)
2167  */
2168 static int
2169 ubsec_kprocess_modexp_sw(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
2170 {
2171         struct ubsec_q2_modexp *me;
2172         struct ubsec_mcr *mcr;
2173         struct ubsec_ctx_modexp *ctx;
2174         struct ubsec_pktbuf *epb;
2175         int err = 0;
2176         u_int nbits, normbits, mbits, shiftbits, ebits;
2177
2178         me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT);
2179         if (me == NULL) {
2180                 err = ENOMEM;
2181                 goto errout;
2182         }
2183         bzero(me, sizeof *me);
2184         me->me_krp = krp;
2185         me->me_q.q_type = UBS_CTXOP_MODEXP;
2186
2187         nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
2188         if (nbits <= 512)
2189                 normbits = 512;
2190         else if (nbits <= 768)
2191                 normbits = 768;
2192         else if (nbits <= 1024)
2193                 normbits = 1024;
2194         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536)
2195                 normbits = 1536;
2196         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048)
2197                 normbits = 2048;
2198         else {
2199                 err = E2BIG;
2200                 goto errout;
2201         }
2202
2203         shiftbits = normbits - nbits;
2204
2205         me->me_modbits = nbits;
2206         me->me_shiftbits = shiftbits;
2207         me->me_normbits = normbits;
2208
2209         /* Sanity check: result bits must be >= true modulus bits. */
2210         if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
2211                 err = ERANGE;
2212                 goto errout;
2213         }
2214
2215         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2216             &me->me_q.q_mcr, 0)) {
2217                 err = ENOMEM;
2218                 goto errout;
2219         }
2220         mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
2221
2222         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
2223             &me->me_q.q_ctx, 0)) {
2224                 err = ENOMEM;
2225                 goto errout;
2226         }
2227
2228         mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
2229         if (mbits > nbits) {
2230                 err = E2BIG;
2231                 goto errout;
2232         }
2233         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) {
2234                 err = ENOMEM;
2235                 goto errout;
2236         }
2237         ubsec_kshift_r(shiftbits,
2238             krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits,
2239             me->me_M.dma_vaddr, normbits);
2240
2241         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) {
2242                 err = ENOMEM;
2243                 goto errout;
2244         }
2245         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
2246
2247         ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
2248         if (ebits > nbits) {
2249                 err = E2BIG;
2250                 goto errout;
2251         }
2252         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) {
2253                 err = ENOMEM;
2254                 goto errout;
2255         }
2256         ubsec_kshift_r(shiftbits,
2257             krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits,
2258             me->me_E.dma_vaddr, normbits);
2259
2260         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
2261             &me->me_epb, 0)) {
2262                 err = ENOMEM;
2263                 goto errout;
2264         }
2265         epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
2266         epb->pb_addr = htole32(me->me_E.dma_paddr);
2267         epb->pb_next = 0;
2268         epb->pb_len = htole32(normbits / 8);
2269
2270 #ifdef UBSEC_DEBUG
2271         if (ubsec_debug) {
2272                 printf("Epb ");
2273                 ubsec_dump_pb(epb);
2274         }
2275 #endif
2276
2277         mcr->mcr_pkts = htole16(1);
2278         mcr->mcr_flags = 0;
2279         mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
2280         mcr->mcr_reserved = 0;
2281         mcr->mcr_pktlen = 0;
2282
2283         mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
2284         mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8);
2285         mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
2286
2287         mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
2288         mcr->mcr_opktbuf.pb_next = 0;
2289         mcr->mcr_opktbuf.pb_len = htole32(normbits / 8);
2290
2291 #ifdef DIAGNOSTIC
2292         /* Misaligned output buffer will hang the chip. */
2293         if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0)
2294                 panic("%s: modexp invalid addr 0x%x\n",
2295                     device_get_nameunit(sc->sc_dev),
2296                     letoh32(mcr->mcr_opktbuf.pb_addr));
2297         if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0)
2298                 panic("%s: modexp invalid len 0x%x\n",
2299                     device_get_nameunit(sc->sc_dev),
2300                     letoh32(mcr->mcr_opktbuf.pb_len));
2301 #endif
2302
2303         ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
2304         bzero(ctx, sizeof(*ctx));
2305         ubsec_kshift_r(shiftbits,
2306             krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits,
2307             ctx->me_N, normbits);
2308         ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t)));
2309         ctx->me_op = htole16(UBS_CTXOP_MODEXP);
2310         ctx->me_E_len = htole16(nbits);
2311         ctx->me_N_len = htole16(nbits);
2312
2313 #ifdef UBSEC_DEBUG
2314         if (ubsec_debug) {
2315                 ubsec_dump_mcr(mcr);
2316                 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
2317         }
2318 #endif
2319
2320         /*
2321          * ubsec_feed2 will sync mcr and ctx, we just need to sync
2322          * everything else.
2323          */
2324         ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE);
2325         ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE);
2326         ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD);
2327         ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE);
2328
2329         /* Enqueue and we're done... */
2330         mtx_lock(&sc->sc_mcr2lock);
2331         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
2332         ubsec_feed2(sc);
2333         ubsecstats.hst_modexp++;
2334         mtx_unlock(&sc->sc_mcr2lock);
2335
2336         return (0);
2337
2338 errout:
2339         if (me != NULL) {
2340                 if (me->me_q.q_mcr.dma_map != NULL)
2341                         ubsec_dma_free(sc, &me->me_q.q_mcr);
2342                 if (me->me_q.q_ctx.dma_map != NULL) {
2343                         bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
2344                         ubsec_dma_free(sc, &me->me_q.q_ctx);
2345                 }
2346                 if (me->me_M.dma_map != NULL) {
2347                         bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
2348                         ubsec_dma_free(sc, &me->me_M);
2349                 }
2350                 if (me->me_E.dma_map != NULL) {
2351                         bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
2352                         ubsec_dma_free(sc, &me->me_E);
2353                 }
2354                 if (me->me_C.dma_map != NULL) {
2355                         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
2356                         ubsec_dma_free(sc, &me->me_C);
2357                 }
2358                 if (me->me_epb.dma_map != NULL)
2359                         ubsec_dma_free(sc, &me->me_epb);
2360                 free(me, M_DEVBUF);
2361         }
2362         krp->krp_status = err;
2363         crypto_kdone(krp);
2364         return (0);
2365 }
2366
2367 /*
2368  * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization)
2369  */
2370 static int
2371 ubsec_kprocess_modexp_hw(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
2372 {
2373         struct ubsec_q2_modexp *me;
2374         struct ubsec_mcr *mcr;
2375         struct ubsec_ctx_modexp *ctx;
2376         struct ubsec_pktbuf *epb;
2377         int err = 0;
2378         u_int nbits, normbits, mbits, shiftbits, ebits;
2379
2380         me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT);
2381         if (me == NULL) {
2382                 err = ENOMEM;
2383                 goto errout;
2384         }
2385         bzero(me, sizeof *me);
2386         me->me_krp = krp;
2387         me->me_q.q_type = UBS_CTXOP_MODEXP;
2388
2389         nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
2390         if (nbits <= 512)
2391                 normbits = 512;
2392         else if (nbits <= 768)
2393                 normbits = 768;
2394         else if (nbits <= 1024)
2395                 normbits = 1024;
2396         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536)
2397                 normbits = 1536;
2398         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048)
2399                 normbits = 2048;
2400         else {
2401                 err = E2BIG;
2402                 goto errout;
2403         }
2404
2405         shiftbits = normbits - nbits;
2406
2407         /* XXX ??? */
2408         me->me_modbits = nbits;
2409         me->me_shiftbits = shiftbits;
2410         me->me_normbits = normbits;
2411
2412         /* Sanity check: result bits must be >= true modulus bits. */
2413         if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
2414                 err = ERANGE;
2415                 goto errout;
2416         }
2417
2418         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2419             &me->me_q.q_mcr, 0)) {
2420                 err = ENOMEM;
2421                 goto errout;
2422         }
2423         mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
2424
2425         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
2426             &me->me_q.q_ctx, 0)) {
2427                 err = ENOMEM;
2428                 goto errout;
2429         }
2430
2431         mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
2432         if (mbits > nbits) {
2433                 err = E2BIG;
2434                 goto errout;
2435         }
2436         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) {
2437                 err = ENOMEM;
2438                 goto errout;
2439         }
2440         bzero(me->me_M.dma_vaddr, normbits / 8);
2441         bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p,
2442             me->me_M.dma_vaddr, (mbits + 7) / 8);
2443
2444         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) {
2445                 err = ENOMEM;
2446                 goto errout;
2447         }
2448         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
2449
2450         ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
2451         if (ebits > nbits) {
2452                 err = E2BIG;
2453                 goto errout;
2454         }
2455         if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) {
2456                 err = ENOMEM;
2457                 goto errout;
2458         }
2459         bzero(me->me_E.dma_vaddr, normbits / 8);
2460         bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p,
2461             me->me_E.dma_vaddr, (ebits + 7) / 8);
2462
2463         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
2464             &me->me_epb, 0)) {
2465                 err = ENOMEM;
2466                 goto errout;
2467         }
2468         epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
2469         epb->pb_addr = htole32(me->me_E.dma_paddr);
2470         epb->pb_next = 0;
2471         epb->pb_len = htole32((ebits + 7) / 8);
2472
2473 #ifdef UBSEC_DEBUG
2474         if (ubsec_debug) {
2475                 printf("Epb ");
2476                 ubsec_dump_pb(epb);
2477         }
2478 #endif
2479
2480         mcr->mcr_pkts = htole16(1);
2481         mcr->mcr_flags = 0;
2482         mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
2483         mcr->mcr_reserved = 0;
2484         mcr->mcr_pktlen = 0;
2485
2486         mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
2487         mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8);
2488         mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
2489
2490         mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
2491         mcr->mcr_opktbuf.pb_next = 0;
2492         mcr->mcr_opktbuf.pb_len = htole32(normbits / 8);
2493
2494 #ifdef DIAGNOSTIC
2495         /* Misaligned output buffer will hang the chip. */
2496         if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0)
2497                 panic("%s: modexp invalid addr 0x%x\n",
2498                     device_get_nameunit(sc->sc_dev),
2499                     letoh32(mcr->mcr_opktbuf.pb_addr));
2500         if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0)
2501                 panic("%s: modexp invalid len 0x%x\n",
2502                     device_get_nameunit(sc->sc_dev),
2503                     letoh32(mcr->mcr_opktbuf.pb_len));
2504 #endif
2505
2506         ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
2507         bzero(ctx, sizeof(*ctx));
2508         bcopy(krp->krp_param[UBS_MODEXP_PAR_N].crp_p, ctx->me_N,
2509             (nbits + 7) / 8);
2510         ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t)));
2511         ctx->me_op = htole16(UBS_CTXOP_MODEXP);
2512         ctx->me_E_len = htole16(ebits);
2513         ctx->me_N_len = htole16(nbits);
2514
2515 #ifdef UBSEC_DEBUG
2516         if (ubsec_debug) {
2517                 ubsec_dump_mcr(mcr);
2518                 ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
2519         }
2520 #endif
2521
2522         /*
2523          * ubsec_feed2 will sync mcr and ctx, we just need to sync
2524          * everything else.
2525          */
2526         ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE);
2527         ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE);
2528         ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD);
2529         ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE);
2530
2531         /* Enqueue and we're done... */
2532         mtx_lock(&sc->sc_mcr2lock);
2533         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
2534         ubsec_feed2(sc);
2535         mtx_unlock(&sc->sc_mcr2lock);
2536
2537         return (0);
2538
2539 errout:
2540         if (me != NULL) {
2541                 if (me->me_q.q_mcr.dma_map != NULL)
2542                         ubsec_dma_free(sc, &me->me_q.q_mcr);
2543                 if (me->me_q.q_ctx.dma_map != NULL) {
2544                         bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
2545                         ubsec_dma_free(sc, &me->me_q.q_ctx);
2546                 }
2547                 if (me->me_M.dma_map != NULL) {
2548                         bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
2549                         ubsec_dma_free(sc, &me->me_M);
2550                 }
2551                 if (me->me_E.dma_map != NULL) {
2552                         bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
2553                         ubsec_dma_free(sc, &me->me_E);
2554                 }
2555                 if (me->me_C.dma_map != NULL) {
2556                         bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
2557                         ubsec_dma_free(sc, &me->me_C);
2558                 }
2559                 if (me->me_epb.dma_map != NULL)
2560                         ubsec_dma_free(sc, &me->me_epb);
2561                 free(me, M_DEVBUF);
2562         }
2563         krp->krp_status = err;
2564         crypto_kdone(krp);
2565         return (0);
2566 }
2567
2568 static int
2569 ubsec_kprocess_rsapriv(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
2570 {
2571         struct ubsec_q2_rsapriv *rp = NULL;
2572         struct ubsec_mcr *mcr;
2573         struct ubsec_ctx_rsapriv *ctx;
2574         int err = 0;
2575         u_int padlen, msglen;
2576
2577         msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]);
2578         padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]);
2579         if (msglen > padlen)
2580                 padlen = msglen;
2581
2582         if (padlen <= 256)
2583                 padlen = 256;
2584         else if (padlen <= 384)
2585                 padlen = 384;
2586         else if (padlen <= 512)
2587                 padlen = 512;
2588         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 768)
2589                 padlen = 768;
2590         else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 1024)
2591                 padlen = 1024;
2592         else {
2593                 err = E2BIG;
2594                 goto errout;
2595         }
2596
2597         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) {
2598                 err = E2BIG;
2599                 goto errout;
2600         }
2601
2602         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) {
2603                 err = E2BIG;
2604                 goto errout;
2605         }
2606
2607         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) {
2608                 err = E2BIG;
2609                 goto errout;
2610         }
2611
2612         rp = (struct ubsec_q2_rsapriv *)malloc(sizeof *rp, M_DEVBUF, M_NOWAIT);
2613         if (rp == NULL)
2614                 return (ENOMEM);
2615         bzero(rp, sizeof *rp);
2616         rp->rpr_krp = krp;
2617         rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV;
2618
2619         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
2620             &rp->rpr_q.q_mcr, 0)) {
2621                 err = ENOMEM;
2622                 goto errout;
2623         }
2624         mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr;
2625
2626         if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv),
2627             &rp->rpr_q.q_ctx, 0)) {
2628                 err = ENOMEM;
2629                 goto errout;
2630         }
2631         ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr;
2632         bzero(ctx, sizeof *ctx);
2633
2634         /* Copy in p */
2635         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p,
2636             &ctx->rpr_buf[0 * (padlen / 8)],
2637             (krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + 7) / 8);
2638
2639         /* Copy in q */
2640         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p,
2641             &ctx->rpr_buf[1 * (padlen / 8)],
2642             (krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + 7) / 8);
2643
2644         /* Copy in dp */
2645         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p,
2646             &ctx->rpr_buf[2 * (padlen / 8)],
2647             (krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + 7) / 8);
2648
2649         /* Copy in dq */
2650         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p,
2651             &ctx->rpr_buf[3 * (padlen / 8)],
2652             (krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + 7) / 8);
2653
2654         /* Copy in pinv */
2655         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p,
2656             &ctx->rpr_buf[4 * (padlen / 8)],
2657             (krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + 7) / 8);
2658
2659         msglen = padlen * 2;
2660
2661         /* Copy in input message (aligned buffer/length). */
2662         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) {
2663                 /* Is this likely? */
2664                 err = E2BIG;
2665                 goto errout;
2666         }
2667         if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgin, 0)) {
2668                 err = ENOMEM;
2669                 goto errout;
2670         }
2671         bzero(rp->rpr_msgin.dma_vaddr, (msglen + 7) / 8);
2672         bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p,
2673             rp->rpr_msgin.dma_vaddr,
2674             (krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + 7) / 8);
2675
2676         /* Prepare space for output message (aligned buffer/length). */
2677         if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) {
2678                 /* Is this likely? */
2679                 err = E2BIG;
2680                 goto errout;
2681         }
2682         if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgout, 0)) {
2683                 err = ENOMEM;
2684                 goto errout;
2685         }
2686         bzero(rp->rpr_msgout.dma_vaddr, (msglen + 7) / 8);
2687
2688         mcr->mcr_pkts = htole16(1);
2689         mcr->mcr_flags = 0;
2690         mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr);
2691         mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr);
2692         mcr->mcr_ipktbuf.pb_next = 0;
2693         mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size);
2694         mcr->mcr_reserved = 0;
2695         mcr->mcr_pktlen = htole16(msglen);
2696         mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr);
2697         mcr->mcr_opktbuf.pb_next = 0;
2698         mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size);
2699
2700 #ifdef DIAGNOSTIC
2701         if (rp->rpr_msgin.dma_paddr & 3 || rp->rpr_msgin.dma_size & 3) {
2702                 panic("%s: rsapriv: invalid msgin %x(0x%jx)",
2703                     device_get_nameunit(sc->sc_dev),
2704                     rp->rpr_msgin.dma_paddr, (uintmax_t)rp->rpr_msgin.dma_size);
2705         }
2706         if (rp->rpr_msgout.dma_paddr & 3 || rp->rpr_msgout.dma_size & 3) {
2707                 panic("%s: rsapriv: invalid msgout %x(0x%jx)",
2708                     device_get_nameunit(sc->sc_dev),
2709                     rp->rpr_msgout.dma_paddr, (uintmax_t)rp->rpr_msgout.dma_size);
2710         }
2711 #endif
2712
2713         ctx->rpr_len = (sizeof(u_int16_t) * 4) + (5 * (padlen / 8));
2714         ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV);
2715         ctx->rpr_q_len = htole16(padlen);
2716         ctx->rpr_p_len = htole16(padlen);
2717
2718         /*
2719          * ubsec_feed2 will sync mcr and ctx, we just need to sync
2720          * everything else.
2721          */
2722         ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_PREWRITE);
2723         ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_PREREAD);
2724
2725         /* Enqueue and we're done... */
2726         mtx_lock(&sc->sc_mcr2lock);
2727         SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next);
2728         ubsec_feed2(sc);
2729         ubsecstats.hst_modexpcrt++;
2730         mtx_unlock(&sc->sc_mcr2lock);
2731         return (0);
2732
2733 errout:
2734         if (rp != NULL) {
2735                 if (rp->rpr_q.q_mcr.dma_map != NULL)
2736                         ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
2737                 if (rp->rpr_msgin.dma_map != NULL) {
2738                         bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size);
2739                         ubsec_dma_free(sc, &rp->rpr_msgin);
2740                 }
2741                 if (rp->rpr_msgout.dma_map != NULL) {
2742                         bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size);
2743                         ubsec_dma_free(sc, &rp->rpr_msgout);
2744                 }
2745                 free(rp, M_DEVBUF);
2746         }
2747         krp->krp_status = err;
2748         crypto_kdone(krp);
2749         return (0);
2750 }
2751
2752 #ifdef UBSEC_DEBUG
2753 static void
2754 ubsec_dump_pb(volatile struct ubsec_pktbuf *pb)
2755 {
2756         printf("addr 0x%x (0x%x) next 0x%x\n",
2757             pb->pb_addr, pb->pb_len, pb->pb_next);
2758 }
2759
2760 static void
2761 ubsec_dump_ctx2(struct ubsec_ctx_keyop *c)
2762 {
2763         printf("CTX (0x%x):\n", c->ctx_len);
2764         switch (letoh16(c->ctx_op)) {
2765         case UBS_CTXOP_RNGBYPASS:
2766         case UBS_CTXOP_RNGSHA1:
2767                 break;
2768         case UBS_CTXOP_MODEXP:
2769         {
2770                 struct ubsec_ctx_modexp *cx = (void *)c;
2771                 int i, len;
2772
2773                 printf(" Elen %u, Nlen %u\n",
2774                     letoh16(cx->me_E_len), letoh16(cx->me_N_len));
2775                 len = (cx->me_N_len + 7)/8;
2776                 for (i = 0; i < len; i++)
2777                         printf("%s%02x", (i == 0) ? " N: " : ":", cx->me_N[i]);
2778                 printf("\n");
2779                 break;
2780         }
2781         default:
2782                 printf("unknown context: %x\n", c->ctx_op);
2783         }
2784         printf("END CTX\n");
2785 }
2786
2787 static void
2788 ubsec_dump_mcr(struct ubsec_mcr *mcr)
2789 {
2790         volatile struct ubsec_mcr_add *ma;
2791         int i;
2792
2793         printf("MCR:\n");
2794         printf(" pkts: %u, flags 0x%x\n",
2795             letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags));
2796         ma = (volatile struct ubsec_mcr_add *)&mcr->mcr_cmdctxp;
2797         for (i = 0; i < letoh16(mcr->mcr_pkts); i++) {
2798                 printf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i,
2799                     letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen),
2800                     letoh16(ma->mcr_reserved));
2801                 printf(" %d: ipkt ", i);
2802                 ubsec_dump_pb(&ma->mcr_ipktbuf);
2803                 printf(" %d: opkt ", i);
2804                 ubsec_dump_pb(&ma->mcr_opktbuf);
2805                 ma++;
2806         }
2807         printf("END MCR\n");
2808 }
2809 #endif /* UBSEC_DEBUG */
2810
2811 /*
2812  * Return the number of significant bits of a big number.
2813  */
2814 static int
2815 ubsec_ksigbits(struct crparam *cr)
2816 {
2817         u_int plen = (cr->crp_nbits + 7) / 8;
2818         int i, sig = plen * 8;
2819         u_int8_t c, *p = cr->crp_p;
2820
2821         for (i = plen - 1; i >= 0; i--) {
2822                 c = p[i];
2823                 if (c != 0) {
2824                         while ((c & 0x80) == 0) {
2825                                 sig--;
2826                                 c <<= 1;
2827                         }
2828                         break;
2829                 }
2830                 sig -= 8;
2831         }
2832         return (sig);
2833 }
2834
2835 static void
2836 ubsec_kshift_r(
2837         u_int shiftbits,
2838         u_int8_t *src, u_int srcbits,
2839         u_int8_t *dst, u_int dstbits)
2840 {
2841         u_int slen, dlen;
2842         int i, si, di, n;
2843
2844         slen = (srcbits + 7) / 8;
2845         dlen = (dstbits + 7) / 8;
2846
2847         for (i = 0; i < slen; i++)
2848                 dst[i] = src[i];
2849         for (i = 0; i < dlen - slen; i++)
2850                 dst[slen + i] = 0;
2851
2852         n = shiftbits / 8;
2853         if (n != 0) {
2854                 si = dlen - n - 1;
2855                 di = dlen - 1;
2856                 while (si >= 0)
2857                         dst[di--] = dst[si--];
2858                 while (di >= 0)
2859                         dst[di--] = 0;
2860         }
2861
2862         n = shiftbits % 8;
2863         if (n != 0) {
2864                 for (i = dlen - 1; i > 0; i--)
2865                         dst[i] = (dst[i] << n) |
2866                             (dst[i - 1] >> (8 - n));
2867                 dst[0] = dst[0] << n;
2868         }
2869 }
2870
2871 static void
2872 ubsec_kshift_l(
2873         u_int shiftbits,
2874         u_int8_t *src, u_int srcbits,
2875         u_int8_t *dst, u_int dstbits)
2876 {
2877         int slen, dlen, i, n;
2878
2879         slen = (srcbits + 7) / 8;
2880         dlen = (dstbits + 7) / 8;
2881
2882         n = shiftbits / 8;
2883         for (i = 0; i < slen; i++)
2884                 dst[i] = src[i + n];
2885         for (i = 0; i < dlen - slen; i++)
2886                 dst[slen + i] = 0;
2887
2888         n = shiftbits % 8;
2889         if (n != 0) {
2890                 for (i = 0; i < (dlen - 1); i++)
2891                         dst[i] = (dst[i] >> n) | (dst[i + 1] << (8 - n));
2892                 dst[dlen - 1] = dst[dlen - 1] >> n;
2893         }
2894 }