sys/dev/safe/safe.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2003 Sam Leffler, Errno Consulting
   5  * Copyright (c) 2003 Global Technology Associates, Inc.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  */
  29
  30 #include <sys/cdefs.h>
  31 __FBSDID("$FreeBSD$");
  32
  33 /*
  34  * SafeNet SafeXcel-1141 hardware crypto accelerator
  35  */
  36 #include "opt_safe.h"
  37
  38 #include <sys/param.h>
  39 #include <sys/systm.h>
  40 #include <sys/proc.h>
  41 #include <sys/errno.h>
  42 #include <sys/malloc.h>
  43 #include <sys/kernel.h>
  44 #include <sys/mbuf.h>
  45 #include <sys/module.h>
  46 #include <sys/lock.h>
  47 #include <sys/mutex.h>
  48 #include <sys/sysctl.h>
  49 #include <sys/endian.h>
  50 #include <sys/uio.h>
  51
  52 #include <vm/vm.h>
  53 #include <vm/pmap.h>
  54
  55 #include <machine/bus.h>
  56 #include <machine/resource.h>
  57 #include <sys/bus.h>
  58 #include <sys/rman.h>
  59
  60 #include <opencrypto/cryptodev.h>
  61 #include <opencrypto/xform_auth.h>
  62 #include <sys/random.h>
  63 #include <sys/kobj.h>
  64
  65 #include "cryptodev_if.h"
  66
  67 #include <dev/pci/pcivar.h>
  68 #include <dev/pci/pcireg.h>
  69
  70 #ifdef SAFE_RNDTEST
  71 #include <dev/rndtest/rndtest.h>
  72 #endif
  73 #include <dev/safe/safereg.h>
  74 #include <dev/safe/safevar.h>
  75
  76 #ifndef bswap32
  77 #define bswap32 NTOHL
  78 #endif
  79
  80 /*
  81  * Prototypes and count for the pci_device structure
  82  */
  83 static  int safe_probe(device_t);
  84 static  int safe_attach(device_t);
  85 static  int safe_detach(device_t);
  86 static  int safe_suspend(device_t);
  87 static  int safe_resume(device_t);
  88 static  int safe_shutdown(device_t);
  89
  90 static  int safe_probesession(device_t, const struct crypto_session_params *);
  91 static  int safe_newsession(device_t, crypto_session_t,
  92             const struct crypto_session_params *);
  93 static  int safe_process(device_t, struct cryptop *, int);
  94
  95 static device_method_t safe_methods[] = {
  96         /* Device interface */
  97         DEVMETHOD(device_probe,         safe_probe),
  98         DEVMETHOD(device_attach,        safe_attach),
  99         DEVMETHOD(device_detach,        safe_detach),
 100         DEVMETHOD(device_suspend,       safe_suspend),
 101         DEVMETHOD(device_resume,        safe_resume),
 102         DEVMETHOD(device_shutdown,      safe_shutdown),
 103
 104         /* crypto device methods */
 105         DEVMETHOD(cryptodev_probesession, safe_probesession),
 106         DEVMETHOD(cryptodev_newsession, safe_newsession),
 107         DEVMETHOD(cryptodev_process,    safe_process),
 108
 109         DEVMETHOD_END
 110 };
 111 static driver_t safe_driver = {
 112         "safe",
 113         safe_methods,
 114         sizeof (struct safe_softc)
 115 };
 116 static devclass_t safe_devclass;
 117
 118 DRIVER_MODULE(safe, pci, safe_driver, safe_devclass, 0, 0);
 119 MODULE_DEPEND(safe, crypto, 1, 1, 1);
 120 #ifdef SAFE_RNDTEST
 121 MODULE_DEPEND(safe, rndtest, 1, 1, 1);
 122 #endif
 123
 124 static  void safe_intr(void *);
 125 static  void safe_callback(struct safe_softc *, struct safe_ringentry *);
 126 static  void safe_feed(struct safe_softc *, struct safe_ringentry *);
 127 static  void safe_mcopy(struct mbuf *, struct mbuf *, u_int);
 128 #ifndef SAFE_NO_RNG
 129 static  void safe_rng_init(struct safe_softc *);
 130 static  void safe_rng(void *);
 131 #endif /* SAFE_NO_RNG */
 132 static  int safe_dma_malloc(struct safe_softc *, bus_size_t,
 133                 struct safe_dma_alloc *, int);
 134 #define safe_dma_sync(_dma, _flags) \
 135         bus_dmamap_sync((_dma)->dma_tag, (_dma)->dma_map, (_flags))
 136 static  void safe_dma_free(struct safe_softc *, struct safe_dma_alloc *);
 137 static  int safe_dmamap_aligned(const struct safe_operand *);
 138 static  int safe_dmamap_uniform(const struct safe_operand *);
 139
 140 static  void safe_reset_board(struct safe_softc *);
 141 static  void safe_init_board(struct safe_softc *);
 142 static  void safe_init_pciregs(device_t dev);
 143 static  void safe_cleanchip(struct safe_softc *);
 144 static  void safe_totalreset(struct safe_softc *);
 145
 146 static  int safe_free_entry(struct safe_softc *, struct safe_ringentry *);
 147
 148 static SYSCTL_NODE(_hw, OID_AUTO, safe, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
 149     "SafeNet driver parameters");
 150
 151 #ifdef SAFE_DEBUG
 152 static  void safe_dump_dmastatus(struct safe_softc *, const char *);
 153 static  void safe_dump_ringstate(struct safe_softc *, const char *);
 154 static  void safe_dump_intrstate(struct safe_softc *, const char *);
 155 static  void safe_dump_request(struct safe_softc *, const char *,
 156                 struct safe_ringentry *);
 157
 158 static  struct safe_softc *safec;               /* for use by hw.safe.dump */
 159
 160 static  int safe_debug = 0;
 161 SYSCTL_INT(_hw_safe, OID_AUTO, debug, CTLFLAG_RW, &safe_debug,
 162             0, "control debugging msgs");
 163 #define DPRINTF(_x)     if (safe_debug) printf _x
 164 #else
 165 #define DPRINTF(_x)
 166 #endif
 167
 168 #define READ_REG(sc,r) \
 169         bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r))
 170
 171 #define WRITE_REG(sc,reg,val) \
 172         bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val)
 173
 174 struct safe_stats safestats;
 175 SYSCTL_STRUCT(_hw_safe, OID_AUTO, stats, CTLFLAG_RD, &safestats,
 176             safe_stats, "driver statistics");
 177 #ifndef SAFE_NO_RNG
 178 static  int safe_rnginterval = 1;               /* poll once a second */
 179 SYSCTL_INT(_hw_safe, OID_AUTO, rnginterval, CTLFLAG_RW, &safe_rnginterval,
 180             0, "RNG polling interval (secs)");
 181 static  int safe_rngbufsize = 16;               /* 64 bytes each poll  */
 182 SYSCTL_INT(_hw_safe, OID_AUTO, rngbufsize, CTLFLAG_RW, &safe_rngbufsize,
 183             0, "RNG polling buffer size (32-bit words)");
 184 static  int safe_rngmaxalarm = 8;               /* max alarms before reset */
 185 SYSCTL_INT(_hw_safe, OID_AUTO, rngmaxalarm, CTLFLAG_RW, &safe_rngmaxalarm,
 186             0, "RNG max alarms before reset");
 187 #endif /* SAFE_NO_RNG */
 188
 189 static int
 190 safe_probe(device_t dev)
 191 {
 192         if (pci_get_vendor(dev) == PCI_VENDOR_SAFENET &&
 193             pci_get_device(dev) == PCI_PRODUCT_SAFEXCEL)
 194                 return (BUS_PROBE_DEFAULT);
 195         return (ENXIO);
 196 }
 197
 198 static const char*
 199 safe_partname(struct safe_softc *sc)
 200 {
 201         /* XXX sprintf numbers when not decoded */
 202         switch (pci_get_vendor(sc->sc_dev)) {
 203         case PCI_VENDOR_SAFENET:
 204                 switch (pci_get_device(sc->sc_dev)) {
 205                 case PCI_PRODUCT_SAFEXCEL: return "SafeNet SafeXcel-1141";
 206                 }
 207                 return "SafeNet unknown-part";
 208         }
 209         return "Unknown-vendor unknown-part";
 210 }
 211
 212 #ifndef SAFE_NO_RNG
 213 static void
 214 default_harvest(struct rndtest_state *rsp, void *buf, u_int count)
 215 {
 216         /* MarkM: FIX!! Check that this does not swamp the harvester! */
 217         random_harvest_queue(buf, count, RANDOM_PURE_SAFE);
 218 }
 219 #endif /* SAFE_NO_RNG */
 220
 221 static int
 222 safe_attach(device_t dev)
 223 {
 224         struct safe_softc *sc = device_get_softc(dev);
 225         u_int32_t raddr;
 226         u_int32_t i;
 227         int rid;
 228
 229         bzero(sc, sizeof (*sc));
 230         sc->sc_dev = dev;
 231
 232         /* XXX handle power management */
 233
 234         pci_enable_busmaster(dev);
 235
 236         /*
 237          * Setup memory-mapping of PCI registers.
 238          */
 239         rid = BS_BAR;
 240         sc->sc_sr = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 241                                            RF_ACTIVE);
 242         if (sc->sc_sr == NULL) {
 243                 device_printf(dev, "cannot map register space\n");
 244                 goto bad;
 245         }
 246         sc->sc_st = rman_get_bustag(sc->sc_sr);
 247         sc->sc_sh = rman_get_bushandle(sc->sc_sr);
 248
 249         /*
 250          * Arrange interrupt line.
 251          */
 252         rid = 0;
 253         sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 254                                             RF_SHAREABLE|RF_ACTIVE);
 255         if (sc->sc_irq == NULL) {
 256                 device_printf(dev, "could not map interrupt\n");
 257                 goto bad1;
 258         }
 259         /*
 260          * NB: Network code assumes we are blocked with splimp()
 261          *     so make sure the IRQ is mapped appropriately.
 262          */
 263         if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE,
 264                            NULL, safe_intr, sc, &sc->sc_ih)) {
 265                 device_printf(dev, "could not establish interrupt\n");
 266                 goto bad2;
 267         }
 268
 269         sc->sc_cid = crypto_get_driverid(dev, sizeof(struct safe_session),
 270             CRYPTOCAP_F_HARDWARE);
 271         if (sc->sc_cid < 0) {
 272                 device_printf(dev, "could not get crypto driver id\n");
 273                 goto bad3;
 274         }
 275
 276         sc->sc_chiprev = READ_REG(sc, SAFE_DEVINFO) &
 277                 (SAFE_DEVINFO_REV_MAJ | SAFE_DEVINFO_REV_MIN);
 278
 279         /*
 280          * Setup DMA descriptor area.
 281          */
 282         if (bus_dma_tag_create(bus_get_dma_tag(dev),    /* parent */
 283                                1,                       /* alignment */
 284                                SAFE_DMA_BOUNDARY,       /* boundary */
 285                                BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
 286                                BUS_SPACE_MAXADDR,       /* highaddr */
 287                                NULL, NULL,              /* filter, filterarg */
 288                                SAFE_MAX_DMA,            /* maxsize */
 289                                SAFE_MAX_PART,           /* nsegments */
 290                                SAFE_MAX_SSIZE,          /* maxsegsize */
 291                                BUS_DMA_ALLOCNOW,        /* flags */
 292                                NULL, NULL,              /* locking */
 293                                &sc->sc_srcdmat)) {
 294                 device_printf(dev, "cannot allocate DMA tag\n");
 295                 goto bad4;
 296         }
 297         if (bus_dma_tag_create(bus_get_dma_tag(dev),    /* parent */
 298                                1,                       /* alignment */
 299                                SAFE_MAX_DSIZE,          /* boundary */
 300                                BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
 301                                BUS_SPACE_MAXADDR,       /* highaddr */
 302                                NULL, NULL,              /* filter, filterarg */
 303                                SAFE_MAX_DMA,            /* maxsize */
 304                                SAFE_MAX_PART,           /* nsegments */
 305                                SAFE_MAX_DSIZE,          /* maxsegsize */
 306                                BUS_DMA_ALLOCNOW,        /* flags */
 307                                NULL, NULL,              /* locking */
 308                                &sc->sc_dstdmat)) {
 309                 device_printf(dev, "cannot allocate DMA tag\n");
 310                 goto bad4;
 311         }
 312
 313         /*
 314          * Allocate packet engine descriptors.
 315          */
 316         if (safe_dma_malloc(sc,
 317             SAFE_MAX_NQUEUE * sizeof (struct safe_ringentry),
 318             &sc->sc_ringalloc, 0)) {
 319                 device_printf(dev, "cannot allocate PE descriptor ring\n");
 320                 bus_dma_tag_destroy(sc->sc_srcdmat);
 321                 goto bad4;
 322         }
 323         /*
 324          * Hookup the static portion of all our data structures.
 325          */
 326         sc->sc_ring = (struct safe_ringentry *) sc->sc_ringalloc.dma_vaddr;
 327         sc->sc_ringtop = sc->sc_ring + SAFE_MAX_NQUEUE;
 328         sc->sc_front = sc->sc_ring;
 329         sc->sc_back = sc->sc_ring;
 330         raddr = sc->sc_ringalloc.dma_paddr;
 331         bzero(sc->sc_ring, SAFE_MAX_NQUEUE * sizeof(struct safe_ringentry));
 332         for (i = 0; i < SAFE_MAX_NQUEUE; i++) {
 333                 struct safe_ringentry *re = &sc->sc_ring[i];
 334
 335                 re->re_desc.d_sa = raddr +
 336                         offsetof(struct safe_ringentry, re_sa);
 337                 re->re_sa.sa_staterec = raddr +
 338                         offsetof(struct safe_ringentry, re_sastate);
 339
 340                 raddr += sizeof (struct safe_ringentry);
 341         }
 342         mtx_init(&sc->sc_ringmtx, device_get_nameunit(dev),
 343                 "packet engine ring", MTX_DEF);
 344
 345         /*
 346          * Allocate scatter and gather particle descriptors.
 347          */
 348         if (safe_dma_malloc(sc, SAFE_TOTAL_SPART * sizeof (struct safe_pdesc),
 349             &sc->sc_spalloc, 0)) {
 350                 device_printf(dev, "cannot allocate source particle "
 351                         "descriptor ring\n");
 352                 mtx_destroy(&sc->sc_ringmtx);
 353                 safe_dma_free(sc, &sc->sc_ringalloc);
 354                 bus_dma_tag_destroy(sc->sc_srcdmat);
 355                 goto bad4;
 356         }
 357         sc->sc_spring = (struct safe_pdesc *) sc->sc_spalloc.dma_vaddr;
 358         sc->sc_springtop = sc->sc_spring + SAFE_TOTAL_SPART;
 359         sc->sc_spfree = sc->sc_spring;
 360         bzero(sc->sc_spring, SAFE_TOTAL_SPART * sizeof(struct safe_pdesc));
 361
 362         if (safe_dma_malloc(sc, SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
 363             &sc->sc_dpalloc, 0)) {
 364                 device_printf(dev, "cannot allocate destination particle "
 365                         "descriptor ring\n");
 366                 mtx_destroy(&sc->sc_ringmtx);
 367                 safe_dma_free(sc, &sc->sc_spalloc);
 368                 safe_dma_free(sc, &sc->sc_ringalloc);
 369                 bus_dma_tag_destroy(sc->sc_dstdmat);
 370                 goto bad4;
 371         }
 372         sc->sc_dpring = (struct safe_pdesc *) sc->sc_dpalloc.dma_vaddr;
 373         sc->sc_dpringtop = sc->sc_dpring + SAFE_TOTAL_DPART;
 374         sc->sc_dpfree = sc->sc_dpring;
 375         bzero(sc->sc_dpring, SAFE_TOTAL_DPART * sizeof(struct safe_pdesc));
 376
 377         device_printf(sc->sc_dev, "%s", safe_partname(sc));
 378
 379         sc->sc_devinfo = READ_REG(sc, SAFE_DEVINFO);
 380         if (sc->sc_devinfo & SAFE_DEVINFO_RNG) {
 381                 sc->sc_flags |= SAFE_FLAGS_RNG;
 382                 printf(" rng");
 383         }
 384         if (sc->sc_devinfo & SAFE_DEVINFO_PKEY) {
 385 #if 0
 386                 printf(" key");
 387                 sc->sc_flags |= SAFE_FLAGS_KEY;
 388 #endif
 389         }
 390         if (sc->sc_devinfo & SAFE_DEVINFO_DES) {
 391                 printf(" des/3des");
 392         }
 393         if (sc->sc_devinfo & SAFE_DEVINFO_AES) {
 394                 printf(" aes");
 395         }
 396         if (sc->sc_devinfo & SAFE_DEVINFO_MD5) {
 397                 printf(" md5");
 398         }
 399         if (sc->sc_devinfo & SAFE_DEVINFO_SHA1) {
 400                 printf(" sha1");
 401         }
 402         /* XXX other supported algorithms */
 403         printf("\n");
 404
 405         safe_reset_board(sc);           /* reset h/w */
 406         safe_init_pciregs(dev);         /* init pci settings */
 407         safe_init_board(sc);            /* init h/w */
 408
 409 #ifndef SAFE_NO_RNG
 410         if (sc->sc_flags & SAFE_FLAGS_RNG) {
 411 #ifdef SAFE_RNDTEST
 412                 sc->sc_rndtest = rndtest_attach(dev);
 413                 if (sc->sc_rndtest)
 414                         sc->sc_harvest = rndtest_harvest;
 415                 else
 416                         sc->sc_harvest = default_harvest;
 417 #else
 418                 sc->sc_harvest = default_harvest;
 419 #endif
 420                 safe_rng_init(sc);
 421
 422                 callout_init(&sc->sc_rngto, 1);
 423                 callout_reset(&sc->sc_rngto, hz*safe_rnginterval, safe_rng, sc);
 424         }
 425 #endif /* SAFE_NO_RNG */
 426 #ifdef SAFE_DEBUG
 427         safec = sc;                     /* for use by hw.safe.dump */
 428 #endif
 429         return (0);
 430 bad4:
 431         crypto_unregister_all(sc->sc_cid);
 432 bad3:
 433         bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
 434 bad2:
 435         bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
 436 bad1:
 437         bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
 438 bad:
 439         return (ENXIO);
 440 }
 441
 442 /*
 443  * Detach a device that successfully probed.
 444  */
 445 static int
 446 safe_detach(device_t dev)
 447 {
 448         struct safe_softc *sc = device_get_softc(dev);
 449
 450         /* XXX wait/abort active ops */
 451
 452         WRITE_REG(sc, SAFE_HI_MASK, 0);         /* disable interrupts */
 453
 454         callout_stop(&sc->sc_rngto);
 455
 456         crypto_unregister_all(sc->sc_cid);
 457
 458 #ifdef SAFE_RNDTEST
 459         if (sc->sc_rndtest)
 460                 rndtest_detach(sc->sc_rndtest);
 461 #endif
 462
 463         safe_cleanchip(sc);
 464         safe_dma_free(sc, &sc->sc_dpalloc);
 465         safe_dma_free(sc, &sc->sc_spalloc);
 466         mtx_destroy(&sc->sc_ringmtx);
 467         safe_dma_free(sc, &sc->sc_ringalloc);
 468
 469         bus_generic_detach(dev);
 470         bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
 471         bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
 472
 473         bus_dma_tag_destroy(sc->sc_srcdmat);
 474         bus_dma_tag_destroy(sc->sc_dstdmat);
 475         bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
 476
 477         return (0);
 478 }
 479
 480 /*
 481  * Stop all chip i/o so that the kernel's probe routines don't
 482  * get confused by errant DMAs when rebooting.
 483  */
 484 static int
 485 safe_shutdown(device_t dev)
 486 {
 487 #ifdef notyet
 488         safe_stop(device_get_softc(dev));
 489 #endif
 490         return (0);
 491 }
 492
 493 /*
 494  * Device suspend routine.
 495  */
 496 static int
 497 safe_suspend(device_t dev)
 498 {
 499         struct safe_softc *sc = device_get_softc(dev);
 500
 501 #ifdef notyet
 502         /* XXX stop the device and save PCI settings */
 503 #endif
 504         sc->sc_suspended = 1;
 505
 506         return (0);
 507 }
 508
 509 static int
 510 safe_resume(device_t dev)
 511 {
 512         struct safe_softc *sc = device_get_softc(dev);
 513
 514 #ifdef notyet
 515         /* XXX retore PCI settings and start the device */
 516 #endif
 517         sc->sc_suspended = 0;
 518         return (0);
 519 }
 520
 521 /*
 522  * SafeXcel Interrupt routine
 523  */
 524 static void
 525 safe_intr(void *arg)
 526 {
 527         struct safe_softc *sc = arg;
 528         volatile u_int32_t stat;
 529
 530         stat = READ_REG(sc, SAFE_HM_STAT);
 531         if (stat == 0)                  /* shared irq, not for us */
 532                 return;
 533
 534         WRITE_REG(sc, SAFE_HI_CLR, stat);       /* IACK */
 535
 536         if ((stat & SAFE_INT_PE_DDONE)) {
 537                 /*
 538                  * Descriptor(s) done; scan the ring and
 539                  * process completed operations.
 540                  */
 541                 mtx_lock(&sc->sc_ringmtx);
 542                 while (sc->sc_back != sc->sc_front) {
 543                         struct safe_ringentry *re = sc->sc_back;
 544 #ifdef SAFE_DEBUG
 545                         if (safe_debug) {
 546                                 safe_dump_ringstate(sc, __func__);
 547                                 safe_dump_request(sc, __func__, re);
 548                         }
 549 #endif
 550                         /*
 551                          * safe_process marks ring entries that were allocated
 552                          * but not used with a csr of zero.  This insures the
 553                          * ring front pointer never needs to be set backwards
 554                          * in the event that an entry is allocated but not used
 555                          * because of a setup error.
 556                          */
 557                         if (re->re_desc.d_csr != 0) {
 558                                 if (!SAFE_PE_CSR_IS_DONE(re->re_desc.d_csr))
 559                                         break;
 560                                 if (!SAFE_PE_LEN_IS_DONE(re->re_desc.d_len))
 561                                         break;
 562                                 sc->sc_nqchip--;
 563                                 safe_callback(sc, re);
 564                         }
 565                         if (++(sc->sc_back) == sc->sc_ringtop)
 566                                 sc->sc_back = sc->sc_ring;
 567                 }
 568                 mtx_unlock(&sc->sc_ringmtx);
 569         }
 570
 571         /*
 572          * Check to see if we got any DMA Error
 573          */
 574         if (stat & SAFE_INT_PE_ERROR) {
 575                 DPRINTF(("dmaerr dmastat %08x\n",
 576                         READ_REG(sc, SAFE_PE_DMASTAT)));
 577                 safestats.st_dmaerr++;
 578                 safe_totalreset(sc);
 579 #if 0
 580                 safe_feed(sc);
 581 #endif
 582         }
 583
 584         if (sc->sc_needwakeup) {                /* XXX check high watermark */
 585                 int wakeup = sc->sc_needwakeup & CRYPTO_SYMQ;
 586                 DPRINTF(("%s: wakeup crypto %x\n", __func__,
 587                         sc->sc_needwakeup));
 588                 sc->sc_needwakeup &= ~wakeup;
 589                 crypto_unblock(sc->sc_cid, wakeup);
 590         }
 591 }
 592
 593 /*
 594  * safe_feed() - post a request to chip
 595  */
 596 static void
 597 safe_feed(struct safe_softc *sc, struct safe_ringentry *re)
 598 {
 599         bus_dmamap_sync(sc->sc_srcdmat, re->re_src_map, BUS_DMASYNC_PREWRITE);
 600         if (re->re_dst_map != NULL)
 601                 bus_dmamap_sync(sc->sc_dstdmat, re->re_dst_map,
 602                         BUS_DMASYNC_PREREAD);
 603         /* XXX have no smaller granularity */
 604         safe_dma_sync(&sc->sc_ringalloc,
 605                 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 606         safe_dma_sync(&sc->sc_spalloc, BUS_DMASYNC_PREWRITE);
 607         safe_dma_sync(&sc->sc_dpalloc, BUS_DMASYNC_PREWRITE);
 608
 609 #ifdef SAFE_DEBUG
 610         if (safe_debug) {
 611                 safe_dump_ringstate(sc, __func__);
 612                 safe_dump_request(sc, __func__, re);
 613         }
 614 #endif
 615         sc->sc_nqchip++;
 616         if (sc->sc_nqchip > safestats.st_maxqchip)
 617                 safestats.st_maxqchip = sc->sc_nqchip;
 618         /* poke h/w to check descriptor ring, any value can be written */
 619         WRITE_REG(sc, SAFE_HI_RD_DESCR, 0);
 620 }
 621
 622 #define N(a)    (sizeof(a) / sizeof (a[0]))
 623 static void
 624 safe_setup_enckey(struct safe_session *ses, const void *key)
 625 {
 626         int i;
 627
 628         bcopy(key, ses->ses_key, ses->ses_klen);
 629
 630         /* PE is little-endian, insure proper byte order */
 631         for (i = 0; i < N(ses->ses_key); i++)
 632                 ses->ses_key[i] = htole32(ses->ses_key[i]);
 633 }
 634
 635 static void
 636 safe_setup_mackey(struct safe_session *ses, int algo, const uint8_t *key,
 637     int klen)
 638 {
 639         SHA1_CTX sha1ctx;
 640         int i;
 641
 642         hmac_init_ipad(&auth_hash_hmac_sha1, key, klen, &sha1ctx);
 643         bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32));
 644
 645         hmac_init_opad(&auth_hash_hmac_sha1, key, klen, &sha1ctx);
 646         bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32));
 647
 648         explicit_bzero(&sha1ctx, sizeof(sha1ctx));
 649
 650         /* PE is little-endian, insure proper byte order */
 651         for (i = 0; i < N(ses->ses_hminner); i++) {
 652                 ses->ses_hminner[i] = htole32(ses->ses_hminner[i]);
 653                 ses->ses_hmouter[i] = htole32(ses->ses_hmouter[i]);
 654         }
 655 }
 656 #undef N
 657
 658 static bool
 659 safe_auth_supported(struct safe_softc *sc,
 660     const struct crypto_session_params *csp)
 661 {
 662
 663         switch (csp->csp_auth_alg) {
 664         case CRYPTO_SHA1_HMAC:
 665                 if ((sc->sc_devinfo & SAFE_DEVINFO_SHA1) == 0)
 666                         return (false);
 667                 break;
 668         default:
 669                 return (false);
 670         }
 671         return (true);
 672 }
 673
 674 static bool
 675 safe_cipher_supported(struct safe_softc *sc,
 676     const struct crypto_session_params *csp)
 677 {
 678
 679         switch (csp->csp_cipher_alg) {
 680         case CRYPTO_AES_CBC:
 681                 if ((sc->sc_devinfo & SAFE_DEVINFO_AES) == 0)
 682                         return (false);
 683                 if (csp->csp_ivlen != 16)
 684                         return (false);
 685                 if (csp->csp_cipher_klen != 16 &&
 686                     csp->csp_cipher_klen != 24 &&
 687                     csp->csp_cipher_klen != 32)
 688                         return (false);
 689                 break;
 690         }
 691         return (true);
 692 }
 693
 694 static int
 695 safe_probesession(device_t dev, const struct crypto_session_params *csp)
 696 {
 697         struct safe_softc *sc = device_get_softc(dev);
 698
 699         if (csp->csp_flags != 0)
 700                 return (EINVAL);
 701         switch (csp->csp_mode) {
 702         case CSP_MODE_DIGEST:
 703                 if (!safe_auth_supported(sc, csp))
 704                         return (EINVAL);
 705                 break;
 706         case CSP_MODE_CIPHER:
 707                 if (!safe_cipher_supported(sc, csp))
 708                         return (EINVAL);
 709                 break;
 710         case CSP_MODE_ETA:
 711                 if (!safe_auth_supported(sc, csp) ||
 712                     !safe_cipher_supported(sc, csp))
 713                         return (EINVAL);
 714                 break;
 715         default:
 716                 return (EINVAL);
 717         }
 718
 719         return (CRYPTODEV_PROBE_HARDWARE);
 720 }
 721
 722 /*
 723  * Allocate a new 'session'.
 724  */
 725 static int
 726 safe_newsession(device_t dev, crypto_session_t cses,
 727     const struct crypto_session_params *csp)
 728 {
 729         struct safe_session *ses;
 730
 731         ses = crypto_get_driver_session(cses);
 732         if (csp->csp_cipher_alg != 0) {
 733                 ses->ses_klen = csp->csp_cipher_klen;
 734                 if (csp->csp_cipher_key != NULL)
 735                         safe_setup_enckey(ses, csp->csp_cipher_key);
 736         }
 737
 738         if (csp->csp_auth_alg != 0) {
 739                 ses->ses_mlen = csp->csp_auth_mlen;
 740                 if (ses->ses_mlen == 0) {
 741                         ses->ses_mlen = SHA1_HASH_LEN;
 742                 }
 743
 744                 if (csp->csp_auth_key != NULL) {
 745                         safe_setup_mackey(ses, csp->csp_auth_alg,
 746                             csp->csp_auth_key, csp->csp_auth_klen);
 747                 }
 748         }
 749
 750         return (0);
 751 }
 752
 753 static void
 754 safe_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, int error)
 755 {
 756         struct safe_operand *op = arg;
 757
 758         DPRINTF(("%s: nsegs %d error %d\n", __func__,
 759                 nsegs, error));
 760         if (error != 0)
 761                 return;
 762         op->nsegs = nsegs;
 763         bcopy(seg, op->segs, nsegs * sizeof (seg[0]));
 764 }
 765
 766 static int
 767 safe_process(device_t dev, struct cryptop *crp, int hint)
 768 {
 769         struct safe_softc *sc = device_get_softc(dev);
 770         const struct crypto_session_params *csp;
 771         int err = 0, i, nicealign, uniform;
 772         int bypass, oplen;
 773         int16_t coffset;
 774         struct safe_session *ses;
 775         struct safe_ringentry *re;
 776         struct safe_sarec *sa;
 777         struct safe_pdesc *pd;
 778         u_int32_t cmd0, cmd1, staterec;
 779
 780         mtx_lock(&sc->sc_ringmtx);
 781         if (sc->sc_front == sc->sc_back && sc->sc_nqchip != 0) {
 782                 safestats.st_ringfull++;
 783                 sc->sc_needwakeup |= CRYPTO_SYMQ;
 784                 mtx_unlock(&sc->sc_ringmtx);
 785                 return (ERESTART);
 786         }
 787         re = sc->sc_front;
 788
 789         staterec = re->re_sa.sa_staterec;       /* save */
 790         /* NB: zero everything but the PE descriptor */
 791         bzero(&re->re_sa, sizeof(struct safe_ringentry) - sizeof(re->re_desc));
 792         re->re_sa.sa_staterec = staterec;       /* restore */
 793
 794         re->re_crp = crp;
 795
 796         sa = &re->re_sa;
 797         ses = crypto_get_driver_session(crp->crp_session);
 798         csp = crypto_get_params(crp->crp_session);
 799
 800         cmd0 = SAFE_SA_CMD0_BASIC;              /* basic group operation */
 801         cmd1 = 0;
 802         switch (csp->csp_mode) {
 803         case CSP_MODE_DIGEST:
 804                 cmd0 |= SAFE_SA_CMD0_OP_HASH;
 805                 break;
 806         case CSP_MODE_CIPHER:
 807                 cmd0 |= SAFE_SA_CMD0_OP_CRYPT;
 808                 break;
 809         case CSP_MODE_ETA:
 810                 cmd0 |= SAFE_SA_CMD0_OP_BOTH;
 811                 break;
 812         }
 813
 814         if (csp->csp_cipher_alg != 0) {
 815                 if (crp->crp_cipher_key != NULL)
 816                         safe_setup_enckey(ses, crp->crp_cipher_key);
 817
 818                 switch (csp->csp_cipher_alg) {
 819                 case CRYPTO_AES_CBC:
 820                         cmd0 |= SAFE_SA_CMD0_AES;
 821                         cmd1 |= SAFE_SA_CMD1_CBC;
 822                         if (ses->ses_klen * 8 == 128)
 823                              cmd1 |=  SAFE_SA_CMD1_AES128;
 824                         else if (ses->ses_klen * 8 == 192)
 825                              cmd1 |=  SAFE_SA_CMD1_AES192;
 826                         else
 827                              cmd1 |=  SAFE_SA_CMD1_AES256;
 828                 }
 829
 830                 /*
 831                  * Setup encrypt/decrypt state.  When using basic ops
 832                  * we can't use an inline IV because hash/crypt offset
 833                  * must be from the end of the IV to the start of the
 834                  * crypt data and this leaves out the preceding header
 835                  * from the hash calculation.  Instead we place the IV
 836                  * in the state record and set the hash/crypt offset to
 837                  * copy both the header+IV.
 838                  */
 839                 crypto_read_iv(crp, re->re_sastate.sa_saved_iv);
 840                 cmd0 |= SAFE_SA_CMD0_IVLD_STATE;
 841
 842                 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
 843                         cmd0 |= SAFE_SA_CMD0_OUTBOUND;
 844
 845                         /*
 846                          * XXX: I suspect we don't need this since we
 847                          * don't save the returned IV.
 848                          */
 849                         cmd0 |= SAFE_SA_CMD0_SAVEIV;
 850                 } else {
 851                         cmd0 |= SAFE_SA_CMD0_INBOUND;
 852                 }
 853                 /*
 854                  * For basic encryption use the zero pad algorithm.
 855                  * This pads results to an 8-byte boundary and
 856                  * suppresses padding verification for inbound (i.e.
 857                  * decrypt) operations.
 858                  *
 859                  * NB: Not sure if the 8-byte pad boundary is a problem.
 860                  */
 861                 cmd0 |= SAFE_SA_CMD0_PAD_ZERO;
 862
 863                 /* XXX assert key bufs have the same size */
 864                 bcopy(ses->ses_key, sa->sa_key, sizeof(sa->sa_key));
 865         }
 866
 867         if (csp->csp_auth_alg != 0) {
 868                 if (crp->crp_auth_key != NULL) {
 869                         safe_setup_mackey(ses, csp->csp_auth_alg,
 870                             crp->crp_auth_key, csp->csp_auth_klen);
 871                 }
 872
 873                 switch (csp->csp_auth_alg) {
 874                 case CRYPTO_SHA1_HMAC:
 875                         cmd0 |= SAFE_SA_CMD0_SHA1;
 876                         cmd1 |= SAFE_SA_CMD1_HMAC;      /* NB: enable HMAC */
 877                         break;
 878                 }
 879
 880                 /*
 881                  * Digest data is loaded from the SA and the hash
 882                  * result is saved to the state block where we
 883                  * retrieve it for return to the caller.
 884                  */
 885                 /* XXX assert digest bufs have the same size */
 886                 bcopy(ses->ses_hminner, sa->sa_indigest,
 887                         sizeof(sa->sa_indigest));
 888                 bcopy(ses->ses_hmouter, sa->sa_outdigest,
 889                         sizeof(sa->sa_outdigest));
 890
 891                 cmd0 |= SAFE_SA_CMD0_HSLD_SA | SAFE_SA_CMD0_SAVEHASH;
 892                 re->re_flags |= SAFE_QFLAGS_COPYOUTICV;
 893         }
 894
 895         if (csp->csp_mode == CSP_MODE_ETA) {
 896                 /*
 897                  * The driver only supports ETA requests where there
 898                  * is no gap between the AAD and payload.
 899                  */
 900                 if (crp->crp_aad_length != 0 &&
 901                     crp->crp_aad_start + crp->crp_aad_length !=
 902                     crp->crp_payload_start) {
 903                         safestats.st_lenmismatch++;
 904                         err = EINVAL;
 905                         goto errout;
 906                 }
 907                 if (crp->crp_aad_length != 0)
 908                         bypass = crp->crp_aad_start;
 909                 else
 910                         bypass = crp->crp_payload_start;
 911                 coffset = crp->crp_aad_length;
 912                 oplen = crp->crp_payload_start + crp->crp_payload_length;
 913 #ifdef SAFE_DEBUG
 914                 if (safe_debug) {
 915                         printf("AAD: skip %d, len %d, digest %d\n",
 916                             crp->crp_aad_start, crp->crp_aad_length,
 917                             crp->crp_digest_start);
 918                         printf("payload: skip %d, len %d, IV %d\n",
 919                             crp->crp_payload_start, crp->crp_payload_length,
 920                             crp->crp_iv_start);
 921                         printf("bypass %d coffset %d oplen %d\n",
 922                                 bypass, coffset, oplen);
 923                 }
 924 #endif
 925                 if (coffset & 3) {      /* offset must be 32-bit aligned */
 926                         DPRINTF(("%s: coffset %u misaligned\n",
 927                                 __func__, coffset));
 928                         safestats.st_coffmisaligned++;
 929                         err = EINVAL;
 930                         goto errout;
 931                 }
 932                 coffset >>= 2;
 933                 if (coffset > 255) {    /* offset must be <256 dwords */
 934                         DPRINTF(("%s: coffset %u too big\n",
 935                                 __func__, coffset));
 936                         safestats.st_cofftoobig++;
 937                         err = EINVAL;
 938                         goto errout;
 939                 }
 940                 /*
 941                  * Tell the hardware to copy the header to the output.
 942                  * The header is defined as the data from the end of
 943                  * the bypass to the start of data to be encrypted.
 944                  * Typically this is the inline IV.  Note that you need
 945                  * to do this even if src+dst are the same; it appears
 946                  * that w/o this bit the crypted data is written
 947                  * immediately after the bypass data.
 948                  */
 949                 cmd1 |= SAFE_SA_CMD1_HDRCOPY;
 950                 /*
 951                  * Disable IP header mutable bit handling.  This is
 952                  * needed to get correct HMAC calculations.
 953                  */
 954                 cmd1 |= SAFE_SA_CMD1_MUTABLE;
 955         } else {
 956                 bypass = crp->crp_payload_start;
 957                 oplen = bypass + crp->crp_payload_length;
 958                 coffset = 0;
 959         }
 960         /* XXX verify multiple of 4 when using s/g */
 961         if (bypass > 96) {              /* bypass offset must be <= 96 bytes */
 962                 DPRINTF(("%s: bypass %u too big\n", __func__, bypass));
 963                 safestats.st_bypasstoobig++;
 964                 err = EINVAL;
 965                 goto errout;
 966         }
 967
 968         if (bus_dmamap_create(sc->sc_srcdmat, BUS_DMA_NOWAIT, &re->re_src_map)) {
 969                 safestats.st_nomap++;
 970                 err = ENOMEM;
 971                 goto errout;
 972         }
 973         if (bus_dmamap_load_crp(sc->sc_srcdmat, re->re_src_map, crp, safe_op_cb,
 974             &re->re_src, BUS_DMA_NOWAIT) != 0) {
 975                 bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);
 976                 re->re_src_map = NULL;
 977                 safestats.st_noload++;
 978                 err = ENOMEM;
 979                 goto errout;
 980         }
 981         re->re_src_mapsize = crypto_buffer_len(&crp->crp_buf);
 982         nicealign = safe_dmamap_aligned(&re->re_src);
 983         uniform = safe_dmamap_uniform(&re->re_src);
 984
 985         DPRINTF(("src nicealign %u uniform %u nsegs %u\n",
 986                 nicealign, uniform, re->re_src.nsegs));
 987         if (re->re_src.nsegs > 1) {
 988                 re->re_desc.d_src = sc->sc_spalloc.dma_paddr +
 989                         ((caddr_t) sc->sc_spfree - (caddr_t) sc->sc_spring);
 990                 for (i = 0; i < re->re_src_nsegs; i++) {
 991                         /* NB: no need to check if there's space */
 992                         pd = sc->sc_spfree;
 993                         if (++(sc->sc_spfree) == sc->sc_springtop)
 994                                 sc->sc_spfree = sc->sc_spring;
 995
 996                         KASSERT((pd->pd_flags&3) == 0 ||
 997                                 (pd->pd_flags&3) == SAFE_PD_DONE,
 998                                 ("bogus source particle descriptor; flags %x",
 999                                 pd->pd_flags));
1000                         pd->pd_addr = re->re_src_segs[i].ds_addr;
1001                         pd->pd_size = re->re_src_segs[i].ds_len;
1002                         pd->pd_flags = SAFE_PD_READY;
1003                 }
1004                 cmd0 |= SAFE_SA_CMD0_IGATHER;
1005         } else {
1006                 /*
1007                  * No need for gather, reference the operand directly.
1008                  */
1009                 re->re_desc.d_src = re->re_src_segs[0].ds_addr;
1010         }
1011
1012         if (csp->csp_mode == CSP_MODE_DIGEST) {
1013                 /*
1014                  * Hash op; no destination needed.
1015                  */
1016         } else {
1017                 if (nicealign && uniform == 1) {
1018                         /*
1019                          * Source layout is suitable for direct
1020                          * sharing of the DMA map and segment list.
1021                          */
1022                         re->re_dst = re->re_src;
1023                 } else if (nicealign && uniform == 2) {
1024                         /*
1025                          * The source is properly aligned but requires a
1026                          * different particle list to handle DMA of the
1027                          * result.  Create a new map and do the load to
1028                          * create the segment list.  The particle
1029                          * descriptor setup code below will handle the
1030                          * rest.
1031                          */
1032                         if (bus_dmamap_create(sc->sc_dstdmat, BUS_DMA_NOWAIT,
1033                             &re->re_dst_map)) {
1034                                 safestats.st_nomap++;
1035                                 err = ENOMEM;
1036                                 goto errout;
1037                         }
1038                         if (bus_dmamap_load_crp(sc->sc_dstdmat, re->re_dst_map,
1039                             crp, safe_op_cb, &re->re_dst, BUS_DMA_NOWAIT) !=
1040                             0) {
1041                                 bus_dmamap_destroy(sc->sc_dstdmat,
1042                                     re->re_dst_map);
1043                                 re->re_dst_map = NULL;
1044                                 safestats.st_noload++;
1045                                 err = ENOMEM;
1046                                 goto errout;
1047                         }
1048                 } else if (crp->crp_buf.cb_type == CRYPTO_BUF_MBUF) {
1049                         int totlen, len;
1050                         struct mbuf *m, *top, **mp;
1051
1052                         /*
1053                          * DMA constraints require that we allocate a
1054                          * new mbuf chain for the destination.  We
1055                          * allocate an entire new set of mbufs of
1056                          * optimal/required size and then tell the
1057                          * hardware to copy any bits that are not
1058                          * created as a byproduct of the operation.
1059                          */
1060                         if (!nicealign)
1061                                 safestats.st_unaligned++;
1062                         if (!uniform)
1063                                 safestats.st_notuniform++;
1064                         totlen = re->re_src_mapsize;
1065                         if (crp->crp_buf.cb_mbuf->m_flags & M_PKTHDR) {
1066                                 len = MHLEN;
1067                                 MGETHDR(m, M_NOWAIT, MT_DATA);
1068                                 if (m && !m_dup_pkthdr(m, crp->crp_buf.cb_mbuf,
1069                                     M_NOWAIT)) {
1070                                         m_free(m);
1071                                         m = NULL;
1072                                 }
1073                         } else {
1074                                 len = MLEN;
1075                                 MGET(m, M_NOWAIT, MT_DATA);
1076                         }
1077                         if (m == NULL) {
1078                                 safestats.st_nombuf++;
1079                                 err = sc->sc_nqchip ? ERESTART : ENOMEM;
1080                                 goto errout;
1081                         }
1082                         if (totlen >= MINCLSIZE) {
1083                                 if (!(MCLGET(m, M_NOWAIT))) {
1084                                         m_free(m);
1085                                         safestats.st_nomcl++;
1086                                         err = sc->sc_nqchip ?
1087                                             ERESTART : ENOMEM;
1088                                         goto errout;
1089                                 }
1090                                 len = MCLBYTES;
1091                         }
1092                         m->m_len = len;
1093                         top = NULL;
1094                         mp = &top;
1095
1096                         while (totlen > 0) {
1097                                 if (top) {
1098                                         MGET(m, M_NOWAIT, MT_DATA);
1099                                         if (m == NULL) {
1100                                                 m_freem(top);
1101                                                 safestats.st_nombuf++;
1102                                                 err = sc->sc_nqchip ?
1103                                                     ERESTART : ENOMEM;
1104                                                 goto errout;
1105                                         }
1106                                         len = MLEN;
1107                                 }
1108                                 if (top && totlen >= MINCLSIZE) {
1109                                         if (!(MCLGET(m, M_NOWAIT))) {
1110                                                 *mp = m;
1111                                                 m_freem(top);
1112                                                 safestats.st_nomcl++;
1113                                                 err = sc->sc_nqchip ?
1114                                                     ERESTART : ENOMEM;
1115                                                 goto errout;
1116                                         }
1117                                         len = MCLBYTES;
1118                                 }
1119                                 m->m_len = len = min(totlen, len);
1120                                 totlen -= len;
1121                                 *mp = m;
1122                                 mp = &m->m_next;
1123                         }
1124                         re->re_dst_m = top;
1125                         if (bus_dmamap_create(sc->sc_dstdmat,
1126                             BUS_DMA_NOWAIT, &re->re_dst_map) != 0) {
1127                                 safestats.st_nomap++;
1128                                 err = ENOMEM;
1129                                 goto errout;
1130                         }
1131                         if (bus_dmamap_load_mbuf_sg(sc->sc_dstdmat,
1132                             re->re_dst_map, top, re->re_dst_segs,
1133                             &re->re_dst_nsegs, 0) != 0) {
1134                                 bus_dmamap_destroy(sc->sc_dstdmat,
1135                                     re->re_dst_map);
1136                                 re->re_dst_map = NULL;
1137                                 safestats.st_noload++;
1138                                 err = ENOMEM;
1139                                 goto errout;
1140                         }
1141                         re->re_dst_mapsize = re->re_src_mapsize;
1142                         if (re->re_src.mapsize > oplen) {
1143                                 /*
1144                                  * There's data following what the
1145                                  * hardware will copy for us.  If this
1146                                  * isn't just the ICV (that's going to
1147                                  * be written on completion), copy it
1148                                  * to the new mbufs
1149                                  */
1150                                 if (!(csp->csp_mode == CSP_MODE_ETA &&
1151                                     (re->re_src.mapsize-oplen) == ses->ses_mlen &&
1152                                     crp->crp_digest_start == oplen))
1153                                         safe_mcopy(crp->crp_buf.cb_mbuf,
1154                                             re->re_dst_m, oplen);
1155                                 else
1156                                         safestats.st_noicvcopy++;
1157                         }
1158                 } else {
1159                         if (!nicealign) {
1160                                 safestats.st_iovmisaligned++;
1161                                 err = EINVAL;
1162                                 goto errout;
1163                         } else {
1164                                 /*
1165                                  * There's no way to handle the DMA
1166                                  * requirements with this uio.  We
1167                                  * could create a separate DMA area for
1168                                  * the result and then copy it back,
1169                                  * but for now we just bail and return
1170                                  * an error.  Note that uio requests
1171                                  * > SAFE_MAX_DSIZE are handled because
1172                                  * the DMA map and segment list for the
1173                                  * destination wil result in a
1174                                  * destination particle list that does
1175                                  * the necessary scatter DMA.
1176                                  */
1177                                 safestats.st_iovnotuniform++;
1178                                 err = EINVAL;
1179                                 goto errout;
1180                         }
1181                 }
1182
1183                 if (re->re_dst.nsegs > 1) {
1184                         re->re_desc.d_dst = sc->sc_dpalloc.dma_paddr +
1185                             ((caddr_t) sc->sc_dpfree - (caddr_t) sc->sc_dpring);
1186                         for (i = 0; i < re->re_dst_nsegs; i++) {
1187                                 pd = sc->sc_dpfree;
1188                                 KASSERT((pd->pd_flags&3) == 0 ||
1189                                         (pd->pd_flags&3) == SAFE_PD_DONE,
1190                                         ("bogus dest particle descriptor; flags %x",
1191                                                 pd->pd_flags));
1192                                 if (++(sc->sc_dpfree) == sc->sc_dpringtop)
1193                                         sc->sc_dpfree = sc->sc_dpring;
1194                                 pd->pd_addr = re->re_dst_segs[i].ds_addr;
1195                                 pd->pd_flags = SAFE_PD_READY;
1196                         }
1197                         cmd0 |= SAFE_SA_CMD0_OSCATTER;
1198                 } else {
1199                         /*
1200                          * No need for scatter, reference the operand directly.
1201                          */
1202                         re->re_desc.d_dst = re->re_dst_segs[0].ds_addr;
1203                 }
1204         }
1205
1206         /*
1207          * All done with setup; fillin the SA command words
1208          * and the packet engine descriptor.  The operation
1209          * is now ready for submission to the hardware.
1210          */
1211         sa->sa_cmd0 = cmd0 | SAFE_SA_CMD0_IPCI | SAFE_SA_CMD0_OPCI;
1212         sa->sa_cmd1 = cmd1
1213                     | (coffset << SAFE_SA_CMD1_OFFSET_S)
1214                     | SAFE_SA_CMD1_SAREV1       /* Rev 1 SA data structure */
1215                     | SAFE_SA_CMD1_SRPCI
1216                     ;
1217         /*
1218          * NB: the order of writes is important here.  In case the
1219          * chip is scanning the ring because of an outstanding request
1220          * it might nab this one too.  In that case we need to make
1221          * sure the setup is complete before we write the length
1222          * field of the descriptor as it signals the descriptor is
1223          * ready for processing.
1224          */
1225         re->re_desc.d_csr = SAFE_PE_CSR_READY | SAFE_PE_CSR_SAPCI;
1226         if (csp->csp_auth_alg != 0)
1227                 re->re_desc.d_csr |= SAFE_PE_CSR_LOADSA | SAFE_PE_CSR_HASHFINAL;
1228         re->re_desc.d_len = oplen
1229                           | SAFE_PE_LEN_READY
1230                           | (bypass << SAFE_PE_LEN_BYPASS_S)
1231                           ;
1232
1233         safestats.st_ipackets++;
1234         safestats.st_ibytes += oplen;
1235
1236         if (++(sc->sc_front) == sc->sc_ringtop)
1237                 sc->sc_front = sc->sc_ring;
1238
1239         /* XXX honor batching */
1240         safe_feed(sc, re);
1241         mtx_unlock(&sc->sc_ringmtx);
1242         return (0);
1243
1244 errout:
1245         if (re->re_dst_m != NULL)
1246                 m_freem(re->re_dst_m);
1247
1248         if (re->re_dst_map != NULL && re->re_dst_map != re->re_src_map) {
1249                 bus_dmamap_unload(sc->sc_dstdmat, re->re_dst_map);
1250                 bus_dmamap_destroy(sc->sc_dstdmat, re->re_dst_map);
1251         }
1252         if (re->re_src_map != NULL) {
1253                 bus_dmamap_unload(sc->sc_srcdmat, re->re_src_map);
1254                 bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);
1255         }
1256         mtx_unlock(&sc->sc_ringmtx);
1257         if (err != ERESTART) {
1258                 crp->crp_etype = err;
1259                 crypto_done(crp);
1260                 err = 0;
1261         } else {
1262                 sc->sc_needwakeup |= CRYPTO_SYMQ;
1263         }
1264         return (err);
1265 }
1266
1267 static void
1268 safe_callback(struct safe_softc *sc, struct safe_ringentry *re)
1269 {
1270         const struct crypto_session_params *csp;
1271         struct cryptop *crp = (struct cryptop *)re->re_crp;
1272         struct safe_session *ses;
1273         uint8_t hash[HASH_MAX_LEN];
1274
1275         ses = crypto_get_driver_session(crp->crp_session);
1276         csp = crypto_get_params(crp->crp_session);
1277
1278         safestats.st_opackets++;
1279         safestats.st_obytes += re->re_dst.mapsize;
1280
1281         safe_dma_sync(&sc->sc_ringalloc,
1282                 BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
1283         if (re->re_desc.d_csr & SAFE_PE_CSR_STATUS) {
1284                 device_printf(sc->sc_dev, "csr 0x%x cmd0 0x%x cmd1 0x%x\n",
1285                         re->re_desc.d_csr,
1286                         re->re_sa.sa_cmd0, re->re_sa.sa_cmd1);
1287                 safestats.st_peoperr++;
1288                 crp->crp_etype = EIO;           /* something more meaningful? */
1289         }
1290
1291         /*
1292          * XXX: Should crp_buf.cb_mbuf be updated to re->re_dst_m if
1293          * it is non-NULL?
1294          */
1295
1296         if (re->re_dst_map != NULL && re->re_dst_map != re->re_src_map) {
1297                 bus_dmamap_sync(sc->sc_dstdmat, re->re_dst_map,
1298                     BUS_DMASYNC_POSTREAD);
1299                 bus_dmamap_unload(sc->sc_dstdmat, re->re_dst_map);
1300                 bus_dmamap_destroy(sc->sc_dstdmat, re->re_dst_map);
1301         }
1302         bus_dmamap_sync(sc->sc_srcdmat, re->re_src_map, BUS_DMASYNC_POSTWRITE);
1303         bus_dmamap_unload(sc->sc_srcdmat, re->re_src_map);
1304         bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);
1305
1306         if (re->re_flags & SAFE_QFLAGS_COPYOUTICV) {
1307                 if (csp->csp_auth_alg == CRYPTO_SHA1_HMAC) {
1308                         /*
1309                          * SHA-1 ICV's are byte-swapped; fix 'em up
1310                          * before copying them to their destination.
1311                          */
1312                         re->re_sastate.sa_saved_indigest[0] =
1313                             bswap32(re->re_sastate.sa_saved_indigest[0]);
1314                         re->re_sastate.sa_saved_indigest[1] =
1315                             bswap32(re->re_sastate.sa_saved_indigest[1]);
1316                         re->re_sastate.sa_saved_indigest[2] =
1317                             bswap32(re->re_sastate.sa_saved_indigest[2]);
1318                 }
1319
1320                 if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
1321                         crypto_copydata(crp, crp->crp_digest_start,
1322                             ses->ses_mlen, hash);
1323                         if (timingsafe_bcmp(re->re_sastate.sa_saved_indigest,
1324                             hash, ses->ses_mlen) != 0)
1325                                 crp->crp_etype = EBADMSG;
1326                 } else
1327                         crypto_copyback(crp, crp->crp_digest_start,
1328                             ses->ses_mlen, re->re_sastate.sa_saved_indigest);
1329         }
1330         crypto_done(crp);
1331 }
1332
1333 /*
1334  * Copy all data past offset from srcm to dstm.
1335  */
1336 static void
1337 safe_mcopy(struct mbuf *srcm, struct mbuf *dstm, u_int offset)
1338 {
1339         u_int j, dlen, slen;
1340         caddr_t dptr, sptr;
1341
1342         /*
1343          * Advance src and dst to offset.
1344          */
1345         j = offset;
1346         while (j >= srcm->m_len) {
1347                 j -= srcm->m_len;
1348                 srcm = srcm->m_next;
1349                 if (srcm == NULL)
1350                         return;
1351         }
1352         sptr = mtod(srcm, caddr_t) + j;
1353         slen = srcm->m_len - j;
1354
1355         j = offset;
1356         while (j >= dstm->m_len) {
1357                 j -= dstm->m_len;
1358                 dstm = dstm->m_next;
1359                 if (dstm == NULL)
1360                         return;
1361         }
1362         dptr = mtod(dstm, caddr_t) + j;
1363         dlen = dstm->m_len - j;
1364
1365         /*
1366          * Copy everything that remains.
1367          */
1368         for (;;) {
1369                 j = min(slen, dlen);
1370                 bcopy(sptr, dptr, j);
1371                 if (slen == j) {
1372                         srcm = srcm->m_next;
1373                         if (srcm == NULL)
1374                                 return;
1375                         sptr = srcm->m_data;
1376                         slen = srcm->m_len;
1377                 } else
1378                         sptr += j, slen -= j;
1379                 if (dlen == j) {
1380                         dstm = dstm->m_next;
1381                         if (dstm == NULL)
1382                                 return;
1383                         dptr = dstm->m_data;
1384                         dlen = dstm->m_len;
1385                 } else
1386                         dptr += j, dlen -= j;
1387         }
1388 }
1389
1390 #ifndef SAFE_NO_RNG
1391 #define SAFE_RNG_MAXWAIT        1000
1392
1393 static void
1394 safe_rng_init(struct safe_softc *sc)
1395 {
1396         u_int32_t w, v;
1397         int i;
1398
1399         WRITE_REG(sc, SAFE_RNG_CTRL, 0);
1400         /* use default value according to the manual */
1401         WRITE_REG(sc, SAFE_RNG_CNFG, 0x834);    /* magic from SafeNet */
1402         WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
1403
1404         /*
1405          * There is a bug in rev 1.0 of the 1140 that when the RNG
1406          * is brought out of reset the ready status flag does not
1407          * work until the RNG has finished its internal initialization.
1408          *
1409          * So in order to determine the device is through its
1410          * initialization we must read the data register, using the
1411          * status reg in the read in case it is initialized.  Then read
1412          * the data register until it changes from the first read.
1413          * Once it changes read the data register until it changes
1414          * again.  At this time the RNG is considered initialized.
1415          * This could take between 750ms - 1000ms in time.
1416          */
1417         i = 0;
1418         w = READ_REG(sc, SAFE_RNG_OUT);
1419         do {
1420                 v = READ_REG(sc, SAFE_RNG_OUT);
1421                 if (v != w) {
1422                         w = v;
1423                         break;
1424                 }
1425                 DELAY(10);
1426         } while (++i < SAFE_RNG_MAXWAIT);
1427
1428         /* Wait Until data changes again */
1429         i = 0;
1430         do {
1431                 v = READ_REG(sc, SAFE_RNG_OUT);
1432                 if (v != w)
1433                         break;
1434                 DELAY(10);
1435         } while (++i < SAFE_RNG_MAXWAIT);
1436 }
1437
1438 static __inline void
1439 safe_rng_disable_short_cycle(struct safe_softc *sc)
1440 {
1441         WRITE_REG(sc, SAFE_RNG_CTRL,
1442                 READ_REG(sc, SAFE_RNG_CTRL) &~ SAFE_RNG_CTRL_SHORTEN);
1443 }
1444
1445 static __inline void
1446 safe_rng_enable_short_cycle(struct safe_softc *sc)
1447 {
1448         WRITE_REG(sc, SAFE_RNG_CTRL,
1449                 READ_REG(sc, SAFE_RNG_CTRL) | SAFE_RNG_CTRL_SHORTEN);
1450 }
1451
1452 static __inline u_int32_t
1453 safe_rng_read(struct safe_softc *sc)
1454 {
1455         int i;
1456
1457         i = 0;
1458         while (READ_REG(sc, SAFE_RNG_STAT) != 0 && ++i < SAFE_RNG_MAXWAIT)
1459                 ;
1460         return READ_REG(sc, SAFE_RNG_OUT);
1461 }
1462
1463 static void
1464 safe_rng(void *arg)
1465 {
1466         struct safe_softc *sc = arg;
1467         u_int32_t buf[SAFE_RNG_MAXBUFSIZ];      /* NB: maybe move to softc */
1468         u_int maxwords;
1469         int i;
1470
1471         safestats.st_rng++;
1472         /*
1473          * Fetch the next block of data.
1474          */
1475         maxwords = safe_rngbufsize;
1476         if (maxwords > SAFE_RNG_MAXBUFSIZ)
1477                 maxwords = SAFE_RNG_MAXBUFSIZ;
1478 retry:
1479         for (i = 0; i < maxwords; i++)
1480                 buf[i] = safe_rng_read(sc);
1481         /*
1482          * Check the comparator alarm count and reset the h/w if
1483          * it exceeds our threshold.  This guards against the
1484          * hardware oscillators resonating with external signals.
1485          */
1486         if (READ_REG(sc, SAFE_RNG_ALM_CNT) > safe_rngmaxalarm) {
1487                 u_int32_t freq_inc, w;
1488
1489                 DPRINTF(("%s: alarm count %u exceeds threshold %u\n", __func__,
1490                         READ_REG(sc, SAFE_RNG_ALM_CNT), safe_rngmaxalarm));
1491                 safestats.st_rngalarm++;
1492                 safe_rng_enable_short_cycle(sc);
1493                 freq_inc = 18;
1494                 for (i = 0; i < 64; i++) {
1495                         w = READ_REG(sc, SAFE_RNG_CNFG);
1496                         freq_inc = ((w + freq_inc) & 0x3fL);
1497                         w = ((w & ~0x3fL) | freq_inc);
1498                         WRITE_REG(sc, SAFE_RNG_CNFG, w);
1499
1500                         WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
1501
1502                         (void) safe_rng_read(sc);
1503                         DELAY(25);
1504
1505                         if (READ_REG(sc, SAFE_RNG_ALM_CNT) == 0) {
1506                                 safe_rng_disable_short_cycle(sc);
1507                                 goto retry;
1508                         }
1509                         freq_inc = 1;
1510                 }
1511                 safe_rng_disable_short_cycle(sc);
1512         } else
1513                 WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
1514
1515         (*sc->sc_harvest)(sc->sc_rndtest, buf, maxwords*sizeof (u_int32_t));
1516         callout_reset(&sc->sc_rngto,
1517                 hz * (safe_rnginterval ? safe_rnginterval : 1), safe_rng, sc);
1518 }
1519 #endif /* SAFE_NO_RNG */
1520
1521 static void
1522 safe_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
1523 {
1524         bus_addr_t *paddr = (bus_addr_t*) arg;
1525         *paddr = segs->ds_addr;
1526 }
1527
1528 static int
1529 safe_dma_malloc(
1530         struct safe_softc *sc,
1531         bus_size_t size,
1532         struct safe_dma_alloc *dma,
1533         int mapflags
1534 )
1535 {
1536         int r;
1537
1538         r = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev),     /* parent */
1539                                sizeof(u_int32_t), 0,    /* alignment, bounds */
1540                                BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
1541                                BUS_SPACE_MAXADDR,       /* highaddr */
1542                                NULL, NULL,              /* filter, filterarg */
1543                                size,                    /* maxsize */
1544                                1,                       /* nsegments */
1545                                size,                    /* maxsegsize */
1546                                BUS_DMA_ALLOCNOW,        /* flags */
1547                                NULL, NULL,              /* locking */
1548                                &dma->dma_tag);
1549         if (r != 0) {
1550                 device_printf(sc->sc_dev, "safe_dma_malloc: "
1551                         "bus_dma_tag_create failed; error %u\n", r);
1552                 goto fail_0;
1553         }
1554
1555         r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
1556                              BUS_DMA_NOWAIT, &dma->dma_map);
1557         if (r != 0) {
1558                 device_printf(sc->sc_dev, "safe_dma_malloc: "
1559                         "bus_dmammem_alloc failed; size %ju, error %u\n",
1560                         (uintmax_t)size, r);
1561                 goto fail_1;
1562         }
1563
1564         r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
1565                             size,
1566                             safe_dmamap_cb,
1567                             &dma->dma_paddr,
1568                             mapflags | BUS_DMA_NOWAIT);
1569         if (r != 0) {
1570                 device_printf(sc->sc_dev, "safe_dma_malloc: "
1571                         "bus_dmamap_load failed; error %u\n", r);
1572                 goto fail_2;
1573         }
1574
1575         dma->dma_size = size;
1576         return (0);
1577
1578         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
1579 fail_2:
1580         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1581 fail_1:
1582         bus_dma_tag_destroy(dma->dma_tag);
1583 fail_0:
1584         dma->dma_tag = NULL;
1585         return (r);
1586 }
1587
1588 static void
1589 safe_dma_free(struct safe_softc *sc, struct safe_dma_alloc *dma)
1590 {
1591         bus_dmamap_unload(dma->dma_tag, dma->dma_map);
1592         bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
1593         bus_dma_tag_destroy(dma->dma_tag);
1594 }
1595
1596 /*
1597  * Resets the board.  Values in the regesters are left as is
1598  * from the reset (i.e. initial values are assigned elsewhere).
1599  */
1600 static void
1601 safe_reset_board(struct safe_softc *sc)
1602 {
1603         u_int32_t v;
1604         /*
1605          * Reset the device.  The manual says no delay
1606          * is needed between marking and clearing reset.
1607          */
1608         v = READ_REG(sc, SAFE_PE_DMACFG) &~
1609                 (SAFE_PE_DMACFG_PERESET | SAFE_PE_DMACFG_PDRRESET |
1610                  SAFE_PE_DMACFG_SGRESET);
1611         WRITE_REG(sc, SAFE_PE_DMACFG, v
1612                                     | SAFE_PE_DMACFG_PERESET
1613                                     | SAFE_PE_DMACFG_PDRRESET
1614                                     | SAFE_PE_DMACFG_SGRESET);
1615         WRITE_REG(sc, SAFE_PE_DMACFG, v);
1616 }
1617
1618 /*
1619  * Initialize registers we need to touch only once.
1620  */
1621 static void
1622 safe_init_board(struct safe_softc *sc)
1623 {
1624         u_int32_t v, dwords;
1625
1626         v = READ_REG(sc, SAFE_PE_DMACFG);
1627         v &=~ SAFE_PE_DMACFG_PEMODE;
1628         v |= SAFE_PE_DMACFG_FSENA               /* failsafe enable */
1629           |  SAFE_PE_DMACFG_GPRPCI              /* gather ring on PCI */
1630           |  SAFE_PE_DMACFG_SPRPCI              /* scatter ring on PCI */
1631           |  SAFE_PE_DMACFG_ESDESC              /* endian-swap descriptors */
1632           |  SAFE_PE_DMACFG_ESSA                /* endian-swap SA's */
1633           |  SAFE_PE_DMACFG_ESPDESC             /* endian-swap part. desc's */
1634           ;
1635         WRITE_REG(sc, SAFE_PE_DMACFG, v);
1636 #if 0
1637         /* XXX select byte swap based on host byte order */
1638         WRITE_REG(sc, SAFE_ENDIAN, 0x1b);
1639 #endif
1640         if (sc->sc_chiprev == SAFE_REV(1,0)) {
1641                 /*
1642                  * Avoid large PCI DMA transfers.  Rev 1.0 has a bug where
1643                  * "target mode transfers" done while the chip is DMA'ing
1644                  * >1020 bytes cause the hardware to lockup.  To avoid this
1645                  * we reduce the max PCI transfer size and use small source
1646                  * particle descriptors (<= 256 bytes).
1647                  */
1648                 WRITE_REG(sc, SAFE_DMA_CFG, 256);
1649                 device_printf(sc->sc_dev,
1650                         "Reduce max DMA size to %u words for rev %u.%u WAR\n",
1651                         (READ_REG(sc, SAFE_DMA_CFG)>>2) & 0xff,
1652                         SAFE_REV_MAJ(sc->sc_chiprev),
1653                         SAFE_REV_MIN(sc->sc_chiprev));
1654         }
1655
1656         /* NB: operands+results are overlaid */
1657         WRITE_REG(sc, SAFE_PE_PDRBASE, sc->sc_ringalloc.dma_paddr);
1658         WRITE_REG(sc, SAFE_PE_RDRBASE, sc->sc_ringalloc.dma_paddr);
1659         /*
1660          * Configure ring entry size and number of items in the ring.
1661          */
1662         KASSERT((sizeof(struct safe_ringentry) % sizeof(u_int32_t)) == 0,
1663                 ("PE ring entry not 32-bit aligned!"));
1664         dwords = sizeof(struct safe_ringentry) / sizeof(u_int32_t);
1665         WRITE_REG(sc, SAFE_PE_RINGCFG,
1666                 (dwords << SAFE_PE_RINGCFG_OFFSET_S) | SAFE_MAX_NQUEUE);
1667         WRITE_REG(sc, SAFE_PE_RINGPOLL, 0);     /* disable polling */
1668
1669         WRITE_REG(sc, SAFE_PE_GRNGBASE, sc->sc_spalloc.dma_paddr);
1670         WRITE_REG(sc, SAFE_PE_SRNGBASE, sc->sc_dpalloc.dma_paddr);
1671         WRITE_REG(sc, SAFE_PE_PARTSIZE,
1672                 (SAFE_TOTAL_DPART<<16) | SAFE_TOTAL_SPART);
1673         /*
1674          * NB: destination particles are fixed size.  We use
1675          *     an mbuf cluster and require all results go to
1676          *     clusters or smaller.
1677          */
1678         WRITE_REG(sc, SAFE_PE_PARTCFG, SAFE_MAX_DSIZE);
1679
1680         /* it's now safe to enable PE mode, do it */
1681         WRITE_REG(sc, SAFE_PE_DMACFG, v | SAFE_PE_DMACFG_PEMODE);
1682
1683         /*
1684          * Configure hardware to use level-triggered interrupts and
1685          * to interrupt after each descriptor is processed.
1686          */
1687         WRITE_REG(sc, SAFE_HI_CFG, SAFE_HI_CFG_LEVEL);
1688         WRITE_REG(sc, SAFE_HI_DESC_CNT, 1);
1689         WRITE_REG(sc, SAFE_HI_MASK, SAFE_INT_PE_DDONE | SAFE_INT_PE_ERROR);
1690 }
1691
1692 /*
1693  * Init PCI registers
1694  */
1695 static void
1696 safe_init_pciregs(device_t dev)
1697 {
1698 }
1699
1700 /*
1701  * Clean up after a chip crash.
1702  * It is assumed that the caller in splimp()
1703  */
1704 static void
1705 safe_cleanchip(struct safe_softc *sc)
1706 {
1707
1708         if (sc->sc_nqchip != 0) {
1709                 struct safe_ringentry *re = sc->sc_back;
1710
1711                 while (re != sc->sc_front) {
1712                         if (re->re_desc.d_csr != 0)
1713                                 safe_free_entry(sc, re);
1714                         if (++re == sc->sc_ringtop)
1715                                 re = sc->sc_ring;
1716                 }
1717                 sc->sc_back = re;
1718                 sc->sc_nqchip = 0;
1719         }
1720 }
1721
1722 /*
1723  * free a safe_q
1724  * It is assumed that the caller is within splimp().
1725  */
1726 static int
1727 safe_free_entry(struct safe_softc *sc, struct safe_ringentry *re)
1728 {
1729         struct cryptop *crp;
1730
1731         /*
1732          * Free header MCR
1733          */
1734         if (re->re_dst_m != NULL)
1735                 m_freem(re->re_dst_m);
1736
1737         crp = (struct cryptop *)re->re_crp;
1738
1739         re->re_desc.d_csr = 0;
1740
1741         crp->crp_etype = EFAULT;
1742         crypto_done(crp);
1743         return(0);
1744 }
1745
1746 /*
1747  * Routine to reset the chip and clean up.
1748  * It is assumed that the caller is in splimp()
1749  */
1750 static void
1751 safe_totalreset(struct safe_softc *sc)
1752 {
1753         safe_reset_board(sc);
1754         safe_init_board(sc);
1755         safe_cleanchip(sc);
1756 }
1757
1758 /*
1759  * Is the operand suitable aligned for direct DMA.  Each
1760  * segment must be aligned on a 32-bit boundary and all
1761  * but the last segment must be a multiple of 4 bytes.
1762  */
1763 static int
1764 safe_dmamap_aligned(const struct safe_operand *op)
1765 {
1766         int i;
1767
1768         for (i = 0; i < op->nsegs; i++) {
1769                 if (op->segs[i].ds_addr & 3)
1770                         return (0);
1771                 if (i != (op->nsegs - 1) && (op->segs[i].ds_len & 3))
1772                         return (0);
1773         }
1774         return (1);
1775 }
1776
1777 /*
1778  * Is the operand suitable for direct DMA as the destination
1779  * of an operation.  The hardware requires that each ``particle''
1780  * but the last in an operation result have the same size.  We
1781  * fix that size at SAFE_MAX_DSIZE bytes.  This routine returns
1782  * 0 if some segment is not a multiple of of this size, 1 if all
1783  * segments are exactly this size, or 2 if segments are at worst
1784  * a multple of this size.
1785  */
1786 static int
1787 safe_dmamap_uniform(const struct safe_operand *op)
1788 {
1789         int result = 1;
1790
1791         if (op->nsegs > 0) {
1792                 int i;
1793
1794                 for (i = 0; i < op->nsegs-1; i++) {
1795                         if (op->segs[i].ds_len % SAFE_MAX_DSIZE)
1796                                 return (0);
1797                         if (op->segs[i].ds_len != SAFE_MAX_DSIZE)
1798                                 result = 2;
1799                 }
1800         }
1801         return (result);
1802 }
1803
1804 #ifdef SAFE_DEBUG
1805 static void
1806 safe_dump_dmastatus(struct safe_softc *sc, const char *tag)
1807 {
1808         printf("%s: ENDIAN 0x%x SRC 0x%x DST 0x%x STAT 0x%x\n"
1809                 , tag
1810                 , READ_REG(sc, SAFE_DMA_ENDIAN)
1811                 , READ_REG(sc, SAFE_DMA_SRCADDR)
1812                 , READ_REG(sc, SAFE_DMA_DSTADDR)
1813                 , READ_REG(sc, SAFE_DMA_STAT)
1814         );
1815 }
1816
1817 static void
1818 safe_dump_intrstate(struct safe_softc *sc, const char *tag)
1819 {
1820         printf("%s: HI_CFG 0x%x HI_MASK 0x%x HI_DESC_CNT 0x%x HU_STAT 0x%x HM_STAT 0x%x\n"
1821                 , tag
1822                 , READ_REG(sc, SAFE_HI_CFG)
1823                 , READ_REG(sc, SAFE_HI_MASK)
1824                 , READ_REG(sc, SAFE_HI_DESC_CNT)
1825                 , READ_REG(sc, SAFE_HU_STAT)
1826                 , READ_REG(sc, SAFE_HM_STAT)
1827         );
1828 }
1829
1830 static void
1831 safe_dump_ringstate(struct safe_softc *sc, const char *tag)
1832 {
1833         u_int32_t estat = READ_REG(sc, SAFE_PE_ERNGSTAT);
1834
1835         /* NB: assume caller has lock on ring */
1836         printf("%s: ERNGSTAT %x (next %u) back %lu front %lu\n",
1837                 tag,
1838                 estat, (estat >> SAFE_PE_ERNGSTAT_NEXT_S),
1839                 (unsigned long)(sc->sc_back - sc->sc_ring),
1840                 (unsigned long)(sc->sc_front - sc->sc_ring));
1841 }
1842
1843 static void
1844 safe_dump_request(struct safe_softc *sc, const char* tag, struct safe_ringentry *re)
1845 {
1846         int ix, nsegs;
1847
1848         ix = re - sc->sc_ring;
1849         printf("%s: %p (%u): csr %x src %x dst %x sa %x len %x\n"
1850                 , tag
1851                 , re, ix
1852                 , re->re_desc.d_csr
1853                 , re->re_desc.d_src
1854                 , re->re_desc.d_dst
1855                 , re->re_desc.d_sa
1856                 , re->re_desc.d_len
1857         );
1858         if (re->re_src.nsegs > 1) {
1859                 ix = (re->re_desc.d_src - sc->sc_spalloc.dma_paddr) /
1860                         sizeof(struct safe_pdesc);
1861                 for (nsegs = re->re_src.nsegs; nsegs; nsegs--) {
1862                         printf(" spd[%u] %p: %p size %u flags %x"
1863                                 , ix, &sc->sc_spring[ix]
1864                                 , (caddr_t)(uintptr_t) sc->sc_spring[ix].pd_addr
1865                                 , sc->sc_spring[ix].pd_size
1866                                 , sc->sc_spring[ix].pd_flags
1867                         );
1868                         if (sc->sc_spring[ix].pd_size == 0)
1869                                 printf(" (zero!)");
1870                         printf("\n");
1871                         if (++ix == SAFE_TOTAL_SPART)
1872                                 ix = 0;
1873                 }
1874         }
1875         if (re->re_dst.nsegs > 1) {
1876                 ix = (re->re_desc.d_dst - sc->sc_dpalloc.dma_paddr) /
1877                         sizeof(struct safe_pdesc);
1878                 for (nsegs = re->re_dst.nsegs; nsegs; nsegs--) {
1879                         printf(" dpd[%u] %p: %p flags %x\n"
1880                                 , ix, &sc->sc_dpring[ix]
1881                                 , (caddr_t)(uintptr_t) sc->sc_dpring[ix].pd_addr
1882                                 , sc->sc_dpring[ix].pd_flags
1883                         );
1884                         if (++ix == SAFE_TOTAL_DPART)
1885                                 ix = 0;
1886                 }
1887         }
1888         printf("sa: cmd0 %08x cmd1 %08x staterec %x\n",
1889                 re->re_sa.sa_cmd0, re->re_sa.sa_cmd1, re->re_sa.sa_staterec);
1890         printf("sa: key %x %x %x %x %x %x %x %x\n"
1891                 , re->re_sa.sa_key[0]
1892                 , re->re_sa.sa_key[1]
1893                 , re->re_sa.sa_key[2]
1894                 , re->re_sa.sa_key[3]
1895                 , re->re_sa.sa_key[4]
1896                 , re->re_sa.sa_key[5]
1897                 , re->re_sa.sa_key[6]
1898                 , re->re_sa.sa_key[7]
1899         );
1900         printf("sa: indigest %x %x %x %x %x\n"
1901                 , re->re_sa.sa_indigest[0]
1902                 , re->re_sa.sa_indigest[1]
1903                 , re->re_sa.sa_indigest[2]
1904                 , re->re_sa.sa_indigest[3]
1905                 , re->re_sa.sa_indigest[4]
1906         );
1907         printf("sa: outdigest %x %x %x %x %x\n"
1908                 , re->re_sa.sa_outdigest[0]
1909                 , re->re_sa.sa_outdigest[1]
1910                 , re->re_sa.sa_outdigest[2]
1911                 , re->re_sa.sa_outdigest[3]
1912                 , re->re_sa.sa_outdigest[4]
1913         );
1914         printf("sr: iv %x %x %x %x\n"
1915                 , re->re_sastate.sa_saved_iv[0]
1916                 , re->re_sastate.sa_saved_iv[1]
1917                 , re->re_sastate.sa_saved_iv[2]
1918                 , re->re_sastate.sa_saved_iv[3]
1919         );
1920         printf("sr: hashbc %u indigest %x %x %x %x %x\n"
1921                 , re->re_sastate.sa_saved_hashbc
1922                 , re->re_sastate.sa_saved_indigest[0]
1923                 , re->re_sastate.sa_saved_indigest[1]
1924                 , re->re_sastate.sa_saved_indigest[2]
1925                 , re->re_sastate.sa_saved_indigest[3]
1926                 , re->re_sastate.sa_saved_indigest[4]
1927         );
1928 }
1929
1930 static void
1931 safe_dump_ring(struct safe_softc *sc, const char *tag)
1932 {
1933         mtx_lock(&sc->sc_ringmtx);
1934         printf("\nSafeNet Ring State:\n");
1935         safe_dump_intrstate(sc, tag);
1936         safe_dump_dmastatus(sc, tag);
1937         safe_dump_ringstate(sc, tag);
1938         if (sc->sc_nqchip) {
1939                 struct safe_ringentry *re = sc->sc_back;
1940                 do {
1941                         safe_dump_request(sc, tag, re);
1942                         if (++re == sc->sc_ringtop)
1943                                 re = sc->sc_ring;
1944                 } while (re != sc->sc_front);
1945         }
1946         mtx_unlock(&sc->sc_ringmtx);
1947 }
1948
1949 static int
1950 sysctl_hw_safe_dump(SYSCTL_HANDLER_ARGS)
1951 {
1952         char dmode[64];
1953         int error;
1954
1955         strncpy(dmode, "", sizeof(dmode) - 1);
1956         dmode[sizeof(dmode) - 1] = '\0';
1957         error = sysctl_handle_string(oidp, &dmode[0], sizeof(dmode), req);
1958
1959         if (error == 0 && req->newptr != NULL) {
1960                 struct safe_softc *sc = safec;
1961
1962                 if (!sc)
1963                         return EINVAL;
1964                 if (strncmp(dmode, "dma", 3) == 0)
1965                         safe_dump_dmastatus(sc, "safe0");
1966                 else if (strncmp(dmode, "int", 3) == 0)
1967                         safe_dump_intrstate(sc, "safe0");
1968                 else if (strncmp(dmode, "ring", 4) == 0)
1969                         safe_dump_ring(sc, "safe0");
1970                 else
1971                         return EINVAL;
1972         }
1973         return error;
1974 }
1975 SYSCTL_PROC(_hw_safe, OID_AUTO, dump,
1976     CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 0, 0,
1977     sysctl_hw_safe_dump, "A",
1978     "Dump driver state");
1979 #endif /* SAFE_DEBUG */