sys/dev/hifn/hifn7751.c

   1 /*      $OpenBSD: hifn7751.c,v 1.120 2002/05/17 00:33:34 deraadt Exp $  */
   2
   3 /*-
   4  * Invertex AEON / Hifn 7751 driver
   5  * Copyright (c) 1999 Invertex Inc. All rights reserved.
   6  * Copyright (c) 1999 Theo de Raadt
   7  * Copyright (c) 2000-2001 Network Security Technologies, Inc.
   8  *                      http://www.netsec.net
   9  * Copyright (c) 2003 Hifn Inc.
  10  *
  11  * This driver is based on a previous driver by Invertex, for which they
  12  * requested:  Please send any comments, feedback, bug-fixes, or feature
  13  * requests to software@invertex.com.
  14  *
  15  * Redistribution and use in source and binary forms, with or without
  16  * modification, are permitted provided that the following conditions
  17  * are met:
  18  *
  19  * 1. Redistributions of source code must retain the above copyright
  20  *   notice, this list of conditions and the following disclaimer.
  21  * 2. Redistributions in binary form must reproduce the above copyright
  22  *   notice, this list of conditions and the following disclaimer in the
  23  *   documentation and/or other materials provided with the distribution.
  24  * 3. The name of the author may not be used to endorse or promote products
  25  *   derived from this software without specific prior written permission.
  26  *
  27  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  28  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  29  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  30  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  31  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  32  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  33  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  34  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  35  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  36  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  37  *
  38  * Effort sponsored in part by the Defense Advanced Research Projects
  39  * Agency (DARPA) and Air Force Research Laboratory, Air Force
  40  * Materiel Command, USAF, under agreement number F30602-01-2-0537.
  41  */
  42
  43 #include <sys/cdefs.h>
  44 __FBSDID("$FreeBSD$");
  45
  46 /*
  47  * Driver for various Hifn encryption processors.
  48  */
  49 #include "opt_hifn.h"
  50
  51 #include <sys/param.h>
  52 #include <sys/systm.h>
  53 #include <sys/proc.h>
  54 #include <sys/errno.h>
  55 #include <sys/malloc.h>
  56 #include <sys/kernel.h>
  57 #include <sys/module.h>
  58 #include <sys/mbuf.h>
  59 #include <sys/lock.h>
  60 #include <sys/mutex.h>
  61 #include <sys/sysctl.h>
  62
  63 #include <vm/vm.h>
  64 #include <vm/pmap.h>
  65
  66 #include <machine/bus.h>
  67 #include <machine/resource.h>
  68 #include <sys/bus.h>
  69 #include <sys/rman.h>
  70
  71 #include <opencrypto/cryptodev.h>
  72 #include <sys/random.h>
  73 #include <sys/kobj.h>
  74
  75 #include "cryptodev_if.h"
  76
  77 #include <dev/pci/pcivar.h>
  78 #include <dev/pci/pcireg.h>
  79
  80 #ifdef HIFN_RNDTEST
  81 #include <dev/rndtest/rndtest.h>
  82 #endif
  83 #include <dev/hifn/hifn7751reg.h>
  84 #include <dev/hifn/hifn7751var.h>
  85
  86 #ifdef HIFN_VULCANDEV
  87 #include <sys/conf.h>
  88 #include <sys/uio.h>
  89
  90 static struct cdevsw vulcanpk_cdevsw; /* forward declaration */
  91 #endif
  92
  93 /*
  94  * Prototypes and count for the pci_device structure
  95  */
  96 static  int hifn_probe(device_t);
  97 static  int hifn_attach(device_t);
  98 static  int hifn_detach(device_t);
  99 static  int hifn_suspend(device_t);
 100 static  int hifn_resume(device_t);
 101 static  int hifn_shutdown(device_t);
 102
 103 static  int hifn_newsession(device_t, u_int32_t *, struct cryptoini *);
 104 static  int hifn_freesession(device_t, u_int64_t);
 105 static  int hifn_process(device_t, struct cryptop *, int);
 106
 107 static device_method_t hifn_methods[] = {
 108         /* Device interface */
 109         DEVMETHOD(device_probe,         hifn_probe),
 110         DEVMETHOD(device_attach,        hifn_attach),
 111         DEVMETHOD(device_detach,        hifn_detach),
 112         DEVMETHOD(device_suspend,       hifn_suspend),
 113         DEVMETHOD(device_resume,        hifn_resume),
 114         DEVMETHOD(device_shutdown,      hifn_shutdown),
 115
 116         /* bus interface */
 117         DEVMETHOD(bus_print_child,      bus_generic_print_child),
 118         DEVMETHOD(bus_driver_added,     bus_generic_driver_added),
 119
 120         /* crypto device methods */
 121         DEVMETHOD(cryptodev_newsession, hifn_newsession),
 122         DEVMETHOD(cryptodev_freesession,hifn_freesession),
 123         DEVMETHOD(cryptodev_process,    hifn_process),
 124
 125         { 0, 0 }
 126 };
 127 static driver_t hifn_driver = {
 128         "hifn",
 129         hifn_methods,
 130         sizeof (struct hifn_softc)
 131 };
 132 static devclass_t hifn_devclass;
 133
 134 DRIVER_MODULE(hifn, pci, hifn_driver, hifn_devclass, 0, 0);
 135 MODULE_DEPEND(hifn, crypto, 1, 1, 1);
 136 #ifdef HIFN_RNDTEST
 137 MODULE_DEPEND(hifn, rndtest, 1, 1, 1);
 138 #endif
 139
 140 static  void hifn_reset_board(struct hifn_softc *, int);
 141 static  void hifn_reset_puc(struct hifn_softc *);
 142 static  void hifn_puc_wait(struct hifn_softc *);
 143 static  int hifn_enable_crypto(struct hifn_softc *);
 144 static  void hifn_set_retry(struct hifn_softc *sc);
 145 static  void hifn_init_dma(struct hifn_softc *);
 146 static  void hifn_init_pci_registers(struct hifn_softc *);
 147 static  int hifn_sramsize(struct hifn_softc *);
 148 static  int hifn_dramsize(struct hifn_softc *);
 149 static  int hifn_ramtype(struct hifn_softc *);
 150 static  void hifn_sessions(struct hifn_softc *);
 151 static  void hifn_intr(void *);
 152 static  u_int hifn_write_command(struct hifn_command *, u_int8_t *);
 153 static  u_int32_t hifn_next_signature(u_int32_t a, u_int cnt);
 154 static  void hifn_callback(struct hifn_softc *, struct hifn_command *, u_int8_t *);
 155 static  int hifn_crypto(struct hifn_softc *, struct hifn_command *, struct cryptop *, int);
 156 static  int hifn_readramaddr(struct hifn_softc *, int, u_int8_t *);
 157 static  int hifn_writeramaddr(struct hifn_softc *, int, u_int8_t *);
 158 static  int hifn_dmamap_load_src(struct hifn_softc *, struct hifn_command *);
 159 static  int hifn_dmamap_load_dst(struct hifn_softc *, struct hifn_command *);
 160 static  int hifn_init_pubrng(struct hifn_softc *);
 161 static  void hifn_rng(void *);
 162 static  void hifn_tick(void *);
 163 static  void hifn_abort(struct hifn_softc *);
 164 static  void hifn_alloc_slot(struct hifn_softc *, int *, int *, int *, int *);
 165
 166 static  void hifn_write_reg_0(struct hifn_softc *, bus_size_t, u_int32_t);
 167 static  void hifn_write_reg_1(struct hifn_softc *, bus_size_t, u_int32_t);
 168
 169 static __inline u_int32_t
 170 READ_REG_0(struct hifn_softc *sc, bus_size_t reg)
 171 {
 172     u_int32_t v = bus_space_read_4(sc->sc_st0, sc->sc_sh0, reg);
 173     sc->sc_bar0_lastreg = (bus_size_t) -1;
 174     return (v);
 175 }
 176 #define WRITE_REG_0(sc, reg, val)       hifn_write_reg_0(sc, reg, val)
 177
 178 static __inline u_int32_t
 179 READ_REG_1(struct hifn_softc *sc, bus_size_t reg)
 180 {
 181     u_int32_t v = bus_space_read_4(sc->sc_st1, sc->sc_sh1, reg);
 182     sc->sc_bar1_lastreg = (bus_size_t) -1;
 183     return (v);
 184 }
 185 #define WRITE_REG_1(sc, reg, val)       hifn_write_reg_1(sc, reg, val)
 186
 187 SYSCTL_NODE(_hw, OID_AUTO, hifn, CTLFLAG_RD, 0, "Hifn driver parameters");
 188
 189 #ifdef HIFN_DEBUG
 190 static  int hifn_debug = 0;
 191 SYSCTL_INT(_hw_hifn, OID_AUTO, debug, CTLFLAG_RW, &hifn_debug,
 192             0, "control debugging msgs");
 193 #endif
 194
 195 static  struct hifn_stats hifnstats;
 196 SYSCTL_STRUCT(_hw_hifn, OID_AUTO, stats, CTLFLAG_RD, &hifnstats,
 197             hifn_stats, "driver statistics");
 198 static  int hifn_maxbatch = 1;
 199 SYSCTL_INT(_hw_hifn, OID_AUTO, maxbatch, CTLFLAG_RW, &hifn_maxbatch,
 200             0, "max ops to batch w/o interrupt");
 201
 202 /*
 203  * Probe for a supported device.  The PCI vendor and device
 204  * IDs are used to detect devices we know how to handle.
 205  */
 206 static int
 207 hifn_probe(device_t dev)
 208 {
 209         if (pci_get_vendor(dev) == PCI_VENDOR_INVERTEX &&
 210             pci_get_device(dev) == PCI_PRODUCT_INVERTEX_AEON)
 211                 return (BUS_PROBE_DEFAULT);
 212         if (pci_get_vendor(dev) == PCI_VENDOR_HIFN &&
 213             (pci_get_device(dev) == PCI_PRODUCT_HIFN_7751 ||
 214              pci_get_device(dev) == PCI_PRODUCT_HIFN_7951 ||
 215              pci_get_device(dev) == PCI_PRODUCT_HIFN_7955 ||
 216              pci_get_device(dev) == PCI_PRODUCT_HIFN_7956 ||
 217              pci_get_device(dev) == PCI_PRODUCT_HIFN_7811))
 218                 return (BUS_PROBE_DEFAULT);
 219         if (pci_get_vendor(dev) == PCI_VENDOR_NETSEC &&
 220             pci_get_device(dev) == PCI_PRODUCT_NETSEC_7751)
 221                 return (BUS_PROBE_DEFAULT);
 222         return (ENXIO);
 223 }
 224
 225 static void
 226 hifn_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 227 {
 228         bus_addr_t *paddr = (bus_addr_t*) arg;
 229         *paddr = segs->ds_addr;
 230 }
 231
 232 static const char*
 233 hifn_partname(struct hifn_softc *sc)
 234 {
 235         /* XXX sprintf numbers when not decoded */
 236         switch (pci_get_vendor(sc->sc_dev)) {
 237         case PCI_VENDOR_HIFN:
 238                 switch (pci_get_device(sc->sc_dev)) {
 239                 case PCI_PRODUCT_HIFN_6500:     return "Hifn 6500";
 240                 case PCI_PRODUCT_HIFN_7751:     return "Hifn 7751";
 241                 case PCI_PRODUCT_HIFN_7811:     return "Hifn 7811";
 242                 case PCI_PRODUCT_HIFN_7951:     return "Hifn 7951";
 243                 case PCI_PRODUCT_HIFN_7955:     return "Hifn 7955";
 244                 case PCI_PRODUCT_HIFN_7956:     return "Hifn 7956";
 245                 }
 246                 return "Hifn unknown-part";
 247         case PCI_VENDOR_INVERTEX:
 248                 switch (pci_get_device(sc->sc_dev)) {
 249                 case PCI_PRODUCT_INVERTEX_AEON: return "Invertex AEON";
 250                 }
 251                 return "Invertex unknown-part";
 252         case PCI_VENDOR_NETSEC:
 253                 switch (pci_get_device(sc->sc_dev)) {
 254                 case PCI_PRODUCT_NETSEC_7751:   return "NetSec 7751";
 255                 }
 256                 return "NetSec unknown-part";
 257         }
 258         return "Unknown-vendor unknown-part";
 259 }
 260
 261 static void
 262 default_harvest(struct rndtest_state *rsp, void *buf, u_int count)
 263 {
 264         random_harvest(buf, count, count*NBBY, 0, RANDOM_PURE);
 265 }
 266
 267 static u_int
 268 checkmaxmin(device_t dev, const char *what, u_int v, u_int min, u_int max)
 269 {
 270         if (v > max) {
 271                 device_printf(dev, "Warning, %s %u out of range, "
 272                         "using max %u\n", what, v, max);
 273                 v = max;
 274         } else if (v < min) {
 275                 device_printf(dev, "Warning, %s %u out of range, "
 276                         "using min %u\n", what, v, min);
 277                 v = min;
 278         }
 279         return v;
 280 }
 281
 282 /*
 283  * Select PLL configuration for 795x parts.  This is complicated in
 284  * that we cannot determine the optimal parameters without user input.
 285  * The reference clock is derived from an external clock through a
 286  * multiplier.  The external clock is either the host bus (i.e. PCI)
 287  * or an external clock generator.  When using the PCI bus we assume
 288  * the clock is either 33 or 66 MHz; for an external source we cannot
 289  * tell the speed.
 290  *
 291  * PLL configuration is done with a string: "pci" for PCI bus, or "ext"
 292  * for an external source, followed by the frequency.  We calculate
 293  * the appropriate multiplier and PLL register contents accordingly.
 294  * When no configuration is given we default to "pci66" since that
 295  * always will allow the card to work.  If a card is using the PCI
 296  * bus clock and in a 33MHz slot then it will be operating at half
 297  * speed until the correct information is provided.
 298  *
 299  * We use a default setting of "ext66" because according to Mike Ham
 300  * of HiFn, almost every board in existence has an external crystal
 301  * populated at 66Mhz. Using PCI can be a problem on modern motherboards,
 302  * because PCI33 can have clocks from 0 to 33Mhz, and some have
 303  * non-PCI-compliant spread-spectrum clocks, which can confuse the pll.
 304  */
 305 static void
 306 hifn_getpllconfig(device_t dev, u_int *pll)
 307 {
 308         const char *pllspec;
 309         u_int freq, mul, fl, fh;
 310         u_int32_t pllconfig;
 311         char *nxt;
 312
 313         if (resource_string_value("hifn", device_get_unit(dev),
 314             "pllconfig", &pllspec))
 315                 pllspec = "ext66";
 316         fl = 33, fh = 66;
 317         pllconfig = 0;
 318         if (strncmp(pllspec, "ext", 3) == 0) {
 319                 pllspec += 3;
 320                 pllconfig |= HIFN_PLL_REF_SEL;
 321                 switch (pci_get_device(dev)) {
 322                 case PCI_PRODUCT_HIFN_7955:
 323                 case PCI_PRODUCT_HIFN_7956:
 324                         fl = 20, fh = 100;
 325                         break;
 326 #ifdef notyet
 327                 case PCI_PRODUCT_HIFN_7954:
 328                         fl = 20, fh = 66;
 329                         break;
 330 #endif
 331                 }
 332         } else if (strncmp(pllspec, "pci", 3) == 0)
 333                 pllspec += 3;
 334         freq = strtoul(pllspec, &nxt, 10);
 335         if (nxt == pllspec)
 336                 freq = 66;
 337         else
 338                 freq = checkmaxmin(dev, "frequency", freq, fl, fh);
 339         /*
 340          * Calculate multiplier.  We target a Fck of 266 MHz,
 341          * allowing only even values, possibly rounded down.
 342          * Multipliers > 8 must set the charge pump current.
 343          */
 344         mul = checkmaxmin(dev, "PLL divisor", (266 / freq) &~ 1, 2, 12);
 345         pllconfig |= (mul / 2 - 1) << HIFN_PLL_ND_SHIFT;
 346         if (mul > 8)
 347                 pllconfig |= HIFN_PLL_IS;
 348         *pll = pllconfig;
 349 }
 350
 351 /*
 352  * Attach an interface that successfully probed.
 353  */
 354 static int
 355 hifn_attach(device_t dev)
 356 {
 357         struct hifn_softc *sc = device_get_softc(dev);
 358         u_int32_t cmd;
 359         caddr_t kva;
 360         int rseg, rid;
 361         char rbase;
 362         u_int16_t ena, rev;
 363
 364         KASSERT(sc != NULL, ("hifn_attach: null software carrier!"));
 365         bzero(sc, sizeof (*sc));
 366         sc->sc_dev = dev;
 367
 368         mtx_init(&sc->sc_mtx, device_get_nameunit(dev), "hifn driver", MTX_DEF);
 369
 370         /* XXX handle power management */
 371
 372         /*
 373          * The 7951 and 795x have a random number generator and
 374          * public key support; note this.
 375          */
 376         if (pci_get_vendor(dev) == PCI_VENDOR_HIFN &&
 377             (pci_get_device(dev) == PCI_PRODUCT_HIFN_7951 ||
 378              pci_get_device(dev) == PCI_PRODUCT_HIFN_7955 ||
 379              pci_get_device(dev) == PCI_PRODUCT_HIFN_7956))
 380                 sc->sc_flags = HIFN_HAS_RNG | HIFN_HAS_PUBLIC;
 381         /*
 382          * The 7811 has a random number generator and
 383          * we also note it's identity 'cuz of some quirks.
 384          */
 385         if (pci_get_vendor(dev) == PCI_VENDOR_HIFN &&
 386             pci_get_device(dev) == PCI_PRODUCT_HIFN_7811)
 387                 sc->sc_flags |= HIFN_IS_7811 | HIFN_HAS_RNG;
 388
 389         /*
 390          * The 795x parts support AES.
 391          */
 392         if (pci_get_vendor(dev) == PCI_VENDOR_HIFN &&
 393             (pci_get_device(dev) == PCI_PRODUCT_HIFN_7955 ||
 394              pci_get_device(dev) == PCI_PRODUCT_HIFN_7956)) {
 395                 sc->sc_flags |= HIFN_IS_7956 | HIFN_HAS_AES;
 396                 /*
 397                  * Select PLL configuration.  This depends on the
 398                  * bus and board design and must be manually configured
 399                  * if the default setting is unacceptable.
 400                  */
 401                 hifn_getpllconfig(dev, &sc->sc_pllconfig);
 402         }
 403
 404         /*
 405          * Configure support for memory-mapped access to
 406          * registers and for DMA operations.
 407          */
 408 #define PCIM_ENA        (PCIM_CMD_MEMEN|PCIM_CMD_BUSMASTEREN)
 409         cmd = pci_read_config(dev, PCIR_COMMAND, 4);
 410         cmd |= PCIM_ENA;
 411         pci_write_config(dev, PCIR_COMMAND, cmd, 4);
 412         cmd = pci_read_config(dev, PCIR_COMMAND, 4);
 413         if ((cmd & PCIM_ENA) != PCIM_ENA) {
 414                 device_printf(dev, "failed to enable %s\n",
 415                         (cmd & PCIM_ENA) == 0 ?
 416                                 "memory mapping & bus mastering" :
 417                         (cmd & PCIM_CMD_MEMEN) == 0 ?
 418                                 "memory mapping" : "bus mastering");
 419                 goto fail_pci;
 420         }
 421 #undef PCIM_ENA
 422
 423         /*
 424          * Setup PCI resources. Note that we record the bus
 425          * tag and handle for each register mapping, this is
 426          * used by the READ_REG_0, WRITE_REG_0, READ_REG_1,
 427          * and WRITE_REG_1 macros throughout the driver.
 428          */
 429         rid = HIFN_BAR0;
 430         sc->sc_bar0res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 431                                                 RF_ACTIVE);
 432         if (sc->sc_bar0res == NULL) {
 433                 device_printf(dev, "cannot map bar%d register space\n", 0);
 434                 goto fail_pci;
 435         }
 436         sc->sc_st0 = rman_get_bustag(sc->sc_bar0res);
 437         sc->sc_sh0 = rman_get_bushandle(sc->sc_bar0res);
 438         sc->sc_bar0_lastreg = (bus_size_t) -1;
 439
 440         rid = HIFN_BAR1;
 441         sc->sc_bar1res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 442                                                 RF_ACTIVE);
 443         if (sc->sc_bar1res == NULL) {
 444                 device_printf(dev, "cannot map bar%d register space\n", 1);
 445                 goto fail_io0;
 446         }
 447         sc->sc_st1 = rman_get_bustag(sc->sc_bar1res);
 448         sc->sc_sh1 = rman_get_bushandle(sc->sc_bar1res);
 449         sc->sc_bar1_lastreg = (bus_size_t) -1;
 450
 451         hifn_set_retry(sc);
 452
 453         /*
 454          * Setup the area where the Hifn DMA's descriptors
 455          * and associated data structures.
 456          */
 457         if (bus_dma_tag_create(NULL,                    /* parent */
 458                                1, 0,                    /* alignment,boundary */
 459                                BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
 460                                BUS_SPACE_MAXADDR,       /* highaddr */
 461                                NULL, NULL,              /* filter, filterarg */
 462                                HIFN_MAX_DMALEN,         /* maxsize */
 463                                MAX_SCATTER,             /* nsegments */
 464                                HIFN_MAX_SEGLEN,         /* maxsegsize */
 465                                BUS_DMA_ALLOCNOW,        /* flags */
 466                                NULL,                    /* lockfunc */
 467                                NULL,                    /* lockarg */
 468                                &sc->sc_dmat)) {
 469                 device_printf(dev, "cannot allocate DMA tag\n");
 470                 goto fail_io1;
 471         }
 472         if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &sc->sc_dmamap)) {
 473                 device_printf(dev, "cannot create dma map\n");
 474                 bus_dma_tag_destroy(sc->sc_dmat);
 475                 goto fail_io1;
 476         }
 477         if (bus_dmamem_alloc(sc->sc_dmat, (void**) &kva, BUS_DMA_NOWAIT, &sc->sc_dmamap)) {
 478                 device_printf(dev, "cannot alloc dma buffer\n");
 479                 bus_dmamap_destroy(sc->sc_dmat, sc->sc_dmamap);
 480                 bus_dma_tag_destroy(sc->sc_dmat);
 481                 goto fail_io1;
 482         }
 483         if (bus_dmamap_load(sc->sc_dmat, sc->sc_dmamap, kva,
 484                              sizeof (*sc->sc_dma),
 485                              hifn_dmamap_cb, &sc->sc_dma_physaddr,
 486                              BUS_DMA_NOWAIT)) {
 487                 device_printf(dev, "cannot load dma map\n");
 488                 bus_dmamem_free(sc->sc_dmat, kva, sc->sc_dmamap);
 489                 bus_dmamap_destroy(sc->sc_dmat, sc->sc_dmamap);
 490                 bus_dma_tag_destroy(sc->sc_dmat);
 491                 goto fail_io1;
 492         }
 493         sc->sc_dma = (struct hifn_dma *)kva;
 494         bzero(sc->sc_dma, sizeof(*sc->sc_dma));
 495
 496         KASSERT(sc->sc_st0 != 0, ("hifn_attach: null bar0 tag!"));
 497         KASSERT(sc->sc_sh0 != 0, ("hifn_attach: null bar0 handle!"));
 498         KASSERT(sc->sc_st1 != 0, ("hifn_attach: null bar1 tag!"));
 499         KASSERT(sc->sc_sh1 != 0, ("hifn_attach: null bar1 handle!"));
 500
 501         /*
 502          * Reset the board and do the ``secret handshake''
 503          * to enable the crypto support.  Then complete the
 504          * initialization procedure by setting up the interrupt
 505          * and hooking in to the system crypto support so we'll
 506          * get used for system services like the crypto device,
 507          * IPsec, RNG device, etc.
 508          */
 509         hifn_reset_board(sc, 0);
 510
 511         if (hifn_enable_crypto(sc) != 0) {
 512                 device_printf(dev, "crypto enabling failed\n");
 513                 goto fail_mem;
 514         }
 515         hifn_reset_puc(sc);
 516
 517         hifn_init_dma(sc);
 518         hifn_init_pci_registers(sc);
 519
 520         /* XXX can't dynamically determine ram type for 795x; force dram */
 521         if (sc->sc_flags & HIFN_IS_7956)
 522                 sc->sc_drammodel = 1;
 523         else if (hifn_ramtype(sc))
 524                 goto fail_mem;
 525
 526         if (sc->sc_drammodel == 0)
 527                 hifn_sramsize(sc);
 528         else
 529                 hifn_dramsize(sc);
 530
 531         /*
 532          * Workaround for NetSec 7751 rev A: half ram size because two
 533          * of the address lines were left floating
 534          */
 535         if (pci_get_vendor(dev) == PCI_VENDOR_NETSEC &&
 536             pci_get_device(dev) == PCI_PRODUCT_NETSEC_7751 &&
 537             pci_get_revid(dev) == 0x61) /*XXX???*/
 538                 sc->sc_ramsize >>= 1;
 539
 540         /*
 541          * Arrange the interrupt line.
 542          */
 543         rid = 0;
 544         sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 545                                             RF_SHAREABLE|RF_ACTIVE);
 546         if (sc->sc_irq == NULL) {
 547                 device_printf(dev, "could not map interrupt\n");
 548                 goto fail_mem;
 549         }
 550         /*
 551          * NB: Network code assumes we are blocked with splimp()
 552          *     so make sure the IRQ is marked appropriately.
 553          */
 554         if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE,
 555                            NULL, hifn_intr, sc, &sc->sc_intrhand)) {
 556                 device_printf(dev, "could not setup interrupt\n");
 557                 goto fail_intr2;
 558         }
 559
 560         hifn_sessions(sc);
 561
 562         /*
 563          * NB: Keep only the low 16 bits; this masks the chip id
 564          *     from the 7951.
 565          */
 566         rev = READ_REG_1(sc, HIFN_1_REVID) & 0xffff;
 567
 568         rseg = sc->sc_ramsize / 1024;
 569         rbase = 'K';
 570         if (sc->sc_ramsize >= (1024 * 1024)) {
 571                 rbase = 'M';
 572                 rseg /= 1024;
 573         }
 574         device_printf(sc->sc_dev, "%s, rev %u, %d%cB %cram",
 575                 hifn_partname(sc), rev,
 576                 rseg, rbase, sc->sc_drammodel ? 'd' : 's');
 577         if (sc->sc_flags & HIFN_IS_7956)
 578                 printf(", pll=0x%x<%s clk, %ux mult>",
 579                         sc->sc_pllconfig,
 580                         sc->sc_pllconfig & HIFN_PLL_REF_SEL ? "ext" : "pci",
 581                         2 + 2*((sc->sc_pllconfig & HIFN_PLL_ND) >> 11));
 582         printf("\n");
 583
 584         sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE);
 585         if (sc->sc_cid < 0) {
 586                 device_printf(dev, "could not get crypto driver id\n");
 587                 goto fail_intr;
 588         }
 589
 590         WRITE_REG_0(sc, HIFN_0_PUCNFG,
 591             READ_REG_0(sc, HIFN_0_PUCNFG) | HIFN_PUCNFG_CHIPID);
 592         ena = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
 593
 594         switch (ena) {
 595         case HIFN_PUSTAT_ENA_2:
 596                 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
 597                 crypto_register(sc->sc_cid, CRYPTO_ARC4, 0, 0);
 598                 if (sc->sc_flags & HIFN_HAS_AES)
 599                         crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
 600                 /*FALLTHROUGH*/
 601         case HIFN_PUSTAT_ENA_1:
 602                 crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0);
 603                 crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0);
 604                 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
 605                 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
 606                 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
 607                 break;
 608         }
 609
 610         bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
 611             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 612
 613         if (sc->sc_flags & (HIFN_HAS_PUBLIC | HIFN_HAS_RNG))
 614                 hifn_init_pubrng(sc);
 615
 616         callout_init(&sc->sc_tickto, CALLOUT_MPSAFE);
 617         callout_reset(&sc->sc_tickto, hz, hifn_tick, sc);
 618
 619         return (0);
 620
 621 fail_intr:
 622         bus_teardown_intr(dev, sc->sc_irq, sc->sc_intrhand);
 623 fail_intr2:
 624         /* XXX don't store rid */
 625         bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
 626 fail_mem:
 627         bus_dmamap_unload(sc->sc_dmat, sc->sc_dmamap);
 628         bus_dmamem_free(sc->sc_dmat, sc->sc_dma, sc->sc_dmamap);
 629         bus_dmamap_destroy(sc->sc_dmat, sc->sc_dmamap);
 630         bus_dma_tag_destroy(sc->sc_dmat);
 631
 632         /* Turn off DMA polling */
 633         WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
 634             HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
 635 fail_io1:
 636         bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR1, sc->sc_bar1res);
 637 fail_io0:
 638         bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR0, sc->sc_bar0res);
 639 fail_pci:
 640         mtx_destroy(&sc->sc_mtx);
 641         return (ENXIO);
 642 }
 643
 644 /*
 645  * Detach an interface that successfully probed.
 646  */
 647 static int
 648 hifn_detach(device_t dev)
 649 {
 650         struct hifn_softc *sc = device_get_softc(dev);
 651
 652         KASSERT(sc != NULL, ("hifn_detach: null software carrier!"));
 653
 654         /* disable interrupts */
 655         WRITE_REG_1(sc, HIFN_1_DMA_IER, 0);
 656
 657         /*XXX other resources */
 658         callout_stop(&sc->sc_tickto);
 659         callout_stop(&sc->sc_rngto);
 660 #ifdef HIFN_RNDTEST
 661         if (sc->sc_rndtest)
 662                 rndtest_detach(sc->sc_rndtest);
 663 #endif
 664
 665         /* Turn off DMA polling */
 666         WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
 667             HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
 668
 669         crypto_unregister_all(sc->sc_cid);
 670
 671         bus_generic_detach(dev);        /*XXX should be no children, right? */
 672
 673         bus_teardown_intr(dev, sc->sc_irq, sc->sc_intrhand);
 674         /* XXX don't store rid */
 675         bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
 676
 677         bus_dmamap_unload(sc->sc_dmat, sc->sc_dmamap);
 678         bus_dmamem_free(sc->sc_dmat, sc->sc_dma, sc->sc_dmamap);
 679         bus_dmamap_destroy(sc->sc_dmat, sc->sc_dmamap);
 680         bus_dma_tag_destroy(sc->sc_dmat);
 681
 682         bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR1, sc->sc_bar1res);
 683         bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR0, sc->sc_bar0res);
 684
 685         mtx_destroy(&sc->sc_mtx);
 686
 687         return (0);
 688 }
 689
 690 /*
 691  * Stop all chip I/O so that the kernel's probe routines don't
 692  * get confused by errant DMAs when rebooting.
 693  */
 694 static int
 695 hifn_shutdown(device_t dev)
 696 {
 697 #ifdef notyet
 698         hifn_stop(device_get_softc(dev));
 699 #endif
 700         return (0);
 701 }
 702
 703 /*
 704  * Device suspend routine.  Stop the interface and save some PCI
 705  * settings in case the BIOS doesn't restore them properly on
 706  * resume.
 707  */
 708 static int
 709 hifn_suspend(device_t dev)
 710 {
 711         struct hifn_softc *sc = device_get_softc(dev);
 712 #ifdef notyet
 713         hifn_stop(sc);
 714 #endif
 715         sc->sc_suspended = 1;
 716
 717         return (0);
 718 }
 719
 720 /*
 721  * Device resume routine.  Restore some PCI settings in case the BIOS
 722  * doesn't, re-enable busmastering, and restart the interface if
 723  * appropriate.
 724  */
 725 static int
 726 hifn_resume(device_t dev)
 727 {
 728         struct hifn_softc *sc = device_get_softc(dev);
 729 #ifdef notyet
 730         /* reenable busmastering */
 731         pci_enable_busmaster(dev);
 732         pci_enable_io(dev, HIFN_RES);
 733
 734         /* reinitialize interface if necessary */
 735         if (ifp->if_flags & IFF_UP)
 736                 rl_init(sc);
 737 #endif
 738         sc->sc_suspended = 0;
 739
 740         return (0);
 741 }
 742
 743 static int
 744 hifn_init_pubrng(struct hifn_softc *sc)
 745 {
 746         u_int32_t r;
 747         int i;
 748
 749 #ifdef HIFN_RNDTEST
 750         sc->sc_rndtest = rndtest_attach(sc->sc_dev);
 751         if (sc->sc_rndtest)
 752                 sc->sc_harvest = rndtest_harvest;
 753         else
 754                 sc->sc_harvest = default_harvest;
 755 #else
 756         sc->sc_harvest = default_harvest;
 757 #endif
 758         if ((sc->sc_flags & HIFN_IS_7811) == 0) {
 759                 /* Reset 7951 public key/rng engine */
 760                 WRITE_REG_1(sc, HIFN_1_PUB_RESET,
 761                     READ_REG_1(sc, HIFN_1_PUB_RESET) | HIFN_PUBRST_RESET);
 762
 763                 for (i = 0; i < 100; i++) {
 764                         DELAY(1000);
 765                         if ((READ_REG_1(sc, HIFN_1_PUB_RESET) &
 766                             HIFN_PUBRST_RESET) == 0)
 767                                 break;
 768                 }
 769
 770                 if (i == 100) {
 771                         device_printf(sc->sc_dev, "public key init failed\n");
 772                         return (1);
 773                 }
 774         }
 775
 776         /* Enable the rng, if available */
 777         if (sc->sc_flags & HIFN_HAS_RNG) {
 778                 if (sc->sc_flags & HIFN_IS_7811) {
 779                         r = READ_REG_1(sc, HIFN_1_7811_RNGENA);
 780                         if (r & HIFN_7811_RNGENA_ENA) {
 781                                 r &= ~HIFN_7811_RNGENA_ENA;
 782                                 WRITE_REG_1(sc, HIFN_1_7811_RNGENA, r);
 783                         }
 784                         WRITE_REG_1(sc, HIFN_1_7811_RNGCFG,
 785                             HIFN_7811_RNGCFG_DEFL);
 786                         r |= HIFN_7811_RNGENA_ENA;
 787                         WRITE_REG_1(sc, HIFN_1_7811_RNGENA, r);
 788                 } else
 789                         WRITE_REG_1(sc, HIFN_1_RNG_CONFIG,
 790                             READ_REG_1(sc, HIFN_1_RNG_CONFIG) |
 791                             HIFN_RNGCFG_ENA);
 792
 793                 sc->sc_rngfirst = 1;
 794                 if (hz >= 100)
 795                         sc->sc_rnghz = hz / 100;
 796                 else
 797                         sc->sc_rnghz = 1;
 798                 callout_init(&sc->sc_rngto, CALLOUT_MPSAFE);
 799                 callout_reset(&sc->sc_rngto, sc->sc_rnghz, hifn_rng, sc);
 800         }
 801
 802         /* Enable public key engine, if available */
 803         if (sc->sc_flags & HIFN_HAS_PUBLIC) {
 804                 WRITE_REG_1(sc, HIFN_1_PUB_IEN, HIFN_PUBIEN_DONE);
 805                 sc->sc_dmaier |= HIFN_DMAIER_PUBDONE;
 806                 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
 807 #ifdef HIFN_VULCANDEV
 808                 sc->sc_pkdev = make_dev(&vulcanpk_cdevsw, 0,
 809                                         UID_ROOT, GID_WHEEL, 0666,
 810                                         "vulcanpk");
 811                 sc->sc_pkdev->si_drv1 = sc;
 812 #endif
 813         }
 814
 815         return (0);
 816 }
 817
 818 static void
 819 hifn_rng(void *vsc)
 820 {
 821 #define RANDOM_BITS(n)  (n)*sizeof (u_int32_t), (n)*sizeof (u_int32_t)*NBBY, 0
 822         struct hifn_softc *sc = vsc;
 823         u_int32_t sts, num[2];
 824         int i;
 825
 826         if (sc->sc_flags & HIFN_IS_7811) {
 827                 /* ONLY VALID ON 7811!!!! */
 828                 for (i = 0; i < 5; i++) {
 829                         sts = READ_REG_1(sc, HIFN_1_7811_RNGSTS);
 830                         if (sts & HIFN_7811_RNGSTS_UFL) {
 831                                 device_printf(sc->sc_dev,
 832                                               "RNG underflow: disabling\n");
 833                                 return;
 834                         }
 835                         if ((sts & HIFN_7811_RNGSTS_RDY) == 0)
 836                                 break;
 837
 838                         /*
 839                          * There are at least two words in the RNG FIFO
 840                          * at this point.
 841                          */
 842                         num[0] = READ_REG_1(sc, HIFN_1_7811_RNGDAT);
 843                         num[1] = READ_REG_1(sc, HIFN_1_7811_RNGDAT);
 844                         /* NB: discard first data read */
 845                         if (sc->sc_rngfirst)
 846                                 sc->sc_rngfirst = 0;
 847                         else
 848                                 (*sc->sc_harvest)(sc->sc_rndtest,
 849                                         num, sizeof (num));
 850                 }
 851         } else {
 852                 num[0] = READ_REG_1(sc, HIFN_1_RNG_DATA);
 853
 854                 /* NB: discard first data read */
 855                 if (sc->sc_rngfirst)
 856                         sc->sc_rngfirst = 0;
 857                 else
 858                         (*sc->sc_harvest)(sc->sc_rndtest,
 859                                 num, sizeof (num[0]));
 860         }
 861
 862         callout_reset(&sc->sc_rngto, sc->sc_rnghz, hifn_rng, sc);
 863 #undef RANDOM_BITS
 864 }
 865
 866 static void
 867 hifn_puc_wait(struct hifn_softc *sc)
 868 {
 869         int i;
 870         int reg = HIFN_0_PUCTRL;
 871
 872         if (sc->sc_flags & HIFN_IS_7956) {
 873                 reg = HIFN_0_PUCTRL2;
 874         }
 875
 876         for (i = 5000; i > 0; i--) {
 877                 DELAY(1);
 878                 if (!(READ_REG_0(sc, reg) & HIFN_PUCTRL_RESET))
 879                         break;
 880         }
 881         if (!i)
 882                 device_printf(sc->sc_dev, "proc unit did not reset\n");
 883 }
 884
 885 /*
 886  * Reset the processing unit.
 887  */
 888 static void
 889 hifn_reset_puc(struct hifn_softc *sc)
 890 {
 891         /* Reset processing unit */
 892         int reg = HIFN_0_PUCTRL;
 893
 894         if (sc->sc_flags & HIFN_IS_7956) {
 895                 reg = HIFN_0_PUCTRL2;
 896         }
 897         WRITE_REG_0(sc, reg, HIFN_PUCTRL_DMAENA);
 898
 899         hifn_puc_wait(sc);
 900 }
 901
 902 /*
 903  * Set the Retry and TRDY registers; note that we set them to
 904  * zero because the 7811 locks up when forced to retry (section
 905  * 3.6 of "Specification Update SU-0014-04".  Not clear if we
 906  * should do this for all Hifn parts, but it doesn't seem to hurt.
 907  */
 908 static void
 909 hifn_set_retry(struct hifn_softc *sc)
 910 {
 911         /* NB: RETRY only responds to 8-bit reads/writes */
 912         pci_write_config(sc->sc_dev, HIFN_RETRY_TIMEOUT, 0, 1);
 913         pci_write_config(sc->sc_dev, HIFN_TRDY_TIMEOUT, 0, 4);
 914 }
 915
 916 /*
 917  * Resets the board.  Values in the regesters are left as is
 918  * from the reset (i.e. initial values are assigned elsewhere).
 919  */
 920 static void
 921 hifn_reset_board(struct hifn_softc *sc, int full)
 922 {
 923         u_int32_t reg;
 924
 925         /*
 926          * Set polling in the DMA configuration register to zero.  0x7 avoids
 927          * resetting the board and zeros out the other fields.
 928          */
 929         WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
 930             HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
 931
 932         /*
 933          * Now that polling has been disabled, we have to wait 1 ms
 934          * before resetting the board.
 935          */
 936         DELAY(1000);
 937
 938         /* Reset the DMA unit */
 939         if (full) {
 940                 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE);
 941                 DELAY(1000);
 942         } else {
 943                 WRITE_REG_1(sc, HIFN_1_DMA_CNFG,
 944                     HIFN_DMACNFG_MODE | HIFN_DMACNFG_MSTRESET);
 945                 hifn_reset_puc(sc);
 946         }
 947
 948         KASSERT(sc->sc_dma != NULL, ("hifn_reset_board: null DMA tag!"));
 949         bzero(sc->sc_dma, sizeof(*sc->sc_dma));
 950
 951         /* Bring dma unit out of reset */
 952         WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
 953             HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
 954
 955         hifn_puc_wait(sc);
 956         hifn_set_retry(sc);
 957
 958         if (sc->sc_flags & HIFN_IS_7811) {
 959                 for (reg = 0; reg < 1000; reg++) {
 960                         if (READ_REG_1(sc, HIFN_1_7811_MIPSRST) &
 961                             HIFN_MIPSRST_CRAMINIT)
 962                                 break;
 963                         DELAY(1000);
 964                 }
 965                 if (reg == 1000)
 966                         printf(": cram init timeout\n");
 967         } else {
 968           /* set up DMA configuration register #2 */
 969           /* turn off all PK and BAR0 swaps */
 970           WRITE_REG_1(sc, HIFN_1_DMA_CNFG2,
 971                       (3 << HIFN_DMACNFG2_INIT_WRITE_BURST_SHIFT)|
 972                       (3 << HIFN_DMACNFG2_INIT_READ_BURST_SHIFT)|
 973                       (2 << HIFN_DMACNFG2_TGT_WRITE_BURST_SHIFT)|
 974                       (2 << HIFN_DMACNFG2_TGT_READ_BURST_SHIFT));
 975         }
 976
 977 }
 978
 979 static u_int32_t
 980 hifn_next_signature(u_int32_t a, u_int cnt)
 981 {
 982         int i;
 983         u_int32_t v;
 984
 985         for (i = 0; i < cnt; i++) {
 986
 987                 /* get the parity */
 988                 v = a & 0x80080125;
 989                 v ^= v >> 16;
 990                 v ^= v >> 8;
 991                 v ^= v >> 4;
 992                 v ^= v >> 2;
 993                 v ^= v >> 1;
 994
 995                 a = (v & 1) ^ (a << 1);
 996         }
 997
 998         return a;
 999 }
1000
1001 struct pci2id {
1002         u_short         pci_vendor;
1003         u_short         pci_prod;
1004         char            card_id[13];
1005 };
1006 static struct pci2id pci2id[] = {
1007         {
1008                 PCI_VENDOR_HIFN,
1009                 PCI_PRODUCT_HIFN_7951,
1010                 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1011                   0x00, 0x00, 0x00, 0x00, 0x00 }
1012         }, {
1013                 PCI_VENDOR_HIFN,
1014                 PCI_PRODUCT_HIFN_7955,
1015                 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1016                   0x00, 0x00, 0x00, 0x00, 0x00 }
1017         }, {
1018                 PCI_VENDOR_HIFN,
1019                 PCI_PRODUCT_HIFN_7956,
1020                 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1021                   0x00, 0x00, 0x00, 0x00, 0x00 }
1022         }, {
1023                 PCI_VENDOR_NETSEC,
1024                 PCI_PRODUCT_NETSEC_7751,
1025                 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1026                   0x00, 0x00, 0x00, 0x00, 0x00 }
1027         }, {
1028                 PCI_VENDOR_INVERTEX,
1029                 PCI_PRODUCT_INVERTEX_AEON,
1030                 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1031                   0x00, 0x00, 0x00, 0x00, 0x00 }
1032         }, {
1033                 PCI_VENDOR_HIFN,
1034                 PCI_PRODUCT_HIFN_7811,
1035                 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1036                   0x00, 0x00, 0x00, 0x00, 0x00 }
1037         }, {
1038                 /*
1039                  * Other vendors share this PCI ID as well, such as
1040                  * http://www.powercrypt.com, and obviously they also
1041                  * use the same key.
1042                  */
1043                 PCI_VENDOR_HIFN,
1044                 PCI_PRODUCT_HIFN_7751,
1045                 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1046                   0x00, 0x00, 0x00, 0x00, 0x00 }
1047         },
1048 };
1049
1050 /*
1051  * Checks to see if crypto is already enabled.  If crypto isn't enable,
1052  * "hifn_enable_crypto" is called to enable it.  The check is important,
1053  * as enabling crypto twice will lock the board.
1054  */
1055 static int
1056 hifn_enable_crypto(struct hifn_softc *sc)
1057 {
1058         u_int32_t dmacfg, ramcfg, encl, addr, i;
1059         char *offtbl = NULL;
1060
1061         for (i = 0; i < sizeof(pci2id)/sizeof(pci2id[0]); i++) {
1062                 if (pci2id[i].pci_vendor == pci_get_vendor(sc->sc_dev) &&
1063                     pci2id[i].pci_prod == pci_get_device(sc->sc_dev)) {
1064                         offtbl = pci2id[i].card_id;
1065                         break;
1066                 }
1067         }
1068         if (offtbl == NULL) {
1069                 device_printf(sc->sc_dev, "Unknown card!\n");
1070                 return (1);
1071         }
1072
1073         ramcfg = READ_REG_0(sc, HIFN_0_PUCNFG);
1074         dmacfg = READ_REG_1(sc, HIFN_1_DMA_CNFG);
1075
1076         /*
1077          * The RAM config register's encrypt level bit needs to be set before
1078          * every read performed on the encryption level register.
1079          */
1080         WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg | HIFN_PUCNFG_CHIPID);
1081
1082         encl = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
1083
1084         /*
1085          * Make sure we don't re-unlock.  Two unlocks kills chip until the
1086          * next reboot.
1087          */
1088         if (encl == HIFN_PUSTAT_ENA_1 || encl == HIFN_PUSTAT_ENA_2) {
1089 #ifdef HIFN_DEBUG
1090                 if (hifn_debug)
1091                         device_printf(sc->sc_dev,
1092                             "Strong crypto already enabled!\n");
1093 #endif
1094                 goto report;
1095         }
1096
1097         if (encl != 0 && encl != HIFN_PUSTAT_ENA_0) {
1098 #ifdef HIFN_DEBUG
1099                 if (hifn_debug)
1100                         device_printf(sc->sc_dev,
1101                               "Unknown encryption level 0x%x\n", encl);
1102 #endif
1103                 return 1;
1104         }
1105
1106         WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_UNLOCK |
1107             HIFN_DMACNFG_MSTRESET | HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
1108         DELAY(1000);
1109         addr = READ_REG_1(sc, HIFN_UNLOCK_SECRET1);
1110         DELAY(1000);
1111         WRITE_REG_1(sc, HIFN_UNLOCK_SECRET2, 0);
1112         DELAY(1000);
1113
1114         for (i = 0; i <= 12; i++) {
1115                 addr = hifn_next_signature(addr, offtbl[i] + 0x101);
1116                 WRITE_REG_1(sc, HIFN_UNLOCK_SECRET2, addr);
1117
1118                 DELAY(1000);
1119         }
1120
1121         WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg | HIFN_PUCNFG_CHIPID);
1122         encl = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
1123
1124 #ifdef HIFN_DEBUG
1125         if (hifn_debug) {
1126                 if (encl != HIFN_PUSTAT_ENA_1 && encl != HIFN_PUSTAT_ENA_2)
1127                         device_printf(sc->sc_dev, "Engine is permanently "
1128                                 "locked until next system reset!\n");
1129                 else
1130                         device_printf(sc->sc_dev, "Engine enabled "
1131                                 "successfully!\n");
1132         }
1133 #endif
1134
1135 report:
1136         WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg);
1137         WRITE_REG_1(sc, HIFN_1_DMA_CNFG, dmacfg);
1138
1139         switch (encl) {
1140         case HIFN_PUSTAT_ENA_1:
1141         case HIFN_PUSTAT_ENA_2:
1142                 break;
1143         case HIFN_PUSTAT_ENA_0:
1144         default:
1145                 device_printf(sc->sc_dev, "disabled");
1146                 break;
1147         }
1148
1149         return 0;
1150 }
1151
1152 /*
1153  * Give initial values to the registers listed in the "Register Space"
1154  * section of the HIFN Software Development reference manual.
1155  */
1156 static void
1157 hifn_init_pci_registers(struct hifn_softc *sc)
1158 {
1159         /* write fixed values needed by the Initialization registers */
1160         WRITE_REG_0(sc, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);
1161         WRITE_REG_0(sc, HIFN_0_FIFOCNFG, HIFN_FIFOCNFG_THRESHOLD);
1162         WRITE_REG_0(sc, HIFN_0_PUIER, HIFN_PUIER_DSTOVER);
1163
1164         /* write all 4 ring address registers */
1165         WRITE_REG_1(sc, HIFN_1_DMA_CRAR, sc->sc_dma_physaddr +
1166             offsetof(struct hifn_dma, cmdr[0]));
1167         WRITE_REG_1(sc, HIFN_1_DMA_SRAR, sc->sc_dma_physaddr +
1168             offsetof(struct hifn_dma, srcr[0]));
1169         WRITE_REG_1(sc, HIFN_1_DMA_DRAR, sc->sc_dma_physaddr +
1170             offsetof(struct hifn_dma, dstr[0]));
1171         WRITE_REG_1(sc, HIFN_1_DMA_RRAR, sc->sc_dma_physaddr +
1172             offsetof(struct hifn_dma, resr[0]));
1173
1174         DELAY(2000);
1175
1176         /* write status register */
1177         WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1178             HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS |
1179             HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS |
1180             HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST |
1181             HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER |
1182             HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST |
1183             HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER |
1184             HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST |
1185             HIFN_DMACSR_S_WAIT |
1186             HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST |
1187             HIFN_DMACSR_C_WAIT |
1188             HIFN_DMACSR_ENGINE |
1189             ((sc->sc_flags & HIFN_HAS_PUBLIC) ?
1190                 HIFN_DMACSR_PUBDONE : 0) |
1191             ((sc->sc_flags & HIFN_IS_7811) ?
1192                 HIFN_DMACSR_ILLW | HIFN_DMACSR_ILLR : 0));
1193
1194         sc->sc_d_busy = sc->sc_r_busy = sc->sc_s_busy = sc->sc_c_busy = 0;
1195         sc->sc_dmaier |= HIFN_DMAIER_R_DONE | HIFN_DMAIER_C_ABORT |
1196             HIFN_DMAIER_D_OVER | HIFN_DMAIER_R_OVER |
1197             HIFN_DMAIER_S_ABORT | HIFN_DMAIER_D_ABORT | HIFN_DMAIER_R_ABORT |
1198             ((sc->sc_flags & HIFN_IS_7811) ?
1199                 HIFN_DMAIER_ILLW | HIFN_DMAIER_ILLR : 0);
1200         sc->sc_dmaier &= ~HIFN_DMAIER_C_WAIT;
1201         WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
1202
1203
1204         if (sc->sc_flags & HIFN_IS_7956) {
1205                 u_int32_t pll;
1206
1207                 WRITE_REG_0(sc, HIFN_0_PUCNFG, HIFN_PUCNFG_COMPSING |
1208                     HIFN_PUCNFG_TCALLPHASES |
1209                     HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32);
1210
1211                 /* turn off the clocks and insure bypass is set */
1212                 pll = READ_REG_1(sc, HIFN_1_PLL);
1213                 pll = (pll &~ (HIFN_PLL_PK_CLK_SEL | HIFN_PLL_PE_CLK_SEL))
1214                   | HIFN_PLL_BP | HIFN_PLL_MBSET;
1215                 WRITE_REG_1(sc, HIFN_1_PLL, pll);
1216                 DELAY(10*1000);         /* 10ms */
1217
1218                 /* change configuration */
1219                 pll = (pll &~ HIFN_PLL_CONFIG) | sc->sc_pllconfig;
1220                 WRITE_REG_1(sc, HIFN_1_PLL, pll);
1221                 DELAY(10*1000);         /* 10ms */
1222
1223                 /* disable bypass */
1224                 pll &= ~HIFN_PLL_BP;
1225                 WRITE_REG_1(sc, HIFN_1_PLL, pll);
1226                 /* enable clocks with new configuration */
1227                 pll |= HIFN_PLL_PK_CLK_SEL | HIFN_PLL_PE_CLK_SEL;
1228                 WRITE_REG_1(sc, HIFN_1_PLL, pll);
1229         } else {
1230                 WRITE_REG_0(sc, HIFN_0_PUCNFG, HIFN_PUCNFG_COMPSING |
1231                     HIFN_PUCNFG_DRFR_128 | HIFN_PUCNFG_TCALLPHASES |
1232                     HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32 |
1233                     (sc->sc_drammodel ? HIFN_PUCNFG_DRAM : HIFN_PUCNFG_SRAM));
1234         }
1235
1236         WRITE_REG_0(sc, HIFN_0_PUISR, HIFN_PUISR_DSTOVER);
1237         WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
1238             HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE | HIFN_DMACNFG_LAST |
1239             ((HIFN_POLL_FREQUENCY << 16 ) & HIFN_DMACNFG_POLLFREQ) |
1240             ((HIFN_POLL_SCALAR << 8) & HIFN_DMACNFG_POLLINVAL));
1241 }
1242
1243 /*
1244  * The maximum number of sessions supported by the card
1245  * is dependent on the amount of context ram, which
1246  * encryption algorithms are enabled, and how compression
1247  * is configured.  This should be configured before this
1248  * routine is called.
1249  */
1250 static void
1251 hifn_sessions(struct hifn_softc *sc)
1252 {
1253         u_int32_t pucnfg;
1254         int ctxsize;
1255
1256         pucnfg = READ_REG_0(sc, HIFN_0_PUCNFG);
1257
1258         if (pucnfg & HIFN_PUCNFG_COMPSING) {
1259                 if (pucnfg & HIFN_PUCNFG_ENCCNFG)
1260                         ctxsize = 128;
1261                 else
1262                         ctxsize = 512;
1263                 /*
1264                  * 7955/7956 has internal context memory of 32K
1265                  */
1266                 if (sc->sc_flags & HIFN_IS_7956)
1267                         sc->sc_maxses = 32768 / ctxsize;
1268                 else
1269                         sc->sc_maxses = 1 +
1270                             ((sc->sc_ramsize - 32768) / ctxsize);
1271         } else
1272                 sc->sc_maxses = sc->sc_ramsize / 16384;
1273
1274         if (sc->sc_maxses > 2048)
1275                 sc->sc_maxses = 2048;
1276 }
1277
1278 /*
1279  * Determine ram type (sram or dram).  Board should be just out of a reset
1280  * state when this is called.
1281  */
1282 static int
1283 hifn_ramtype(struct hifn_softc *sc)
1284 {
1285         u_int8_t data[8], dataexpect[8];
1286         int i;
1287
1288         for (i = 0; i < sizeof(data); i++)
1289                 data[i] = dataexpect[i] = 0x55;
1290         if (hifn_writeramaddr(sc, 0, data))
1291                 return (-1);
1292         if (hifn_readramaddr(sc, 0, data))
1293                 return (-1);
1294         if (bcmp(data, dataexpect, sizeof(data)) != 0) {
1295                 sc->sc_drammodel = 1;
1296                 return (0);
1297         }
1298
1299         for (i = 0; i < sizeof(data); i++)
1300                 data[i] = dataexpect[i] = 0xaa;
1301         if (hifn_writeramaddr(sc, 0, data))
1302                 return (-1);
1303         if (hifn_readramaddr(sc, 0, data))
1304                 return (-1);
1305         if (bcmp(data, dataexpect, sizeof(data)) != 0) {
1306                 sc->sc_drammodel = 1;
1307                 return (0);
1308         }
1309
1310         return (0);
1311 }
1312
1313 #define HIFN_SRAM_MAX           (32 << 20)
1314 #define HIFN_SRAM_STEP_SIZE     16384
1315 #define HIFN_SRAM_GRANULARITY   (HIFN_SRAM_MAX / HIFN_SRAM_STEP_SIZE)
1316
1317 static int
1318 hifn_sramsize(struct hifn_softc *sc)
1319 {
1320         u_int32_t a;
1321         u_int8_t data[8];
1322         u_int8_t dataexpect[sizeof(data)];
1323         int32_t i;
1324
1325         for (i = 0; i < sizeof(data); i++)
1326                 data[i] = dataexpect[i] = i ^ 0x5a;
1327
1328         for (i = HIFN_SRAM_GRANULARITY - 1; i >= 0; i--) {
1329                 a = i * HIFN_SRAM_STEP_SIZE;
1330                 bcopy(&i, data, sizeof(i));
1331                 hifn_writeramaddr(sc, a, data);
1332         }
1333
1334         for (i = 0; i < HIFN_SRAM_GRANULARITY; i++) {
1335                 a = i * HIFN_SRAM_STEP_SIZE;
1336                 bcopy(&i, dataexpect, sizeof(i));
1337                 if (hifn_readramaddr(sc, a, data) < 0)
1338                         return (0);
1339                 if (bcmp(data, dataexpect, sizeof(data)) != 0)
1340                         return (0);
1341                 sc->sc_ramsize = a + HIFN_SRAM_STEP_SIZE;
1342         }
1343
1344         return (0);
1345 }
1346
1347 /*
1348  * XXX For dram boards, one should really try all of the
1349  * HIFN_PUCNFG_DSZ_*'s.  This just assumes that PUCNFG
1350  * is already set up correctly.
1351  */
1352 static int
1353 hifn_dramsize(struct hifn_softc *sc)
1354 {
1355         u_int32_t cnfg;
1356
1357         if (sc->sc_flags & HIFN_IS_7956) {
1358                 /*
1359                  * 7955/7956 have a fixed internal ram of only 32K.
1360                  */
1361                 sc->sc_ramsize = 32768;
1362         } else {
1363                 cnfg = READ_REG_0(sc, HIFN_0_PUCNFG) &
1364                     HIFN_PUCNFG_DRAMMASK;
1365                 sc->sc_ramsize = 1 << ((cnfg >> 13) + 18);
1366         }
1367         return (0);
1368 }
1369
1370 static void
1371 hifn_alloc_slot(struct hifn_softc *sc, int *cmdp, int *srcp, int *dstp, int *resp)
1372 {
1373         struct hifn_dma *dma = sc->sc_dma;
1374
1375         if (dma->cmdi == HIFN_D_CMD_RSIZE) {
1376                 dma->cmdi = 0;
1377                 dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID |
1378                     HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1379                 HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE,
1380                     BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1381         }
1382         *cmdp = dma->cmdi++;
1383         dma->cmdk = dma->cmdi;
1384
1385         if (dma->srci == HIFN_D_SRC_RSIZE) {
1386                 dma->srci = 0;
1387                 dma->srcr[HIFN_D_SRC_RSIZE].l = htole32(HIFN_D_VALID |
1388                     HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1389                 HIFN_SRCR_SYNC(sc, HIFN_D_SRC_RSIZE,
1390                     BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1391         }
1392         *srcp = dma->srci++;
1393         dma->srck = dma->srci;
1394
1395         if (dma->dsti == HIFN_D_DST_RSIZE) {
1396                 dma->dsti = 0;
1397                 dma->dstr[HIFN_D_DST_RSIZE].l = htole32(HIFN_D_VALID |
1398                     HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1399                 HIFN_DSTR_SYNC(sc, HIFN_D_DST_RSIZE,
1400                     BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1401         }
1402         *dstp = dma->dsti++;
1403         dma->dstk = dma->dsti;
1404
1405         if (dma->resi == HIFN_D_RES_RSIZE) {
1406                 dma->resi = 0;
1407                 dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID |
1408                     HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1409                 HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE,
1410                     BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1411         }
1412         *resp = dma->resi++;
1413         dma->resk = dma->resi;
1414 }
1415
1416 static int
1417 hifn_writeramaddr(struct hifn_softc *sc, int addr, u_int8_t *data)
1418 {
1419         struct hifn_dma *dma = sc->sc_dma;
1420         hifn_base_command_t wc;
1421         const u_int32_t masks = HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ;
1422         int r, cmdi, resi, srci, dsti;
1423
1424         wc.masks = htole16(3 << 13);
1425         wc.session_num = htole16(addr >> 14);
1426         wc.total_source_count = htole16(8);
1427         wc.total_dest_count = htole16(addr & 0x3fff);
1428
1429         hifn_alloc_slot(sc, &cmdi, &srci, &dsti, &resi);
1430
1431         WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1432             HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA |
1433             HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA);
1434
1435         /* build write command */
1436         bzero(dma->command_bufs[cmdi], HIFN_MAX_COMMAND);
1437         *(hifn_base_command_t *)dma->command_bufs[cmdi] = wc;
1438         bcopy(data, &dma->test_src, sizeof(dma->test_src));
1439
1440         dma->srcr[srci].p = htole32(sc->sc_dma_physaddr
1441             + offsetof(struct hifn_dma, test_src));
1442         dma->dstr[dsti].p = htole32(sc->sc_dma_physaddr
1443             + offsetof(struct hifn_dma, test_dst));
1444
1445         dma->cmdr[cmdi].l = htole32(16 | masks);
1446         dma->srcr[srci].l = htole32(8 | masks);
1447         dma->dstr[dsti].l = htole32(4 | masks);
1448         dma->resr[resi].l = htole32(4 | masks);
1449
1450         bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1451             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1452
1453         for (r = 10000; r >= 0; r--) {
1454                 DELAY(10);
1455                 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1456                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1457                 if ((dma->resr[resi].l & htole32(HIFN_D_VALID)) == 0)
1458                         break;
1459                 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1460                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1461         }
1462         if (r == 0) {
1463                 device_printf(sc->sc_dev, "writeramaddr -- "
1464                     "result[%d](addr %d) still valid\n", resi, addr);
1465                 r = -1;
1466                 return (-1);
1467         } else
1468                 r = 0;
1469
1470         WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1471             HIFN_DMACSR_C_CTRL_DIS | HIFN_DMACSR_S_CTRL_DIS |
1472             HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS);
1473
1474         return (r);
1475 }
1476
1477 static int
1478 hifn_readramaddr(struct hifn_softc *sc, int addr, u_int8_t *data)
1479 {
1480         struct hifn_dma *dma = sc->sc_dma;
1481         hifn_base_command_t rc;
1482         const u_int32_t masks = HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ;
1483         int r, cmdi, srci, dsti, resi;
1484
1485         rc.masks = htole16(2 << 13);
1486         rc.session_num = htole16(addr >> 14);
1487         rc.total_source_count = htole16(addr & 0x3fff);
1488         rc.total_dest_count = htole16(8);
1489
1490         hifn_alloc_slot(sc, &cmdi, &srci, &dsti, &resi);
1491
1492         WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1493             HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA |
1494             HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA);
1495
1496         bzero(dma->command_bufs[cmdi], HIFN_MAX_COMMAND);
1497         *(hifn_base_command_t *)dma->command_bufs[cmdi] = rc;
1498
1499         dma->srcr[srci].p = htole32(sc->sc_dma_physaddr +
1500             offsetof(struct hifn_dma, test_src));
1501         dma->test_src = 0;
1502         dma->dstr[dsti].p =  htole32(sc->sc_dma_physaddr +
1503             offsetof(struct hifn_dma, test_dst));
1504         dma->test_dst = 0;
1505         dma->cmdr[cmdi].l = htole32(8 | masks);
1506         dma->srcr[srci].l = htole32(8 | masks);
1507         dma->dstr[dsti].l = htole32(8 | masks);
1508         dma->resr[resi].l = htole32(HIFN_MAX_RESULT | masks);
1509
1510         bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1511             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1512
1513         for (r = 10000; r >= 0; r--) {
1514                 DELAY(10);
1515                 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1516                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1517                 if ((dma->resr[resi].l & htole32(HIFN_D_VALID)) == 0)
1518                         break;
1519                 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1520                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1521         }
1522         if (r == 0) {
1523                 device_printf(sc->sc_dev, "readramaddr -- "
1524                     "result[%d](addr %d) still valid\n", resi, addr);
1525                 r = -1;
1526         } else {
1527                 r = 0;
1528                 bcopy(&dma->test_dst, data, sizeof(dma->test_dst));
1529         }
1530
1531         WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1532             HIFN_DMACSR_C_CTRL_DIS | HIFN_DMACSR_S_CTRL_DIS |
1533             HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS);
1534
1535         return (r);
1536 }
1537
1538 /*
1539  * Initialize the descriptor rings.
1540  */
1541 static void
1542 hifn_init_dma(struct hifn_softc *sc)
1543 {
1544         struct hifn_dma *dma = sc->sc_dma;
1545         int i;
1546
1547         hifn_set_retry(sc);
1548
1549         /* initialize static pointer values */
1550         for (i = 0; i < HIFN_D_CMD_RSIZE; i++)
1551                 dma->cmdr[i].p = htole32(sc->sc_dma_physaddr +
1552                     offsetof(struct hifn_dma, command_bufs[i][0]));
1553         for (i = 0; i < HIFN_D_RES_RSIZE; i++)
1554                 dma->resr[i].p = htole32(sc->sc_dma_physaddr +
1555                     offsetof(struct hifn_dma, result_bufs[i][0]));
1556
1557         dma->cmdr[HIFN_D_CMD_RSIZE].p =
1558             htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, cmdr[0]));
1559         dma->srcr[HIFN_D_SRC_RSIZE].p =
1560             htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, srcr[0]));
1561         dma->dstr[HIFN_D_DST_RSIZE].p =
1562             htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, dstr[0]));
1563         dma->resr[HIFN_D_RES_RSIZE].p =
1564             htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, resr[0]));
1565
1566         dma->cmdu = dma->srcu = dma->dstu = dma->resu = 0;
1567         dma->cmdi = dma->srci = dma->dsti = dma->resi = 0;
1568         dma->cmdk = dma->srck = dma->dstk = dma->resk = 0;
1569 }
1570
1571 /*
1572  * Writes out the raw command buffer space.  Returns the
1573  * command buffer size.
1574  */
1575 static u_int
1576 hifn_write_command(struct hifn_command *cmd, u_int8_t *buf)
1577 {
1578         u_int8_t *buf_pos;
1579         hifn_base_command_t *base_cmd;
1580         hifn_mac_command_t *mac_cmd;
1581         hifn_crypt_command_t *cry_cmd;
1582         int using_mac, using_crypt, len, ivlen;
1583         u_int32_t dlen, slen;
1584
1585         buf_pos = buf;
1586         using_mac = cmd->base_masks & HIFN_BASE_CMD_MAC;
1587         using_crypt = cmd->base_masks & HIFN_BASE_CMD_CRYPT;
1588
1589         base_cmd = (hifn_base_command_t *)buf_pos;
1590         base_cmd->masks = htole16(cmd->base_masks);
1591         slen = cmd->src_mapsize;
1592         if (cmd->sloplen)
1593                 dlen = cmd->dst_mapsize - cmd->sloplen + sizeof(u_int32_t);
1594         else
1595                 dlen = cmd->dst_mapsize;
1596         base_cmd->total_source_count = htole16(slen & HIFN_BASE_CMD_LENMASK_LO);
1597         base_cmd->total_dest_count = htole16(dlen & HIFN_BASE_CMD_LENMASK_LO);
1598         dlen >>= 16;
1599         slen >>= 16;
1600         base_cmd->session_num = htole16(
1601             ((slen << HIFN_BASE_CMD_SRCLEN_S) & HIFN_BASE_CMD_SRCLEN_M) |
1602             ((dlen << HIFN_BASE_CMD_DSTLEN_S) & HIFN_BASE_CMD_DSTLEN_M));
1603         buf_pos += sizeof(hifn_base_command_t);
1604
1605         if (using_mac) {
1606                 mac_cmd = (hifn_mac_command_t *)buf_pos;
1607                 dlen = cmd->maccrd->crd_len;
1608                 mac_cmd->source_count = htole16(dlen & 0xffff);
1609                 dlen >>= 16;
1610                 mac_cmd->masks = htole16(cmd->mac_masks |
1611                     ((dlen << HIFN_MAC_CMD_SRCLEN_S) & HIFN_MAC_CMD_SRCLEN_M));
1612                 mac_cmd->header_skip = htole16(cmd->maccrd->crd_skip);
1613                 mac_cmd->reserved = 0;
1614                 buf_pos += sizeof(hifn_mac_command_t);
1615         }
1616
1617         if (using_crypt) {
1618                 cry_cmd = (hifn_crypt_command_t *)buf_pos;
1619                 dlen = cmd->enccrd->crd_len;
1620                 cry_cmd->source_count = htole16(dlen & 0xffff);
1621                 dlen >>= 16;
1622                 cry_cmd->masks = htole16(cmd->cry_masks |
1623                     ((dlen << HIFN_CRYPT_CMD_SRCLEN_S) & HIFN_CRYPT_CMD_SRCLEN_M));
1624                 cry_cmd->header_skip = htole16(cmd->enccrd->crd_skip);
1625                 cry_cmd->reserved = 0;
1626                 buf_pos += sizeof(hifn_crypt_command_t);
1627         }
1628
1629         if (using_mac && cmd->mac_masks & HIFN_MAC_CMD_NEW_KEY) {
1630                 bcopy(cmd->mac, buf_pos, HIFN_MAC_KEY_LENGTH);
1631                 buf_pos += HIFN_MAC_KEY_LENGTH;
1632         }
1633
1634         if (using_crypt && cmd->cry_masks & HIFN_CRYPT_CMD_NEW_KEY) {
1635                 switch (cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) {
1636                 case HIFN_CRYPT_CMD_ALG_3DES:
1637                         bcopy(cmd->ck, buf_pos, HIFN_3DES_KEY_LENGTH);
1638                         buf_pos += HIFN_3DES_KEY_LENGTH;
1639                         break;
1640                 case HIFN_CRYPT_CMD_ALG_DES:
1641                         bcopy(cmd->ck, buf_pos, HIFN_DES_KEY_LENGTH);
1642                         buf_pos += HIFN_DES_KEY_LENGTH;
1643                         break;
1644                 case HIFN_CRYPT_CMD_ALG_RC4:
1645                         len = 256;
1646                         do {
1647                                 int clen;
1648
1649                                 clen = MIN(cmd->cklen, len);
1650                                 bcopy(cmd->ck, buf_pos, clen);
1651                                 len -= clen;
1652                                 buf_pos += clen;
1653                         } while (len > 0);
1654                         bzero(buf_pos, 4);
1655                         buf_pos += 4;
1656                         break;
1657                 case HIFN_CRYPT_CMD_ALG_AES:
1658                         /*
1659                          * AES keys are variable 128, 192 and
1660                          * 256 bits (16, 24 and 32 bytes).
1661                          */
1662                         bcopy(cmd->ck, buf_pos, cmd->cklen);
1663                         buf_pos += cmd->cklen;
1664                         break;
1665                 }
1666         }
1667
1668         if (using_crypt && cmd->cry_masks & HIFN_CRYPT_CMD_NEW_IV) {
1669                 switch (cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) {
1670                 case HIFN_CRYPT_CMD_ALG_AES:
1671                         ivlen = HIFN_AES_IV_LENGTH;
1672                         break;
1673                 default:
1674                         ivlen = HIFN_IV_LENGTH;
1675                         break;
1676                 }
1677                 bcopy(cmd->iv, buf_pos, ivlen);
1678                 buf_pos += ivlen;
1679         }
1680
1681         if ((cmd->base_masks & (HIFN_BASE_CMD_MAC|HIFN_BASE_CMD_CRYPT)) == 0) {
1682                 bzero(buf_pos, 8);
1683                 buf_pos += 8;
1684         }
1685
1686         return (buf_pos - buf);
1687 }
1688
1689 static int
1690 hifn_dmamap_aligned(struct hifn_operand *op)
1691 {
1692         int i;
1693
1694         for (i = 0; i < op->nsegs; i++) {
1695                 if (op->segs[i].ds_addr & 3)
1696                         return (0);
1697                 if ((i != (op->nsegs - 1)) && (op->segs[i].ds_len & 3))
1698                         return (0);
1699         }
1700         return (1);
1701 }
1702
1703 static __inline int
1704 hifn_dmamap_dstwrap(struct hifn_softc *sc, int idx)
1705 {
1706         struct hifn_dma *dma = sc->sc_dma;
1707
1708         if (++idx == HIFN_D_DST_RSIZE) {
1709                 dma->dstr[idx].l = htole32(HIFN_D_VALID | HIFN_D_JUMP |
1710                     HIFN_D_MASKDONEIRQ);
1711                 HIFN_DSTR_SYNC(sc, idx,
1712                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1713                 idx = 0;
1714         }
1715         return (idx);
1716 }
1717
1718 static int
1719 hifn_dmamap_load_dst(struct hifn_softc *sc, struct hifn_command *cmd)
1720 {
1721         struct hifn_dma *dma = sc->sc_dma;
1722         struct hifn_operand *dst = &cmd->dst;
1723         u_int32_t p, l;
1724         int idx, used = 0, i;
1725
1726         idx = dma->dsti;
1727         for (i = 0; i < dst->nsegs - 1; i++) {
1728                 dma->dstr[idx].p = htole32(dst->segs[i].ds_addr);
1729                 dma->dstr[idx].l = htole32(HIFN_D_VALID |
1730                     HIFN_D_MASKDONEIRQ | dst->segs[i].ds_len);
1731                 HIFN_DSTR_SYNC(sc, idx,
1732                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1733                 used++;
1734
1735                 idx = hifn_dmamap_dstwrap(sc, idx);
1736         }
1737
1738         if (cmd->sloplen == 0) {
1739                 p = dst->segs[i].ds_addr;
1740                 l = HIFN_D_VALID | HIFN_D_MASKDONEIRQ | HIFN_D_LAST |
1741                     dst->segs[i].ds_len;
1742         } else {
1743                 p = sc->sc_dma_physaddr +
1744                     offsetof(struct hifn_dma, slop[cmd->slopidx]);
1745                 l = HIFN_D_VALID | HIFN_D_MASKDONEIRQ | HIFN_D_LAST |
1746                     sizeof(u_int32_t);
1747
1748                 if ((dst->segs[i].ds_len - cmd->sloplen) != 0) {
1749                         dma->dstr[idx].p = htole32(dst->segs[i].ds_addr);
1750                         dma->dstr[idx].l = htole32(HIFN_D_VALID |
1751                             HIFN_D_MASKDONEIRQ |
1752                             (dst->segs[i].ds_len - cmd->sloplen));
1753                         HIFN_DSTR_SYNC(sc, idx,
1754                             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1755                         used++;
1756
1757                         idx = hifn_dmamap_dstwrap(sc, idx);
1758                 }
1759         }
1760         dma->dstr[idx].p = htole32(p);
1761         dma->dstr[idx].l = htole32(l);
1762         HIFN_DSTR_SYNC(sc, idx, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1763         used++;
1764
1765         idx = hifn_dmamap_dstwrap(sc, idx);
1766
1767         dma->dsti = idx;
1768         dma->dstu += used;
1769         return (idx);
1770 }
1771
1772 static __inline int
1773 hifn_dmamap_srcwrap(struct hifn_softc *sc, int idx)
1774 {
1775         struct hifn_dma *dma = sc->sc_dma;
1776
1777         if (++idx == HIFN_D_SRC_RSIZE) {
1778                 dma->srcr[idx].l = htole32(HIFN_D_VALID |
1779                     HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1780                 HIFN_SRCR_SYNC(sc, HIFN_D_SRC_RSIZE,
1781                     BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1782                 idx = 0;
1783         }
1784         return (idx);
1785 }
1786
1787 static int
1788 hifn_dmamap_load_src(struct hifn_softc *sc, struct hifn_command *cmd)
1789 {
1790         struct hifn_dma *dma = sc->sc_dma;
1791         struct hifn_operand *src = &cmd->src;
1792         int idx, i;
1793         u_int32_t last = 0;
1794
1795         idx = dma->srci;
1796         for (i = 0; i < src->nsegs; i++) {
1797                 if (i == src->nsegs - 1)
1798                         last = HIFN_D_LAST;
1799
1800                 dma->srcr[idx].p = htole32(src->segs[i].ds_addr);
1801                 dma->srcr[idx].l = htole32(src->segs[i].ds_len |
1802                     HIFN_D_VALID | HIFN_D_MASKDONEIRQ | last);
1803                 HIFN_SRCR_SYNC(sc, idx,
1804                     BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1805
1806                 idx = hifn_dmamap_srcwrap(sc, idx);
1807         }
1808         dma->srci = idx;
1809         dma->srcu += src->nsegs;
1810         return (idx);
1811 }
1812
1813 static void
1814 hifn_op_cb(void* arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error)
1815 {
1816         struct hifn_operand *op = arg;
1817
1818         KASSERT(nsegs <= MAX_SCATTER,
1819                 ("hifn_op_cb: too many DMA segments (%u > %u) "
1820                  "returned when mapping operand", nsegs, MAX_SCATTER));
1821         op->mapsize = mapsize;
1822         op->nsegs = nsegs;
1823         bcopy(seg, op->segs, nsegs * sizeof (seg[0]));
1824 }
1825
1826 static int
1827 hifn_crypto(
1828         struct hifn_softc *sc,
1829         struct hifn_command *cmd,
1830         struct cryptop *crp,
1831         int hint)
1832 {
1833         struct  hifn_dma *dma = sc->sc_dma;
1834         u_int32_t cmdlen, csr;
1835         int cmdi, resi, err = 0;
1836
1837         /*
1838          * need 1 cmd, and 1 res
1839          *
1840          * NB: check this first since it's easy.
1841          */
1842         HIFN_LOCK(sc);
1843         if ((dma->cmdu + 1) > HIFN_D_CMD_RSIZE ||
1844             (dma->resu + 1) > HIFN_D_RES_RSIZE) {
1845 #ifdef HIFN_DEBUG
1846                 if (hifn_debug) {
1847                         device_printf(sc->sc_dev,
1848                                 "cmd/result exhaustion, cmdu %u resu %u\n",
1849                                 dma->cmdu, dma->resu);
1850                 }
1851 #endif
1852                 hifnstats.hst_nomem_cr++;
1853                 HIFN_UNLOCK(sc);
1854                 return (ERESTART);
1855         }
1856
1857         if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &cmd->src_map)) {
1858                 hifnstats.hst_nomem_map++;
1859                 HIFN_UNLOCK(sc);
1860                 return (ENOMEM);
1861         }
1862
1863         if (crp->crp_flags & CRYPTO_F_IMBUF) {
1864                 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->src_map,
1865                     cmd->src_m, hifn_op_cb, &cmd->src, BUS_DMA_NOWAIT)) {
1866                         hifnstats.hst_nomem_load++;
1867                         err = ENOMEM;
1868                         goto err_srcmap1;
1869                 }
1870         } else if (crp->crp_flags & CRYPTO_F_IOV) {
1871                 if (bus_dmamap_load_uio(sc->sc_dmat, cmd->src_map,
1872                     cmd->src_io, hifn_op_cb, &cmd->src, BUS_DMA_NOWAIT)) {
1873                         hifnstats.hst_nomem_load++;
1874                         err = ENOMEM;
1875                         goto err_srcmap1;
1876                 }
1877         } else {
1878                 err = EINVAL;
1879                 goto err_srcmap1;
1880         }
1881
1882         if (hifn_dmamap_aligned(&cmd->src)) {
1883                 cmd->sloplen = cmd->src_mapsize & 3;
1884                 cmd->dst = cmd->src;
1885         } else {
1886                 if (crp->crp_flags & CRYPTO_F_IOV) {
1887                         err = EINVAL;
1888                         goto err_srcmap;
1889                 } else if (crp->crp_flags & CRYPTO_F_IMBUF) {
1890                         int totlen, len;
1891                         struct mbuf *m, *m0, *mlast;
1892
1893                         KASSERT(cmd->dst_m == cmd->src_m,
1894                                 ("hifn_crypto: dst_m initialized improperly"));
1895                         hifnstats.hst_unaligned++;
1896                         /*
1897                          * Source is not aligned on a longword boundary.
1898                          * Copy the data to insure alignment.  If we fail
1899                          * to allocate mbufs or clusters while doing this
1900                          * we return ERESTART so the operation is requeued
1901                          * at the crypto later, but only if there are
1902                          * ops already posted to the hardware; otherwise we
1903                          * have no guarantee that we'll be re-entered.
1904                          */
1905                         totlen = cmd->src_mapsize;
1906                         if (cmd->src_m->m_flags & M_PKTHDR) {
1907                                 len = MHLEN;
1908                                 MGETHDR(m0, M_DONTWAIT, MT_DATA);
1909                                 if (m0 && !m_dup_pkthdr(m0, cmd->src_m, M_DONTWAIT)) {
1910                                         m_free(m0);
1911                                         m0 = NULL;
1912                                 }
1913                         } else {
1914                                 len = MLEN;
1915                                 MGET(m0, M_DONTWAIT, MT_DATA);
1916                         }
1917                         if (m0 == NULL) {
1918                                 hifnstats.hst_nomem_mbuf++;
1919                                 err = dma->cmdu ? ERESTART : ENOMEM;
1920                                 goto err_srcmap;
1921                         }
1922                         if (totlen >= MINCLSIZE) {
1923                                 MCLGET(m0, M_DONTWAIT);
1924                                 if ((m0->m_flags & M_EXT) == 0) {
1925                                         hifnstats.hst_nomem_mcl++;
1926                                         err = dma->cmdu ? ERESTART : ENOMEM;
1927                                         m_freem(m0);
1928                                         goto err_srcmap;
1929                                 }
1930                                 len = MCLBYTES;
1931                         }
1932                         totlen -= len;
1933                         m0->m_pkthdr.len = m0->m_len = len;
1934                         mlast = m0;
1935
1936                         while (totlen > 0) {
1937                                 MGET(m, M_DONTWAIT, MT_DATA);
1938                                 if (m == NULL) {
1939                                         hifnstats.hst_nomem_mbuf++;
1940                                         err = dma->cmdu ? ERESTART : ENOMEM;
1941                                         m_freem(m0);
1942                                         goto err_srcmap;
1943                                 }
1944                                 len = MLEN;
1945                                 if (totlen >= MINCLSIZE) {
1946                                         MCLGET(m, M_DONTWAIT);
1947                                         if ((m->m_flags & M_EXT) == 0) {
1948                                                 hifnstats.hst_nomem_mcl++;
1949                                                 err = dma->cmdu ? ERESTART : ENOMEM;
1950                                                 mlast->m_next = m;
1951                                                 m_freem(m0);
1952                                                 goto err_srcmap;
1953                                         }
1954                                         len = MCLBYTES;
1955                                 }
1956
1957                                 m->m_len = len;
1958                                 m0->m_pkthdr.len += len;
1959                                 totlen -= len;
1960
1961                                 mlast->m_next = m;
1962                                 mlast = m;
1963                         }
1964                         cmd->dst_m = m0;
1965                 }
1966         }
1967
1968         if (cmd->dst_map == NULL) {
1969                 if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &cmd->dst_map)) {
1970                         hifnstats.hst_nomem_map++;
1971                         err = ENOMEM;
1972                         goto err_srcmap;
1973                 }
1974                 if (crp->crp_flags & CRYPTO_F_IMBUF) {
1975                         if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->dst_map,
1976                             cmd->dst_m, hifn_op_cb, &cmd->dst, BUS_DMA_NOWAIT)) {
1977                                 hifnstats.hst_nomem_map++;
1978                                 err = ENOMEM;
1979                                 goto err_dstmap1;
1980                         }
1981                 } else if (crp->crp_flags & CRYPTO_F_IOV) {
1982                         if (bus_dmamap_load_uio(sc->sc_dmat, cmd->dst_map,
1983                             cmd->dst_io, hifn_op_cb, &cmd->dst, BUS_DMA_NOWAIT)) {
1984                                 hifnstats.hst_nomem_load++;
1985                                 err = ENOMEM;
1986                                 goto err_dstmap1;
1987                         }
1988                 }
1989         }
1990
1991 #ifdef HIFN_DEBUG
1992         if (hifn_debug) {
1993                 device_printf(sc->sc_dev,
1994                     "Entering cmd: stat %8x ien %8x u %d/%d/%d/%d n %d/%d\n",
1995                     READ_REG_1(sc, HIFN_1_DMA_CSR),
1996                     READ_REG_1(sc, HIFN_1_DMA_IER),
1997                     dma->cmdu, dma->srcu, dma->dstu, dma->resu,
1998                     cmd->src_nsegs, cmd->dst_nsegs);
1999         }
2000 #endif
2001
2002         if (cmd->src_map == cmd->dst_map) {
2003                 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2004                     BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
2005         } else {
2006                 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2007                     BUS_DMASYNC_PREWRITE);
2008                 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
2009                     BUS_DMASYNC_PREREAD);
2010         }
2011
2012         /*
2013          * need N src, and N dst
2014          */
2015         if ((dma->srcu + cmd->src_nsegs) > HIFN_D_SRC_RSIZE ||
2016             (dma->dstu + cmd->dst_nsegs + 1) > HIFN_D_DST_RSIZE) {
2017 #ifdef HIFN_DEBUG
2018                 if (hifn_debug) {
2019                         device_printf(sc->sc_dev,
2020                                 "src/dst exhaustion, srcu %u+%u dstu %u+%u\n",
2021                                 dma->srcu, cmd->src_nsegs,
2022                                 dma->dstu, cmd->dst_nsegs);
2023                 }
2024 #endif
2025                 hifnstats.hst_nomem_sd++;
2026                 err = ERESTART;
2027                 goto err_dstmap;
2028         }
2029
2030         if (dma->cmdi == HIFN_D_CMD_RSIZE) {
2031                 dma->cmdi = 0;
2032                 dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID |
2033                     HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
2034                 HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE,
2035                     BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2036         }
2037         cmdi = dma->cmdi++;
2038         cmdlen = hifn_write_command(cmd, dma->command_bufs[cmdi]);
2039         HIFN_CMD_SYNC(sc, cmdi, BUS_DMASYNC_PREWRITE);
2040
2041         /* .p for command/result already set */
2042         dma->cmdr[cmdi].l = htole32(cmdlen | HIFN_D_VALID | HIFN_D_LAST |
2043             HIFN_D_MASKDONEIRQ);
2044         HIFN_CMDR_SYNC(sc, cmdi,
2045             BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2046         dma->cmdu++;
2047
2048         /*
2049          * We don't worry about missing an interrupt (which a "command wait"
2050          * interrupt salvages us from), unless there is more than one command
2051          * in the queue.
2052          */
2053         if (dma->cmdu > 1) {
2054                 sc->sc_dmaier |= HIFN_DMAIER_C_WAIT;
2055                 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
2056         }
2057
2058         hifnstats.hst_ipackets++;
2059         hifnstats.hst_ibytes += cmd->src_mapsize;
2060
2061         hifn_dmamap_load_src(sc, cmd);
2062
2063         /*
2064          * Unlike other descriptors, we don't mask done interrupt from
2065          * result descriptor.
2066          */
2067 #ifdef HIFN_DEBUG
2068         if (hifn_debug)
2069                 printf("load res\n");
2070 #endif
2071         if (dma->resi == HIFN_D_RES_RSIZE) {
2072                 dma->resi = 0;
2073                 dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID |
2074                     HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
2075                 HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE,
2076                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2077         }
2078         resi = dma->resi++;
2079         KASSERT(dma->hifn_commands[resi] == NULL,
2080                 ("hifn_crypto: command slot %u busy", resi));
2081         dma->hifn_commands[resi] = cmd;
2082         HIFN_RES_SYNC(sc, resi, BUS_DMASYNC_PREREAD);
2083         if ((hint & CRYPTO_HINT_MORE) && sc->sc_curbatch < hifn_maxbatch) {
2084                 dma->resr[resi].l = htole32(HIFN_MAX_RESULT |
2085                     HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ);
2086                 sc->sc_curbatch++;
2087                 if (sc->sc_curbatch > hifnstats.hst_maxbatch)
2088                         hifnstats.hst_maxbatch = sc->sc_curbatch;
2089                 hifnstats.hst_totbatch++;
2090         } else {
2091                 dma->resr[resi].l = htole32(HIFN_MAX_RESULT |
2092                     HIFN_D_VALID | HIFN_D_LAST);
2093                 sc->sc_curbatch = 0;
2094         }
2095         HIFN_RESR_SYNC(sc, resi,
2096             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2097         dma->resu++;
2098
2099         if (cmd->sloplen)
2100                 cmd->slopidx = resi;
2101
2102         hifn_dmamap_load_dst(sc, cmd);
2103
2104         csr = 0;
2105         if (sc->sc_c_busy == 0) {
2106                 csr |= HIFN_DMACSR_C_CTRL_ENA;
2107                 sc->sc_c_busy = 1;
2108         }
2109         if (sc->sc_s_busy == 0) {
2110                 csr |= HIFN_DMACSR_S_CTRL_ENA;
2111                 sc->sc_s_busy = 1;
2112         }
2113         if (sc->sc_r_busy == 0) {
2114                 csr |= HIFN_DMACSR_R_CTRL_ENA;
2115                 sc->sc_r_busy = 1;
2116         }
2117         if (sc->sc_d_busy == 0) {
2118                 csr |= HIFN_DMACSR_D_CTRL_ENA;
2119                 sc->sc_d_busy = 1;
2120         }
2121         if (csr)
2122                 WRITE_REG_1(sc, HIFN_1_DMA_CSR, csr);
2123
2124 #ifdef HIFN_DEBUG
2125         if (hifn_debug) {
2126                 device_printf(sc->sc_dev, "command: stat %8x ier %8x\n",
2127                     READ_REG_1(sc, HIFN_1_DMA_CSR),
2128                     READ_REG_1(sc, HIFN_1_DMA_IER));
2129         }
2130 #endif
2131
2132         sc->sc_active = 5;
2133         HIFN_UNLOCK(sc);
2134         KASSERT(err == 0, ("hifn_crypto: success with error %u", err));
2135         return (err);           /* success */
2136
2137 err_dstmap:
2138         if (cmd->src_map != cmd->dst_map)
2139                 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
2140 err_dstmap1:
2141         if (cmd->src_map != cmd->dst_map)
2142                 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
2143 err_srcmap:
2144         if (crp->crp_flags & CRYPTO_F_IMBUF) {
2145                 if (cmd->src_m != cmd->dst_m)
2146                         m_freem(cmd->dst_m);
2147         }
2148         bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
2149 err_srcmap1:
2150         bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
2151         HIFN_UNLOCK(sc);
2152         return (err);
2153 }
2154
2155 static void
2156 hifn_tick(void* vsc)
2157 {
2158         struct hifn_softc *sc = vsc;
2159
2160         HIFN_LOCK(sc);
2161         if (sc->sc_active == 0) {
2162                 struct hifn_dma *dma = sc->sc_dma;
2163                 u_int32_t r = 0;
2164
2165                 if (dma->cmdu == 0 && sc->sc_c_busy) {
2166                         sc->sc_c_busy = 0;
2167                         r |= HIFN_DMACSR_C_CTRL_DIS;
2168                 }
2169                 if (dma->srcu == 0 && sc->sc_s_busy) {
2170                         sc->sc_s_busy = 0;
2171                         r |= HIFN_DMACSR_S_CTRL_DIS;
2172                 }
2173                 if (dma->dstu == 0 && sc->sc_d_busy) {
2174                         sc->sc_d_busy = 0;
2175                         r |= HIFN_DMACSR_D_CTRL_DIS;
2176                 }
2177                 if (dma->resu == 0 && sc->sc_r_busy) {
2178                         sc->sc_r_busy = 0;
2179                         r |= HIFN_DMACSR_R_CTRL_DIS;
2180                 }
2181                 if (r)
2182                         WRITE_REG_1(sc, HIFN_1_DMA_CSR, r);
2183         } else
2184                 sc->sc_active--;
2185         HIFN_UNLOCK(sc);
2186         callout_reset(&sc->sc_tickto, hz, hifn_tick, sc);
2187 }
2188
2189 static void
2190 hifn_intr(void *arg)
2191 {
2192         struct hifn_softc *sc = arg;
2193         struct hifn_dma *dma;
2194         u_int32_t dmacsr, restart;
2195         int i, u;
2196
2197         dmacsr = READ_REG_1(sc, HIFN_1_DMA_CSR);
2198
2199         /* Nothing in the DMA unit interrupted */
2200         if ((dmacsr & sc->sc_dmaier) == 0)
2201                 return;
2202
2203         HIFN_LOCK(sc);
2204
2205         dma = sc->sc_dma;
2206
2207 #ifdef HIFN_DEBUG
2208         if (hifn_debug) {
2209                 device_printf(sc->sc_dev,
2210                     "irq: stat %08x ien %08x damier %08x i %d/%d/%d/%d k %d/%d/%d/%d u %d/%d/%d/%d\n",
2211                     dmacsr, READ_REG_1(sc, HIFN_1_DMA_IER), sc->sc_dmaier,
2212                     dma->cmdi, dma->srci, dma->dsti, dma->resi,
2213                     dma->cmdk, dma->srck, dma->dstk, dma->resk,
2214                     dma->cmdu, dma->srcu, dma->dstu, dma->resu);
2215         }
2216 #endif
2217
2218         WRITE_REG_1(sc, HIFN_1_DMA_CSR, dmacsr & sc->sc_dmaier);
2219
2220         if ((sc->sc_flags & HIFN_HAS_PUBLIC) &&
2221             (dmacsr & HIFN_DMACSR_PUBDONE))
2222                 WRITE_REG_1(sc, HIFN_1_PUB_STATUS,
2223                     READ_REG_1(sc, HIFN_1_PUB_STATUS) | HIFN_PUBSTS_DONE);
2224
2225         restart = dmacsr & (HIFN_DMACSR_D_OVER | HIFN_DMACSR_R_OVER);
2226         if (restart)
2227                 device_printf(sc->sc_dev, "overrun %x\n", dmacsr);
2228
2229         if (sc->sc_flags & HIFN_IS_7811) {
2230                 if (dmacsr & HIFN_DMACSR_ILLR)
2231                         device_printf(sc->sc_dev, "illegal read\n");
2232                 if (dmacsr & HIFN_DMACSR_ILLW)
2233                         device_printf(sc->sc_dev, "illegal write\n");
2234         }
2235
2236         restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT |
2237             HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT);
2238         if (restart) {
2239                 device_printf(sc->sc_dev, "abort, resetting.\n");
2240                 hifnstats.hst_abort++;
2241                 hifn_abort(sc);
2242                 HIFN_UNLOCK(sc);
2243                 return;
2244         }
2245
2246         if ((dmacsr & HIFN_DMACSR_C_WAIT) && (dma->cmdu == 0)) {
2247                 /*
2248                  * If no slots to process and we receive a "waiting on
2249                  * command" interrupt, we disable the "waiting on command"
2250                  * (by clearing it).
2251                  */
2252                 sc->sc_dmaier &= ~HIFN_DMAIER_C_WAIT;
2253                 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
2254         }
2255
2256         /* clear the rings */
2257         i = dma->resk; u = dma->resu;
2258         while (u != 0) {
2259                 HIFN_RESR_SYNC(sc, i,
2260                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2261                 if (dma->resr[i].l & htole32(HIFN_D_VALID)) {
2262                         HIFN_RESR_SYNC(sc, i,
2263                             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2264                         break;
2265                 }
2266
2267                 if (i != HIFN_D_RES_RSIZE) {
2268                         struct hifn_command *cmd;
2269                         u_int8_t *macbuf = NULL;
2270
2271                         HIFN_RES_SYNC(sc, i, BUS_DMASYNC_POSTREAD);
2272                         cmd = dma->hifn_commands[i];
2273                         KASSERT(cmd != NULL,
2274                                 ("hifn_intr: null command slot %u", i));
2275                         dma->hifn_commands[i] = NULL;
2276
2277                         if (cmd->base_masks & HIFN_BASE_CMD_MAC) {
2278                                 macbuf = dma->result_bufs[i];
2279                                 macbuf += 12;
2280                         }
2281
2282                         hifn_callback(sc, cmd, macbuf);
2283                         hifnstats.hst_opackets++;
2284                         u--;
2285                 }
2286
2287                 if (++i == (HIFN_D_RES_RSIZE + 1))
2288                         i = 0;
2289         }
2290         dma->resk = i; dma->resu = u;
2291
2292         i = dma->srck; u = dma->srcu;
2293         while (u != 0) {
2294                 if (i == HIFN_D_SRC_RSIZE)
2295                         i = 0;
2296                 HIFN_SRCR_SYNC(sc, i,
2297                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2298                 if (dma->srcr[i].l & htole32(HIFN_D_VALID)) {
2299                         HIFN_SRCR_SYNC(sc, i,
2300                             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2301                         break;
2302                 }
2303                 i++, u--;
2304         }
2305         dma->srck = i; dma->srcu = u;
2306
2307         i = dma->cmdk; u = dma->cmdu;
2308         while (u != 0) {
2309                 HIFN_CMDR_SYNC(sc, i,
2310                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2311                 if (dma->cmdr[i].l & htole32(HIFN_D_VALID)) {
2312                         HIFN_CMDR_SYNC(sc, i,
2313                             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2314                         break;
2315                 }
2316                 if (i != HIFN_D_CMD_RSIZE) {
2317                         u--;
2318                         HIFN_CMD_SYNC(sc, i, BUS_DMASYNC_POSTWRITE);
2319                 }
2320                 if (++i == (HIFN_D_CMD_RSIZE + 1))
2321                         i = 0;
2322         }
2323         dma->cmdk = i; dma->cmdu = u;
2324
2325         HIFN_UNLOCK(sc);
2326
2327         if (sc->sc_needwakeup) {                /* XXX check high watermark */
2328                 int wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ);
2329 #ifdef HIFN_DEBUG
2330                 if (hifn_debug)
2331                         device_printf(sc->sc_dev,
2332                                 "wakeup crypto (%x) u %d/%d/%d/%d\n",
2333                                 sc->sc_needwakeup,
2334                                 dma->cmdu, dma->srcu, dma->dstu, dma->resu);
2335 #endif
2336                 sc->sc_needwakeup &= ~wakeup;
2337                 crypto_unblock(sc->sc_cid, wakeup);
2338         }
2339 }
2340
2341 /*
2342  * Allocate a new 'session' and return an encoded session id.  'sidp'
2343  * contains our registration id, and should contain an encoded session
2344  * id on successful allocation.
2345  */
2346 static int
2347 hifn_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri)
2348 {
2349         struct hifn_softc *sc = device_get_softc(dev);
2350         struct cryptoini *c;
2351         int mac = 0, cry = 0, sesn;
2352         struct hifn_session *ses = NULL;
2353
2354         KASSERT(sc != NULL, ("hifn_newsession: null softc"));
2355         if (sidp == NULL || cri == NULL || sc == NULL)
2356                 return (EINVAL);
2357
2358         HIFN_LOCK(sc);
2359         if (sc->sc_sessions == NULL) {
2360                 ses = sc->sc_sessions = (struct hifn_session *)malloc(
2361                     sizeof(*ses), M_DEVBUF, M_NOWAIT);
2362                 if (ses == NULL) {
2363                         HIFN_UNLOCK(sc);
2364                         return (ENOMEM);
2365                 }
2366                 sesn = 0;
2367                 sc->sc_nsessions = 1;
2368         } else {
2369                 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
2370                         if (!sc->sc_sessions[sesn].hs_used) {
2371                                 ses = &sc->sc_sessions[sesn];
2372                                 break;
2373                         }
2374                 }
2375
2376                 if (ses == NULL) {
2377                         sesn = sc->sc_nsessions;
2378                         ses = (struct hifn_session *)malloc((sesn + 1) *
2379                             sizeof(*ses), M_DEVBUF, M_NOWAIT);
2380                         if (ses == NULL) {
2381                                 HIFN_UNLOCK(sc);
2382                                 return (ENOMEM);
2383                         }
2384                         bcopy(sc->sc_sessions, ses, sesn * sizeof(*ses));
2385                         bzero(sc->sc_sessions, sesn * sizeof(*ses));
2386                         free(sc->sc_sessions, M_DEVBUF);
2387                         sc->sc_sessions = ses;
2388                         ses = &sc->sc_sessions[sesn];
2389                         sc->sc_nsessions++;
2390                 }
2391         }
2392         HIFN_UNLOCK(sc);
2393
2394         bzero(ses, sizeof(*ses));
2395         ses->hs_used = 1;
2396
2397         for (c = cri; c != NULL; c = c->cri_next) {
2398                 switch (c->cri_alg) {
2399                 case CRYPTO_MD5:
2400                 case CRYPTO_SHA1:
2401                 case CRYPTO_MD5_HMAC:
2402                 case CRYPTO_SHA1_HMAC:
2403                         if (mac)
2404                                 return (EINVAL);
2405                         mac = 1;
2406                         ses->hs_mlen = c->cri_mlen;
2407                         if (ses->hs_mlen == 0) {
2408                                 switch (c->cri_alg) {
2409                                 case CRYPTO_MD5:
2410                                 case CRYPTO_MD5_HMAC:
2411                                         ses->hs_mlen = 16;
2412                                         break;
2413                                 case CRYPTO_SHA1:
2414                                 case CRYPTO_SHA1_HMAC:
2415                                         ses->hs_mlen = 20;
2416                                         break;
2417                                 }
2418                         }
2419                         break;
2420                 case CRYPTO_DES_CBC:
2421                 case CRYPTO_3DES_CBC:
2422                 case CRYPTO_AES_CBC:
2423                         /* XXX this may read fewer, does it matter? */
2424                         read_random(ses->hs_iv,
2425                                 c->cri_alg == CRYPTO_AES_CBC ?
2426                                         HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
2427                         /*FALLTHROUGH*/
2428                 case CRYPTO_ARC4:
2429                         if (cry)
2430                                 return (EINVAL);
2431                         cry = 1;
2432                         break;
2433                 default:
2434                         return (EINVAL);
2435                 }
2436         }
2437         if (mac == 0 && cry == 0)
2438                 return (EINVAL);
2439
2440         *sidp = HIFN_SID(device_get_unit(sc->sc_dev), sesn);
2441
2442         return (0);
2443 }
2444
2445 /*
2446  * Deallocate a session.
2447  * XXX this routine should run a zero'd mac/encrypt key into context ram.
2448  * XXX to blow away any keys already stored there.
2449  */
2450 static int
2451 hifn_freesession(device_t dev, u_int64_t tid)
2452 {
2453         struct hifn_softc *sc = device_get_softc(dev);
2454         int session, error;
2455         u_int32_t sid = CRYPTO_SESID2LID(tid);
2456
2457         KASSERT(sc != NULL, ("hifn_freesession: null softc"));
2458         if (sc == NULL)
2459                 return (EINVAL);
2460
2461         HIFN_LOCK(sc);
2462         session = HIFN_SESSION(sid);
2463         if (session < sc->sc_nsessions) {
2464                 bzero(&sc->sc_sessions[session], sizeof(struct hifn_session));
2465                 error = 0;
2466         } else
2467                 error = EINVAL;
2468         HIFN_UNLOCK(sc);
2469
2470         return (error);
2471 }
2472
2473 static int
2474 hifn_process(device_t dev, struct cryptop *crp, int hint)
2475 {
2476         struct hifn_softc *sc = device_get_softc(dev);
2477         struct hifn_command *cmd = NULL;
2478         int session, err, ivlen;
2479         struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
2480
2481         if (crp == NULL || crp->crp_callback == NULL) {
2482                 hifnstats.hst_invalid++;
2483                 return (EINVAL);
2484         }
2485         session = HIFN_SESSION(crp->crp_sid);
2486
2487         if (sc == NULL || session >= sc->sc_nsessions) {
2488                 err = EINVAL;
2489                 goto errout;
2490         }
2491
2492         cmd = malloc(sizeof(struct hifn_command), M_DEVBUF, M_NOWAIT | M_ZERO);
2493         if (cmd == NULL) {
2494                 hifnstats.hst_nomem++;
2495                 err = ENOMEM;
2496                 goto errout;
2497         }
2498
2499         if (crp->crp_flags & CRYPTO_F_IMBUF) {
2500                 cmd->src_m = (struct mbuf *)crp->crp_buf;
2501                 cmd->dst_m = (struct mbuf *)crp->crp_buf;
2502         } else if (crp->crp_flags & CRYPTO_F_IOV) {
2503                 cmd->src_io = (struct uio *)crp->crp_buf;
2504                 cmd->dst_io = (struct uio *)crp->crp_buf;
2505         } else {
2506                 err = EINVAL;
2507                 goto errout;    /* XXX we don't handle contiguous buffers! */
2508         }
2509
2510         crd1 = crp->crp_desc;
2511         if (crd1 == NULL) {
2512                 err = EINVAL;
2513                 goto errout;
2514         }
2515         crd2 = crd1->crd_next;
2516
2517         if (crd2 == NULL) {
2518                 if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
2519                     crd1->crd_alg == CRYPTO_SHA1_HMAC ||
2520                     crd1->crd_alg == CRYPTO_SHA1 ||
2521                     crd1->crd_alg == CRYPTO_MD5) {
2522                         maccrd = crd1;
2523                         enccrd = NULL;
2524                 } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
2525                     crd1->crd_alg == CRYPTO_3DES_CBC ||
2526                     crd1->crd_alg == CRYPTO_AES_CBC ||
2527                     crd1->crd_alg == CRYPTO_ARC4) {
2528                         if ((crd1->crd_flags & CRD_F_ENCRYPT) == 0)
2529                                 cmd->base_masks |= HIFN_BASE_CMD_DECODE;
2530                         maccrd = NULL;
2531                         enccrd = crd1;
2532                 } else {
2533                         err = EINVAL;
2534                         goto errout;
2535                 }
2536         } else {
2537                 if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
2538                      crd1->crd_alg == CRYPTO_SHA1_HMAC ||
2539                      crd1->crd_alg == CRYPTO_MD5 ||
2540                      crd1->crd_alg == CRYPTO_SHA1) &&
2541                     (crd2->crd_alg == CRYPTO_DES_CBC ||
2542                      crd2->crd_alg == CRYPTO_3DES_CBC ||
2543                      crd2->crd_alg == CRYPTO_AES_CBC ||
2544                      crd2->crd_alg == CRYPTO_ARC4) &&
2545                     ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
2546                         cmd->base_masks = HIFN_BASE_CMD_DECODE;
2547                         maccrd = crd1;
2548                         enccrd = crd2;
2549                 } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
2550                      crd1->crd_alg == CRYPTO_ARC4 ||
2551                      crd1->crd_alg == CRYPTO_3DES_CBC ||
2552                      crd1->crd_alg == CRYPTO_AES_CBC) &&
2553                     (crd2->crd_alg == CRYPTO_MD5_HMAC ||
2554                      crd2->crd_alg == CRYPTO_SHA1_HMAC ||
2555                      crd2->crd_alg == CRYPTO_MD5 ||
2556                      crd2->crd_alg == CRYPTO_SHA1) &&
2557                     (crd1->crd_flags & CRD_F_ENCRYPT)) {
2558                         enccrd = crd1;
2559                         maccrd = crd2;
2560                 } else {
2561                         /*
2562                          * We cannot order the 7751 as requested
2563                          */
2564                         err = EINVAL;
2565                         goto errout;
2566                 }
2567         }
2568
2569         if (enccrd) {
2570                 cmd->enccrd = enccrd;
2571                 cmd->base_masks |= HIFN_BASE_CMD_CRYPT;
2572                 switch (enccrd->crd_alg) {
2573                 case CRYPTO_ARC4:
2574                         cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_RC4;
2575                         break;
2576                 case CRYPTO_DES_CBC:
2577                         cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_DES |
2578                             HIFN_CRYPT_CMD_MODE_CBC |
2579                             HIFN_CRYPT_CMD_NEW_IV;
2580                         break;
2581                 case CRYPTO_3DES_CBC:
2582                         cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_3DES |
2583                             HIFN_CRYPT_CMD_MODE_CBC |
2584                             HIFN_CRYPT_CMD_NEW_IV;
2585                         break;
2586                 case CRYPTO_AES_CBC:
2587                         cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_AES |
2588                             HIFN_CRYPT_CMD_MODE_CBC |
2589                             HIFN_CRYPT_CMD_NEW_IV;
2590                         break;
2591                 default:
2592                         err = EINVAL;
2593                         goto errout;
2594                 }
2595                 if (enccrd->crd_alg != CRYPTO_ARC4) {
2596                         ivlen = ((enccrd->crd_alg == CRYPTO_AES_CBC) ?
2597                                 HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
2598                         if (enccrd->crd_flags & CRD_F_ENCRYPT) {
2599                                 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
2600                                         bcopy(enccrd->crd_iv, cmd->iv, ivlen);
2601                                 else
2602                                         bcopy(sc->sc_sessions[session].hs_iv,
2603                                             cmd->iv, ivlen);
2604
2605                                 if ((enccrd->crd_flags & CRD_F_IV_PRESENT)
2606                                     == 0) {
2607                                         crypto_copyback(crp->crp_flags,
2608                                             crp->crp_buf, enccrd->crd_inject,
2609                                             ivlen, cmd->iv);
2610                                 }
2611                         } else {
2612                                 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
2613                                         bcopy(enccrd->crd_iv, cmd->iv, ivlen);
2614                                 else {
2615                                         crypto_copydata(crp->crp_flags,
2616                                             crp->crp_buf, enccrd->crd_inject,
2617                                             ivlen, cmd->iv);
2618                                 }
2619                         }
2620                 }
2621
2622                 if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT)
2623                         cmd->cry_masks |= HIFN_CRYPT_CMD_NEW_KEY;
2624                 cmd->ck = enccrd->crd_key;
2625                 cmd->cklen = enccrd->crd_klen >> 3;
2626                 cmd->cry_masks |= HIFN_CRYPT_CMD_NEW_KEY;
2627
2628                 /*
2629                  * Need to specify the size for the AES key in the masks.
2630                  */
2631                 if ((cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) ==
2632                     HIFN_CRYPT_CMD_ALG_AES) {
2633                         switch (cmd->cklen) {
2634                         case 16:
2635                                 cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_128;
2636                                 break;
2637                         case 24:
2638                                 cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_192;
2639                                 break;
2640                         case 32:
2641                                 cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_256;
2642                                 break;
2643                         default:
2644                                 err = EINVAL;
2645                                 goto errout;
2646                         }
2647                 }
2648         }
2649
2650         if (maccrd) {
2651                 cmd->maccrd = maccrd;
2652                 cmd->base_masks |= HIFN_BASE_CMD_MAC;
2653
2654                 switch (maccrd->crd_alg) {
2655                 case CRYPTO_MD5:
2656                         cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 |
2657                             HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH |
2658                             HIFN_MAC_CMD_POS_IPSEC;
2659                        break;
2660                 case CRYPTO_MD5_HMAC:
2661                         cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 |
2662                             HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HMAC |
2663                             HIFN_MAC_CMD_POS_IPSEC | HIFN_MAC_CMD_TRUNC;
2664                         break;
2665                 case CRYPTO_SHA1:
2666                         cmd->mac_masks |= HIFN_MAC_CMD_ALG_SHA1 |
2667                             HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH |
2668                             HIFN_MAC_CMD_POS_IPSEC;
2669                         break;
2670                 case CRYPTO_SHA1_HMAC:
2671                         cmd->mac_masks |= HIFN_MAC_CMD_ALG_SHA1 |
2672                             HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HMAC |
2673                             HIFN_MAC_CMD_POS_IPSEC | HIFN_MAC_CMD_TRUNC;
2674                         break;
2675                 }
2676
2677                 if (maccrd->crd_alg == CRYPTO_SHA1_HMAC ||
2678                      maccrd->crd_alg == CRYPTO_MD5_HMAC) {
2679                         cmd->mac_masks |= HIFN_MAC_CMD_NEW_KEY;
2680                         bcopy(maccrd->crd_key, cmd->mac, maccrd->crd_klen >> 3);
2681                         bzero(cmd->mac + (maccrd->crd_klen >> 3),
2682                             HIFN_MAC_KEY_LENGTH - (maccrd->crd_klen >> 3));
2683                 }
2684         }
2685
2686         cmd->crp = crp;
2687         cmd->session_num = session;
2688         cmd->softc = sc;
2689
2690         err = hifn_crypto(sc, cmd, crp, hint);
2691         if (!err) {
2692                 return 0;
2693         } else if (err == ERESTART) {
2694                 /*
2695                  * There weren't enough resources to dispatch the request
2696                  * to the part.  Notify the caller so they'll requeue this
2697                  * request and resubmit it again soon.
2698                  */
2699 #ifdef HIFN_DEBUG
2700                 if (hifn_debug)
2701                         device_printf(sc->sc_dev, "requeue request\n");
2702 #endif
2703                 free(cmd, M_DEVBUF);
2704                 sc->sc_needwakeup |= CRYPTO_SYMQ;
2705                 return (err);
2706         }
2707
2708 errout:
2709         if (cmd != NULL)
2710                 free(cmd, M_DEVBUF);
2711         if (err == EINVAL)
2712                 hifnstats.hst_invalid++;
2713         else
2714                 hifnstats.hst_nomem++;
2715         crp->crp_etype = err;
2716         crypto_done(crp);
2717         return (err);
2718 }
2719
2720 static void
2721 hifn_abort(struct hifn_softc *sc)
2722 {
2723         struct hifn_dma *dma = sc->sc_dma;
2724         struct hifn_command *cmd;
2725         struct cryptop *crp;
2726         int i, u;
2727
2728         i = dma->resk; u = dma->resu;
2729         while (u != 0) {
2730                 cmd = dma->hifn_commands[i];
2731                 KASSERT(cmd != NULL, ("hifn_abort: null command slot %u", i));
2732                 dma->hifn_commands[i] = NULL;
2733                 crp = cmd->crp;
2734
2735                 if ((dma->resr[i].l & htole32(HIFN_D_VALID)) == 0) {
2736                         /* Salvage what we can. */
2737                         u_int8_t *macbuf;
2738
2739                         if (cmd->base_masks & HIFN_BASE_CMD_MAC) {
2740                                 macbuf = dma->result_bufs[i];
2741                                 macbuf += 12;
2742                         } else
2743                                 macbuf = NULL;
2744                         hifnstats.hst_opackets++;
2745                         hifn_callback(sc, cmd, macbuf);
2746                 } else {
2747                         if (cmd->src_map == cmd->dst_map) {
2748                                 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2749                                     BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
2750                         } else {
2751                                 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2752                                     BUS_DMASYNC_POSTWRITE);
2753                                 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
2754                                     BUS_DMASYNC_POSTREAD);
2755                         }
2756
2757                         if (cmd->src_m != cmd->dst_m) {
2758                                 m_freem(cmd->src_m);
2759                                 crp->crp_buf = (caddr_t)cmd->dst_m;
2760                         }
2761
2762                         /* non-shared buffers cannot be restarted */
2763                         if (cmd->src_map != cmd->dst_map) {
2764                                 /*
2765                                  * XXX should be EAGAIN, delayed until
2766                                  * after the reset.
2767                                  */
2768                                 crp->crp_etype = ENOMEM;
2769                                 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
2770                                 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
2771                         } else
2772                                 crp->crp_etype = ENOMEM;
2773
2774                         bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
2775                         bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
2776
2777                         free(cmd, M_DEVBUF);
2778                         if (crp->crp_etype != EAGAIN)
2779                                 crypto_done(crp);
2780                 }
2781
2782                 if (++i == HIFN_D_RES_RSIZE)
2783                         i = 0;
2784                 u--;
2785         }
2786         dma->resk = i; dma->resu = u;
2787
2788         hifn_reset_board(sc, 1);
2789         hifn_init_dma(sc);
2790         hifn_init_pci_registers(sc);
2791 }
2792
2793 static void
2794 hifn_callback(struct hifn_softc *sc, struct hifn_command *cmd, u_int8_t *macbuf)
2795 {
2796         struct hifn_dma *dma = sc->sc_dma;
2797         struct cryptop *crp = cmd->crp;
2798         struct cryptodesc *crd;
2799         struct mbuf *m;
2800         int totlen, i, u, ivlen;
2801
2802         if (cmd->src_map == cmd->dst_map) {
2803                 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2804                     BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
2805         } else {
2806                 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2807                     BUS_DMASYNC_POSTWRITE);
2808                 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
2809                     BUS_DMASYNC_POSTREAD);
2810         }
2811
2812         if (crp->crp_flags & CRYPTO_F_IMBUF) {
2813                 if (cmd->src_m != cmd->dst_m) {
2814                         crp->crp_buf = (caddr_t)cmd->dst_m;
2815                         totlen = cmd->src_mapsize;
2816                         for (m = cmd->dst_m; m != NULL; m = m->m_next) {
2817                                 if (totlen < m->m_len) {
2818                                         m->m_len = totlen;
2819                                         totlen = 0;
2820                                 } else
2821                                         totlen -= m->m_len;
2822                         }
2823                         cmd->dst_m->m_pkthdr.len = cmd->src_m->m_pkthdr.len;
2824                         m_freem(cmd->src_m);
2825                 }
2826         }
2827
2828         if (cmd->sloplen != 0) {
2829                 crypto_copyback(crp->crp_flags, crp->crp_buf,
2830                     cmd->src_mapsize - cmd->sloplen, cmd->sloplen,
2831                     (caddr_t)&dma->slop[cmd->slopidx]);
2832         }
2833
2834         i = dma->dstk; u = dma->dstu;
2835         while (u != 0) {
2836                 if (i == HIFN_D_DST_RSIZE)
2837                         i = 0;
2838                 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
2839                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2840                 if (dma->dstr[i].l & htole32(HIFN_D_VALID)) {
2841                         bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
2842                             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2843                         break;
2844                 }
2845                 i++, u--;
2846         }
2847         dma->dstk = i; dma->dstu = u;
2848
2849         hifnstats.hst_obytes += cmd->dst_mapsize;
2850
2851         if ((cmd->base_masks & (HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE)) ==
2852             HIFN_BASE_CMD_CRYPT) {
2853                 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
2854                         if (crd->crd_alg != CRYPTO_DES_CBC &&
2855                             crd->crd_alg != CRYPTO_3DES_CBC &&
2856                             crd->crd_alg != CRYPTO_AES_CBC)
2857                                 continue;
2858                         ivlen = ((crd->crd_alg == CRYPTO_AES_CBC) ?
2859                                 HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
2860                         crypto_copydata(crp->crp_flags, crp->crp_buf,
2861                             crd->crd_skip + crd->crd_len - ivlen, ivlen,
2862                             cmd->softc->sc_sessions[cmd->session_num].hs_iv);
2863                         break;
2864                 }
2865         }
2866
2867         if (macbuf != NULL) {
2868                 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
2869                         int len;
2870
2871                         if (crd->crd_alg != CRYPTO_MD5 &&
2872                             crd->crd_alg != CRYPTO_SHA1 &&
2873                             crd->crd_alg != CRYPTO_MD5_HMAC &&
2874                             crd->crd_alg != CRYPTO_SHA1_HMAC) {
2875                                 continue;
2876                         }
2877                         len = cmd->softc->sc_sessions[cmd->session_num].hs_mlen;
2878                         crypto_copyback(crp->crp_flags, crp->crp_buf,
2879                             crd->crd_inject, len, macbuf);
2880                         break;
2881                 }
2882         }
2883
2884         if (cmd->src_map != cmd->dst_map) {
2885                 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
2886                 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
2887         }
2888         bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
2889         bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
2890         free(cmd, M_DEVBUF);
2891         crypto_done(crp);
2892 }
2893
2894 /*
2895  * 7811 PB3 rev/2 parts lock-up on burst writes to Group 0
2896  * and Group 1 registers; avoid conditions that could create
2897  * burst writes by doing a read in between the writes.
2898  *
2899  * NB: The read we interpose is always to the same register;
2900  *     we do this because reading from an arbitrary (e.g. last)
2901  *     register may not always work.
2902  */
2903 static void
2904 hifn_write_reg_0(struct hifn_softc *sc, bus_size_t reg, u_int32_t val)
2905 {
2906         if (sc->sc_flags & HIFN_IS_7811) {
2907                 if (sc->sc_bar0_lastreg == reg - 4)
2908                         bus_space_read_4(sc->sc_st0, sc->sc_sh0, HIFN_0_PUCNFG);
2909                 sc->sc_bar0_lastreg = reg;
2910         }
2911         bus_space_write_4(sc->sc_st0, sc->sc_sh0, reg, val);
2912 }
2913
2914 static void
2915 hifn_write_reg_1(struct hifn_softc *sc, bus_size_t reg, u_int32_t val)
2916 {
2917         if (sc->sc_flags & HIFN_IS_7811) {
2918                 if (sc->sc_bar1_lastreg == reg - 4)
2919                         bus_space_read_4(sc->sc_st1, sc->sc_sh1, HIFN_1_REVID);
2920                 sc->sc_bar1_lastreg = reg;
2921         }
2922         bus_space_write_4(sc->sc_st1, sc->sc_sh1, reg, val);
2923 }
2924
2925 #ifdef HIFN_VULCANDEV
2926 /*
2927  * this code provides support for mapping the PK engine's register
2928  * into a userspace program.
2929  *
2930  */
2931 static int
2932 vulcanpk_mmap(struct cdev *dev, vm_offset_t offset,
2933               vm_paddr_t *paddr, int nprot)
2934 {
2935         struct hifn_softc *sc;
2936         vm_paddr_t pd;
2937         void *b;
2938
2939         sc = dev->si_drv1;
2940
2941         pd = rman_get_start(sc->sc_bar1res);
2942         b = rman_get_virtual(sc->sc_bar1res);
2943
2944 #if 0
2945         printf("vpk mmap: %p(%08x) offset=%d\n", b, pd, offset);
2946         hexdump(b, HIFN_1_PUB_MEMEND, "vpk", 0);
2947 #endif
2948
2949         if (offset == 0) {
2950                 *paddr = pd;
2951                 return (0);
2952         }
2953         return (-1);
2954 }
2955
2956 static struct cdevsw vulcanpk_cdevsw = {
2957         .d_version =    D_VERSION,
2958         .d_mmap =       vulcanpk_mmap,
2959         .d_name =       "vulcanpk",
2960 };
2961 #endif /* HIFN_VULCANDEV */