sys/dev/hifn/hifn7751.c

   1 /*      $OpenBSD: hifn7751.c,v 1.120 2002/05/17 00:33:34 deraadt Exp $  */
   2
   3 /*-
   4  * Invertex AEON / Hifn 7751 driver
   5  * Copyright (c) 1999 Invertex Inc. All rights reserved.
   6  * Copyright (c) 1999 Theo de Raadt
   7  * Copyright (c) 2000-2001 Network Security Technologies, Inc.
   8  *                      http://www.netsec.net
   9  * Copyright (c) 2003 Hifn Inc.
  10  *
  11  * This driver is based on a previous driver by Invertex, for which they
  12  * requested:  Please send any comments, feedback, bug-fixes, or feature
  13  * requests to software@invertex.com.
  14  *
  15  * Redistribution and use in source and binary forms, with or without
  16  * modification, are permitted provided that the following conditions
  17  * are met:
  18  *
  19  * 1. Redistributions of source code must retain the above copyright
  20  *   notice, this list of conditions and the following disclaimer.
  21  * 2. Redistributions in binary form must reproduce the above copyright
  22  *   notice, this list of conditions and the following disclaimer in the
  23  *   documentation and/or other materials provided with the distribution.
  24  * 3. The name of the author may not be used to endorse or promote products
  25  *   derived from this software without specific prior written permission.
  26  *
  27  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  28  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  29  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  30  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  31  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  32  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  33  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  34  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  35  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  36  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  37  *
  38  * Effort sponsored in part by the Defense Advanced Research Projects
  39  * Agency (DARPA) and Air Force Research Laboratory, Air Force
  40  * Materiel Command, USAF, under agreement number F30602-01-2-0537.
  41  */
  42
  43 #include <sys/cdefs.h>
  44 __FBSDID("$FreeBSD$");
  45
  46 /*
  47  * Driver for various Hifn encryption processors.
  48  */
  49 #include "opt_hifn.h"
  50
  51 #include <sys/param.h>
  52 #include <sys/systm.h>
  53 #include <sys/proc.h>
  54 #include <sys/errno.h>
  55 #include <sys/malloc.h>
  56 #include <sys/kernel.h>
  57 #include <sys/module.h>
  58 #include <sys/mbuf.h>
  59 #include <sys/lock.h>
  60 #include <sys/mutex.h>
  61 #include <sys/sysctl.h>
  62
  63 #include <vm/vm.h>
  64 #include <vm/pmap.h>
  65
  66 #include <machine/bus.h>
  67 #include <machine/resource.h>
  68 #include <sys/bus.h>
  69 #include <sys/rman.h>
  70
  71 #include <opencrypto/cryptodev.h>
  72 #include <sys/random.h>
  73 #include <sys/kobj.h>
  74
  75 #include "cryptodev_if.h"
  76
  77 #include <dev/pci/pcivar.h>
  78 #include <dev/pci/pcireg.h>
  79
  80 #ifdef HIFN_RNDTEST
  81 #include <dev/rndtest/rndtest.h>
  82 #endif
  83 #include <dev/hifn/hifn7751reg.h>
  84 #include <dev/hifn/hifn7751var.h>
  85
  86 #ifdef HIFN_VULCANDEV
  87 #include <sys/conf.h>
  88 #include <sys/uio.h>
  89
  90 static struct cdevsw vulcanpk_cdevsw; /* forward declaration */
  91 #endif
  92
  93 /*
  94  * Prototypes and count for the pci_device structure
  95  */
  96 static  int hifn_probe(device_t);
  97 static  int hifn_attach(device_t);
  98 static  int hifn_detach(device_t);
  99 static  int hifn_suspend(device_t);
 100 static  int hifn_resume(device_t);
 101 static  int hifn_shutdown(device_t);
 102
 103 static  int hifn_newsession(device_t, u_int32_t *, struct cryptoini *);
 104 static  int hifn_freesession(device_t, u_int64_t);
 105 static  int hifn_process(device_t, struct cryptop *, int);
 106
 107 static device_method_t hifn_methods[] = {
 108         /* Device interface */
 109         DEVMETHOD(device_probe,         hifn_probe),
 110         DEVMETHOD(device_attach,        hifn_attach),
 111         DEVMETHOD(device_detach,        hifn_detach),
 112         DEVMETHOD(device_suspend,       hifn_suspend),
 113         DEVMETHOD(device_resume,        hifn_resume),
 114         DEVMETHOD(device_shutdown,      hifn_shutdown),
 115
 116         /* crypto device methods */
 117         DEVMETHOD(cryptodev_newsession, hifn_newsession),
 118         DEVMETHOD(cryptodev_freesession,hifn_freesession),
 119         DEVMETHOD(cryptodev_process,    hifn_process),
 120
 121         DEVMETHOD_END
 122 };
 123 static driver_t hifn_driver = {
 124         "hifn",
 125         hifn_methods,
 126         sizeof (struct hifn_softc)
 127 };
 128 static devclass_t hifn_devclass;
 129
 130 DRIVER_MODULE(hifn, pci, hifn_driver, hifn_devclass, 0, 0);
 131 MODULE_DEPEND(hifn, crypto, 1, 1, 1);
 132 #ifdef HIFN_RNDTEST
 133 MODULE_DEPEND(hifn, rndtest, 1, 1, 1);
 134 #endif
 135
 136 static  void hifn_reset_board(struct hifn_softc *, int);
 137 static  void hifn_reset_puc(struct hifn_softc *);
 138 static  void hifn_puc_wait(struct hifn_softc *);
 139 static  int hifn_enable_crypto(struct hifn_softc *);
 140 static  void hifn_set_retry(struct hifn_softc *sc);
 141 static  void hifn_init_dma(struct hifn_softc *);
 142 static  void hifn_init_pci_registers(struct hifn_softc *);
 143 static  int hifn_sramsize(struct hifn_softc *);
 144 static  int hifn_dramsize(struct hifn_softc *);
 145 static  int hifn_ramtype(struct hifn_softc *);
 146 static  void hifn_sessions(struct hifn_softc *);
 147 static  void hifn_intr(void *);
 148 static  u_int hifn_write_command(struct hifn_command *, u_int8_t *);
 149 static  u_int32_t hifn_next_signature(u_int32_t a, u_int cnt);
 150 static  void hifn_callback(struct hifn_softc *, struct hifn_command *, u_int8_t *);
 151 static  int hifn_crypto(struct hifn_softc *, struct hifn_command *, struct cryptop *, int);
 152 static  int hifn_readramaddr(struct hifn_softc *, int, u_int8_t *);
 153 static  int hifn_writeramaddr(struct hifn_softc *, int, u_int8_t *);
 154 static  int hifn_dmamap_load_src(struct hifn_softc *, struct hifn_command *);
 155 static  int hifn_dmamap_load_dst(struct hifn_softc *, struct hifn_command *);
 156 static  int hifn_init_pubrng(struct hifn_softc *);
 157 static  void hifn_rng(void *);
 158 static  void hifn_tick(void *);
 159 static  void hifn_abort(struct hifn_softc *);
 160 static  void hifn_alloc_slot(struct hifn_softc *, int *, int *, int *, int *);
 161
 162 static  void hifn_write_reg_0(struct hifn_softc *, bus_size_t, u_int32_t);
 163 static  void hifn_write_reg_1(struct hifn_softc *, bus_size_t, u_int32_t);
 164
 165 static __inline u_int32_t
 166 READ_REG_0(struct hifn_softc *sc, bus_size_t reg)
 167 {
 168     u_int32_t v = bus_space_read_4(sc->sc_st0, sc->sc_sh0, reg);
 169     sc->sc_bar0_lastreg = (bus_size_t) -1;
 170     return (v);
 171 }
 172 #define WRITE_REG_0(sc, reg, val)       hifn_write_reg_0(sc, reg, val)
 173
 174 static __inline u_int32_t
 175 READ_REG_1(struct hifn_softc *sc, bus_size_t reg)
 176 {
 177     u_int32_t v = bus_space_read_4(sc->sc_st1, sc->sc_sh1, reg);
 178     sc->sc_bar1_lastreg = (bus_size_t) -1;
 179     return (v);
 180 }
 181 #define WRITE_REG_1(sc, reg, val)       hifn_write_reg_1(sc, reg, val)
 182
 183 static SYSCTL_NODE(_hw, OID_AUTO, hifn, CTLFLAG_RD, 0,
 184             "Hifn driver parameters");
 185
 186 #ifdef HIFN_DEBUG
 187 static  int hifn_debug = 0;
 188 SYSCTL_INT(_hw_hifn, OID_AUTO, debug, CTLFLAG_RW, &hifn_debug,
 189             0, "control debugging msgs");
 190 #endif
 191
 192 static  struct hifn_stats hifnstats;
 193 SYSCTL_STRUCT(_hw_hifn, OID_AUTO, stats, CTLFLAG_RD, &hifnstats,
 194             hifn_stats, "driver statistics");
 195 static  int hifn_maxbatch = 1;
 196 SYSCTL_INT(_hw_hifn, OID_AUTO, maxbatch, CTLFLAG_RW, &hifn_maxbatch,
 197             0, "max ops to batch w/o interrupt");
 198
 199 /*
 200  * Probe for a supported device.  The PCI vendor and device
 201  * IDs are used to detect devices we know how to handle.
 202  */
 203 static int
 204 hifn_probe(device_t dev)
 205 {
 206         if (pci_get_vendor(dev) == PCI_VENDOR_INVERTEX &&
 207             pci_get_device(dev) == PCI_PRODUCT_INVERTEX_AEON)
 208                 return (BUS_PROBE_DEFAULT);
 209         if (pci_get_vendor(dev) == PCI_VENDOR_HIFN &&
 210             (pci_get_device(dev) == PCI_PRODUCT_HIFN_7751 ||
 211              pci_get_device(dev) == PCI_PRODUCT_HIFN_7951 ||
 212              pci_get_device(dev) == PCI_PRODUCT_HIFN_7955 ||
 213              pci_get_device(dev) == PCI_PRODUCT_HIFN_7956 ||
 214              pci_get_device(dev) == PCI_PRODUCT_HIFN_7811))
 215                 return (BUS_PROBE_DEFAULT);
 216         if (pci_get_vendor(dev) == PCI_VENDOR_NETSEC &&
 217             pci_get_device(dev) == PCI_PRODUCT_NETSEC_7751)
 218                 return (BUS_PROBE_DEFAULT);
 219         return (ENXIO);
 220 }
 221
 222 static void
 223 hifn_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 224 {
 225         bus_addr_t *paddr = (bus_addr_t*) arg;
 226         *paddr = segs->ds_addr;
 227 }
 228
 229 static const char*
 230 hifn_partname(struct hifn_softc *sc)
 231 {
 232         /* XXX sprintf numbers when not decoded */
 233         switch (pci_get_vendor(sc->sc_dev)) {
 234         case PCI_VENDOR_HIFN:
 235                 switch (pci_get_device(sc->sc_dev)) {
 236                 case PCI_PRODUCT_HIFN_6500:     return "Hifn 6500";
 237                 case PCI_PRODUCT_HIFN_7751:     return "Hifn 7751";
 238                 case PCI_PRODUCT_HIFN_7811:     return "Hifn 7811";
 239                 case PCI_PRODUCT_HIFN_7951:     return "Hifn 7951";
 240                 case PCI_PRODUCT_HIFN_7955:     return "Hifn 7955";
 241                 case PCI_PRODUCT_HIFN_7956:     return "Hifn 7956";
 242                 }
 243                 return "Hifn unknown-part";
 244         case PCI_VENDOR_INVERTEX:
 245                 switch (pci_get_device(sc->sc_dev)) {
 246                 case PCI_PRODUCT_INVERTEX_AEON: return "Invertex AEON";
 247                 }
 248                 return "Invertex unknown-part";
 249         case PCI_VENDOR_NETSEC:
 250                 switch (pci_get_device(sc->sc_dev)) {
 251                 case PCI_PRODUCT_NETSEC_7751:   return "NetSec 7751";
 252                 }
 253                 return "NetSec unknown-part";
 254         }
 255         return "Unknown-vendor unknown-part";
 256 }
 257
 258 static void
 259 default_harvest(struct rndtest_state *rsp, void *buf, u_int count)
 260 {
 261         /* MarkM: FIX!! Check that this does not swamp the harvester! */
 262         random_harvest_queue(buf, count, count*NBBY/2, RANDOM_PURE_HIFN);
 263 }
 264
 265 static u_int
 266 checkmaxmin(device_t dev, const char *what, u_int v, u_int min, u_int max)
 267 {
 268         if (v > max) {
 269                 device_printf(dev, "Warning, %s %u out of range, "
 270                         "using max %u\n", what, v, max);
 271                 v = max;
 272         } else if (v < min) {
 273                 device_printf(dev, "Warning, %s %u out of range, "
 274                         "using min %u\n", what, v, min);
 275                 v = min;
 276         }
 277         return v;
 278 }
 279
 280 /*
 281  * Select PLL configuration for 795x parts.  This is complicated in
 282  * that we cannot determine the optimal parameters without user input.
 283  * The reference clock is derived from an external clock through a
 284  * multiplier.  The external clock is either the host bus (i.e. PCI)
 285  * or an external clock generator.  When using the PCI bus we assume
 286  * the clock is either 33 or 66 MHz; for an external source we cannot
 287  * tell the speed.
 288  *
 289  * PLL configuration is done with a string: "pci" for PCI bus, or "ext"
 290  * for an external source, followed by the frequency.  We calculate
 291  * the appropriate multiplier and PLL register contents accordingly.
 292  * When no configuration is given we default to "pci66" since that
 293  * always will allow the card to work.  If a card is using the PCI
 294  * bus clock and in a 33MHz slot then it will be operating at half
 295  * speed until the correct information is provided.
 296  *
 297  * We use a default setting of "ext66" because according to Mike Ham
 298  * of HiFn, almost every board in existence has an external crystal
 299  * populated at 66Mhz. Using PCI can be a problem on modern motherboards,
 300  * because PCI33 can have clocks from 0 to 33Mhz, and some have
 301  * non-PCI-compliant spread-spectrum clocks, which can confuse the pll.
 302  */
 303 static void
 304 hifn_getpllconfig(device_t dev, u_int *pll)
 305 {
 306         const char *pllspec;
 307         u_int freq, mul, fl, fh;
 308         u_int32_t pllconfig;
 309         char *nxt;
 310
 311         if (resource_string_value("hifn", device_get_unit(dev),
 312             "pllconfig", &pllspec))
 313                 pllspec = "ext66";
 314         fl = 33, fh = 66;
 315         pllconfig = 0;
 316         if (strncmp(pllspec, "ext", 3) == 0) {
 317                 pllspec += 3;
 318                 pllconfig |= HIFN_PLL_REF_SEL;
 319                 switch (pci_get_device(dev)) {
 320                 case PCI_PRODUCT_HIFN_7955:
 321                 case PCI_PRODUCT_HIFN_7956:
 322                         fl = 20, fh = 100;
 323                         break;
 324 #ifdef notyet
 325                 case PCI_PRODUCT_HIFN_7954:
 326                         fl = 20, fh = 66;
 327                         break;
 328 #endif
 329                 }
 330         } else if (strncmp(pllspec, "pci", 3) == 0)
 331                 pllspec += 3;
 332         freq = strtoul(pllspec, &nxt, 10);
 333         if (nxt == pllspec)
 334                 freq = 66;
 335         else
 336                 freq = checkmaxmin(dev, "frequency", freq, fl, fh);
 337         /*
 338          * Calculate multiplier.  We target a Fck of 266 MHz,
 339          * allowing only even values, possibly rounded down.
 340          * Multipliers > 8 must set the charge pump current.
 341          */
 342         mul = checkmaxmin(dev, "PLL divisor", (266 / freq) &~ 1, 2, 12);
 343         pllconfig |= (mul / 2 - 1) << HIFN_PLL_ND_SHIFT;
 344         if (mul > 8)
 345                 pllconfig |= HIFN_PLL_IS;
 346         *pll = pllconfig;
 347 }
 348
 349 /*
 350  * Attach an interface that successfully probed.
 351  */
 352 static int
 353 hifn_attach(device_t dev)
 354 {
 355         struct hifn_softc *sc = device_get_softc(dev);
 356         caddr_t kva;
 357         int rseg, rid;
 358         char rbase;
 359         u_int16_t ena, rev;
 360
 361         sc->sc_dev = dev;
 362
 363         mtx_init(&sc->sc_mtx, device_get_nameunit(dev), "hifn driver", MTX_DEF);
 364
 365         /* XXX handle power management */
 366
 367         /*
 368          * The 7951 and 795x have a random number generator and
 369          * public key support; note this.
 370          */
 371         if (pci_get_vendor(dev) == PCI_VENDOR_HIFN &&
 372             (pci_get_device(dev) == PCI_PRODUCT_HIFN_7951 ||
 373              pci_get_device(dev) == PCI_PRODUCT_HIFN_7955 ||
 374              pci_get_device(dev) == PCI_PRODUCT_HIFN_7956))
 375                 sc->sc_flags = HIFN_HAS_RNG | HIFN_HAS_PUBLIC;
 376         /*
 377          * The 7811 has a random number generator and
 378          * we also note it's identity 'cuz of some quirks.
 379          */
 380         if (pci_get_vendor(dev) == PCI_VENDOR_HIFN &&
 381             pci_get_device(dev) == PCI_PRODUCT_HIFN_7811)
 382                 sc->sc_flags |= HIFN_IS_7811 | HIFN_HAS_RNG;
 383
 384         /*
 385          * The 795x parts support AES.
 386          */
 387         if (pci_get_vendor(dev) == PCI_VENDOR_HIFN &&
 388             (pci_get_device(dev) == PCI_PRODUCT_HIFN_7955 ||
 389              pci_get_device(dev) == PCI_PRODUCT_HIFN_7956)) {
 390                 sc->sc_flags |= HIFN_IS_7956 | HIFN_HAS_AES;
 391                 /*
 392                  * Select PLL configuration.  This depends on the
 393                  * bus and board design and must be manually configured
 394                  * if the default setting is unacceptable.
 395                  */
 396                 hifn_getpllconfig(dev, &sc->sc_pllconfig);
 397         }
 398
 399         /*
 400          * Setup PCI resources. Note that we record the bus
 401          * tag and handle for each register mapping, this is
 402          * used by the READ_REG_0, WRITE_REG_0, READ_REG_1,
 403          * and WRITE_REG_1 macros throughout the driver.
 404          */
 405         pci_enable_busmaster(dev);
 406
 407         rid = HIFN_BAR0;
 408         sc->sc_bar0res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 409                                                 RF_ACTIVE);
 410         if (sc->sc_bar0res == NULL) {
 411                 device_printf(dev, "cannot map bar%d register space\n", 0);
 412                 goto fail_pci;
 413         }
 414         sc->sc_st0 = rman_get_bustag(sc->sc_bar0res);
 415         sc->sc_sh0 = rman_get_bushandle(sc->sc_bar0res);
 416         sc->sc_bar0_lastreg = (bus_size_t) -1;
 417
 418         rid = HIFN_BAR1;
 419         sc->sc_bar1res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 420                                                 RF_ACTIVE);
 421         if (sc->sc_bar1res == NULL) {
 422                 device_printf(dev, "cannot map bar%d register space\n", 1);
 423                 goto fail_io0;
 424         }
 425         sc->sc_st1 = rman_get_bustag(sc->sc_bar1res);
 426         sc->sc_sh1 = rman_get_bushandle(sc->sc_bar1res);
 427         sc->sc_bar1_lastreg = (bus_size_t) -1;
 428
 429         hifn_set_retry(sc);
 430
 431         /*
 432          * Setup the area where the Hifn DMA's descriptors
 433          * and associated data structures.
 434          */
 435         if (bus_dma_tag_create(bus_get_dma_tag(dev),    /* PCI parent */
 436                                1, 0,                    /* alignment,boundary */
 437                                BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
 438                                BUS_SPACE_MAXADDR,       /* highaddr */
 439                                NULL, NULL,              /* filter, filterarg */
 440                                HIFN_MAX_DMALEN,         /* maxsize */
 441                                MAX_SCATTER,             /* nsegments */
 442                                HIFN_MAX_SEGLEN,         /* maxsegsize */
 443                                BUS_DMA_ALLOCNOW,        /* flags */
 444                                NULL,                    /* lockfunc */
 445                                NULL,                    /* lockarg */
 446                                &sc->sc_dmat)) {
 447                 device_printf(dev, "cannot allocate DMA tag\n");
 448                 goto fail_io1;
 449         }
 450         if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &sc->sc_dmamap)) {
 451                 device_printf(dev, "cannot create dma map\n");
 452                 bus_dma_tag_destroy(sc->sc_dmat);
 453                 goto fail_io1;
 454         }
 455         if (bus_dmamem_alloc(sc->sc_dmat, (void**) &kva, BUS_DMA_NOWAIT, &sc->sc_dmamap)) {
 456                 device_printf(dev, "cannot alloc dma buffer\n");
 457                 bus_dmamap_destroy(sc->sc_dmat, sc->sc_dmamap);
 458                 bus_dma_tag_destroy(sc->sc_dmat);
 459                 goto fail_io1;
 460         }
 461         if (bus_dmamap_load(sc->sc_dmat, sc->sc_dmamap, kva,
 462                              sizeof (*sc->sc_dma),
 463                              hifn_dmamap_cb, &sc->sc_dma_physaddr,
 464                              BUS_DMA_NOWAIT)) {
 465                 device_printf(dev, "cannot load dma map\n");
 466                 bus_dmamem_free(sc->sc_dmat, kva, sc->sc_dmamap);
 467                 bus_dma_tag_destroy(sc->sc_dmat);
 468                 goto fail_io1;
 469         }
 470         sc->sc_dma = (struct hifn_dma *)kva;
 471         bzero(sc->sc_dma, sizeof(*sc->sc_dma));
 472
 473         KASSERT(sc->sc_st0 != 0, ("hifn_attach: null bar0 tag!"));
 474         KASSERT(sc->sc_sh0 != 0, ("hifn_attach: null bar0 handle!"));
 475         KASSERT(sc->sc_st1 != 0, ("hifn_attach: null bar1 tag!"));
 476         KASSERT(sc->sc_sh1 != 0, ("hifn_attach: null bar1 handle!"));
 477
 478         /*
 479          * Reset the board and do the ``secret handshake''
 480          * to enable the crypto support.  Then complete the
 481          * initialization procedure by setting up the interrupt
 482          * and hooking in to the system crypto support so we'll
 483          * get used for system services like the crypto device,
 484          * IPsec, RNG device, etc.
 485          */
 486         hifn_reset_board(sc, 0);
 487
 488         if (hifn_enable_crypto(sc) != 0) {
 489                 device_printf(dev, "crypto enabling failed\n");
 490                 goto fail_mem;
 491         }
 492         hifn_reset_puc(sc);
 493
 494         hifn_init_dma(sc);
 495         hifn_init_pci_registers(sc);
 496
 497         /* XXX can't dynamically determine ram type for 795x; force dram */
 498         if (sc->sc_flags & HIFN_IS_7956)
 499                 sc->sc_drammodel = 1;
 500         else if (hifn_ramtype(sc))
 501                 goto fail_mem;
 502
 503         if (sc->sc_drammodel == 0)
 504                 hifn_sramsize(sc);
 505         else
 506                 hifn_dramsize(sc);
 507
 508         /*
 509          * Workaround for NetSec 7751 rev A: half ram size because two
 510          * of the address lines were left floating
 511          */
 512         if (pci_get_vendor(dev) == PCI_VENDOR_NETSEC &&
 513             pci_get_device(dev) == PCI_PRODUCT_NETSEC_7751 &&
 514             pci_get_revid(dev) == 0x61) /*XXX???*/
 515                 sc->sc_ramsize >>= 1;
 516
 517         /*
 518          * Arrange the interrupt line.
 519          */
 520         rid = 0;
 521         sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 522                                             RF_SHAREABLE|RF_ACTIVE);
 523         if (sc->sc_irq == NULL) {
 524                 device_printf(dev, "could not map interrupt\n");
 525                 goto fail_mem;
 526         }
 527         /*
 528          * NB: Network code assumes we are blocked with splimp()
 529          *     so make sure the IRQ is marked appropriately.
 530          */
 531         if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE,
 532                            NULL, hifn_intr, sc, &sc->sc_intrhand)) {
 533                 device_printf(dev, "could not setup interrupt\n");
 534                 goto fail_intr2;
 535         }
 536
 537         hifn_sessions(sc);
 538
 539         /*
 540          * NB: Keep only the low 16 bits; this masks the chip id
 541          *     from the 7951.
 542          */
 543         rev = READ_REG_1(sc, HIFN_1_REVID) & 0xffff;
 544
 545         rseg = sc->sc_ramsize / 1024;
 546         rbase = 'K';
 547         if (sc->sc_ramsize >= (1024 * 1024)) {
 548                 rbase = 'M';
 549                 rseg /= 1024;
 550         }
 551         device_printf(sc->sc_dev, "%s, rev %u, %d%cB %cram",
 552                 hifn_partname(sc), rev,
 553                 rseg, rbase, sc->sc_drammodel ? 'd' : 's');
 554         if (sc->sc_flags & HIFN_IS_7956)
 555                 printf(", pll=0x%x<%s clk, %ux mult>",
 556                         sc->sc_pllconfig,
 557                         sc->sc_pllconfig & HIFN_PLL_REF_SEL ? "ext" : "pci",
 558                         2 + 2*((sc->sc_pllconfig & HIFN_PLL_ND) >> 11));
 559         printf("\n");
 560
 561         sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE);
 562         if (sc->sc_cid < 0) {
 563                 device_printf(dev, "could not get crypto driver id\n");
 564                 goto fail_intr;
 565         }
 566
 567         WRITE_REG_0(sc, HIFN_0_PUCNFG,
 568             READ_REG_0(sc, HIFN_0_PUCNFG) | HIFN_PUCNFG_CHIPID);
 569         ena = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
 570
 571         switch (ena) {
 572         case HIFN_PUSTAT_ENA_2:
 573                 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
 574                 crypto_register(sc->sc_cid, CRYPTO_ARC4, 0, 0);
 575                 if (sc->sc_flags & HIFN_HAS_AES)
 576                         crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
 577                 /*FALLTHROUGH*/
 578         case HIFN_PUSTAT_ENA_1:
 579                 crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0);
 580                 crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0);
 581                 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
 582                 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
 583                 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
 584                 break;
 585         }
 586
 587         bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
 588             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 589
 590         if (sc->sc_flags & (HIFN_HAS_PUBLIC | HIFN_HAS_RNG))
 591                 hifn_init_pubrng(sc);
 592
 593         callout_init(&sc->sc_tickto, 1);
 594         callout_reset(&sc->sc_tickto, hz, hifn_tick, sc);
 595
 596         return (0);
 597
 598 fail_intr:
 599         bus_teardown_intr(dev, sc->sc_irq, sc->sc_intrhand);
 600 fail_intr2:
 601         /* XXX don't store rid */
 602         bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
 603 fail_mem:
 604         bus_dmamap_unload(sc->sc_dmat, sc->sc_dmamap);
 605         bus_dmamem_free(sc->sc_dmat, sc->sc_dma, sc->sc_dmamap);
 606         bus_dma_tag_destroy(sc->sc_dmat);
 607
 608         /* Turn off DMA polling */
 609         WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
 610             HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
 611 fail_io1:
 612         bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR1, sc->sc_bar1res);
 613 fail_io0:
 614         bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR0, sc->sc_bar0res);
 615 fail_pci:
 616         mtx_destroy(&sc->sc_mtx);
 617         return (ENXIO);
 618 }
 619
 620 /*
 621  * Detach an interface that successfully probed.
 622  */
 623 static int
 624 hifn_detach(device_t dev)
 625 {
 626         struct hifn_softc *sc = device_get_softc(dev);
 627
 628         KASSERT(sc != NULL, ("hifn_detach: null software carrier!"));
 629
 630         /* disable interrupts */
 631         WRITE_REG_1(sc, HIFN_1_DMA_IER, 0);
 632
 633         /*XXX other resources */
 634         callout_stop(&sc->sc_tickto);
 635         callout_stop(&sc->sc_rngto);
 636 #ifdef HIFN_RNDTEST
 637         if (sc->sc_rndtest)
 638                 rndtest_detach(sc->sc_rndtest);
 639 #endif
 640
 641         /* Turn off DMA polling */
 642         WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
 643             HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
 644
 645         crypto_unregister_all(sc->sc_cid);
 646
 647         bus_generic_detach(dev);        /*XXX should be no children, right? */
 648
 649         bus_teardown_intr(dev, sc->sc_irq, sc->sc_intrhand);
 650         /* XXX don't store rid */
 651         bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
 652
 653         bus_dmamap_unload(sc->sc_dmat, sc->sc_dmamap);
 654         bus_dmamem_free(sc->sc_dmat, sc->sc_dma, sc->sc_dmamap);
 655         bus_dma_tag_destroy(sc->sc_dmat);
 656
 657         bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR1, sc->sc_bar1res);
 658         bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR0, sc->sc_bar0res);
 659
 660         mtx_destroy(&sc->sc_mtx);
 661
 662         return (0);
 663 }
 664
 665 /*
 666  * Stop all chip I/O so that the kernel's probe routines don't
 667  * get confused by errant DMAs when rebooting.
 668  */
 669 static int
 670 hifn_shutdown(device_t dev)
 671 {
 672 #ifdef notyet
 673         hifn_stop(device_get_softc(dev));
 674 #endif
 675         return (0);
 676 }
 677
 678 /*
 679  * Device suspend routine.  Stop the interface and save some PCI
 680  * settings in case the BIOS doesn't restore them properly on
 681  * resume.
 682  */
 683 static int
 684 hifn_suspend(device_t dev)
 685 {
 686         struct hifn_softc *sc = device_get_softc(dev);
 687 #ifdef notyet
 688         hifn_stop(sc);
 689 #endif
 690         sc->sc_suspended = 1;
 691
 692         return (0);
 693 }
 694
 695 /*
 696  * Device resume routine.  Restore some PCI settings in case the BIOS
 697  * doesn't, re-enable busmastering, and restart the interface if
 698  * appropriate.
 699  */
 700 static int
 701 hifn_resume(device_t dev)
 702 {
 703         struct hifn_softc *sc = device_get_softc(dev);
 704 #ifdef notyet
 705         /* reinitialize interface if necessary */
 706         if (ifp->if_flags & IFF_UP)
 707                 rl_init(sc);
 708 #endif
 709         sc->sc_suspended = 0;
 710
 711         return (0);
 712 }
 713
 714 static int
 715 hifn_init_pubrng(struct hifn_softc *sc)
 716 {
 717         u_int32_t r;
 718         int i;
 719
 720 #ifdef HIFN_RNDTEST
 721         sc->sc_rndtest = rndtest_attach(sc->sc_dev);
 722         if (sc->sc_rndtest)
 723                 sc->sc_harvest = rndtest_harvest;
 724         else
 725                 sc->sc_harvest = default_harvest;
 726 #else
 727         sc->sc_harvest = default_harvest;
 728 #endif
 729         if ((sc->sc_flags & HIFN_IS_7811) == 0) {
 730                 /* Reset 7951 public key/rng engine */
 731                 WRITE_REG_1(sc, HIFN_1_PUB_RESET,
 732                     READ_REG_1(sc, HIFN_1_PUB_RESET) | HIFN_PUBRST_RESET);
 733
 734                 for (i = 0; i < 100; i++) {
 735                         DELAY(1000);
 736                         if ((READ_REG_1(sc, HIFN_1_PUB_RESET) &
 737                             HIFN_PUBRST_RESET) == 0)
 738                                 break;
 739                 }
 740
 741                 if (i == 100) {
 742                         device_printf(sc->sc_dev, "public key init failed\n");
 743                         return (1);
 744                 }
 745         }
 746
 747         /* Enable the rng, if available */
 748         if (sc->sc_flags & HIFN_HAS_RNG) {
 749                 if (sc->sc_flags & HIFN_IS_7811) {
 750                         r = READ_REG_1(sc, HIFN_1_7811_RNGENA);
 751                         if (r & HIFN_7811_RNGENA_ENA) {
 752                                 r &= ~HIFN_7811_RNGENA_ENA;
 753                                 WRITE_REG_1(sc, HIFN_1_7811_RNGENA, r);
 754                         }
 755                         WRITE_REG_1(sc, HIFN_1_7811_RNGCFG,
 756                             HIFN_7811_RNGCFG_DEFL);
 757                         r |= HIFN_7811_RNGENA_ENA;
 758                         WRITE_REG_1(sc, HIFN_1_7811_RNGENA, r);
 759                 } else
 760                         WRITE_REG_1(sc, HIFN_1_RNG_CONFIG,
 761                             READ_REG_1(sc, HIFN_1_RNG_CONFIG) |
 762                             HIFN_RNGCFG_ENA);
 763
 764                 sc->sc_rngfirst = 1;
 765                 if (hz >= 100)
 766                         sc->sc_rnghz = hz / 100;
 767                 else
 768                         sc->sc_rnghz = 1;
 769                 callout_init(&sc->sc_rngto, 1);
 770                 callout_reset(&sc->sc_rngto, sc->sc_rnghz, hifn_rng, sc);
 771         }
 772
 773         /* Enable public key engine, if available */
 774         if (sc->sc_flags & HIFN_HAS_PUBLIC) {
 775                 WRITE_REG_1(sc, HIFN_1_PUB_IEN, HIFN_PUBIEN_DONE);
 776                 sc->sc_dmaier |= HIFN_DMAIER_PUBDONE;
 777                 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
 778 #ifdef HIFN_VULCANDEV
 779                 sc->sc_pkdev = make_dev(&vulcanpk_cdevsw, 0,
 780                                         UID_ROOT, GID_WHEEL, 0666,
 781                                         "vulcanpk");
 782                 sc->sc_pkdev->si_drv1 = sc;
 783 #endif
 784         }
 785
 786         return (0);
 787 }
 788
 789 static void
 790 hifn_rng(void *vsc)
 791 {
 792 #define RANDOM_BITS(n)  (n)*sizeof (u_int32_t), (n)*sizeof (u_int32_t)*NBBY, 0
 793         struct hifn_softc *sc = vsc;
 794         u_int32_t sts, num[2];
 795         int i;
 796
 797         if (sc->sc_flags & HIFN_IS_7811) {
 798                 /* ONLY VALID ON 7811!!!! */
 799                 for (i = 0; i < 5; i++) {
 800                         sts = READ_REG_1(sc, HIFN_1_7811_RNGSTS);
 801                         if (sts & HIFN_7811_RNGSTS_UFL) {
 802                                 device_printf(sc->sc_dev,
 803                                               "RNG underflow: disabling\n");
 804                                 return;
 805                         }
 806                         if ((sts & HIFN_7811_RNGSTS_RDY) == 0)
 807                                 break;
 808
 809                         /*
 810                          * There are at least two words in the RNG FIFO
 811                          * at this point.
 812                          */
 813                         num[0] = READ_REG_1(sc, HIFN_1_7811_RNGDAT);
 814                         num[1] = READ_REG_1(sc, HIFN_1_7811_RNGDAT);
 815                         /* NB: discard first data read */
 816                         if (sc->sc_rngfirst)
 817                                 sc->sc_rngfirst = 0;
 818                         else
 819                                 (*sc->sc_harvest)(sc->sc_rndtest,
 820                                         num, sizeof (num));
 821                 }
 822         } else {
 823                 num[0] = READ_REG_1(sc, HIFN_1_RNG_DATA);
 824
 825                 /* NB: discard first data read */
 826                 if (sc->sc_rngfirst)
 827                         sc->sc_rngfirst = 0;
 828                 else
 829                         (*sc->sc_harvest)(sc->sc_rndtest,
 830                                 num, sizeof (num[0]));
 831         }
 832
 833         callout_reset(&sc->sc_rngto, sc->sc_rnghz, hifn_rng, sc);
 834 #undef RANDOM_BITS
 835 }
 836
 837 static void
 838 hifn_puc_wait(struct hifn_softc *sc)
 839 {
 840         int i;
 841         int reg = HIFN_0_PUCTRL;
 842
 843         if (sc->sc_flags & HIFN_IS_7956) {
 844                 reg = HIFN_0_PUCTRL2;
 845         }
 846
 847         for (i = 5000; i > 0; i--) {
 848                 DELAY(1);
 849                 if (!(READ_REG_0(sc, reg) & HIFN_PUCTRL_RESET))
 850                         break;
 851         }
 852         if (!i)
 853                 device_printf(sc->sc_dev, "proc unit did not reset\n");
 854 }
 855
 856 /*
 857  * Reset the processing unit.
 858  */
 859 static void
 860 hifn_reset_puc(struct hifn_softc *sc)
 861 {
 862         /* Reset processing unit */
 863         int reg = HIFN_0_PUCTRL;
 864
 865         if (sc->sc_flags & HIFN_IS_7956) {
 866                 reg = HIFN_0_PUCTRL2;
 867         }
 868         WRITE_REG_0(sc, reg, HIFN_PUCTRL_DMAENA);
 869
 870         hifn_puc_wait(sc);
 871 }
 872
 873 /*
 874  * Set the Retry and TRDY registers; note that we set them to
 875  * zero because the 7811 locks up when forced to retry (section
 876  * 3.6 of "Specification Update SU-0014-04".  Not clear if we
 877  * should do this for all Hifn parts, but it doesn't seem to hurt.
 878  */
 879 static void
 880 hifn_set_retry(struct hifn_softc *sc)
 881 {
 882         /* NB: RETRY only responds to 8-bit reads/writes */
 883         pci_write_config(sc->sc_dev, HIFN_RETRY_TIMEOUT, 0, 1);
 884         pci_write_config(sc->sc_dev, HIFN_TRDY_TIMEOUT, 0, 1);
 885 }
 886
 887 /*
 888  * Resets the board.  Values in the regesters are left as is
 889  * from the reset (i.e. initial values are assigned elsewhere).
 890  */
 891 static void
 892 hifn_reset_board(struct hifn_softc *sc, int full)
 893 {
 894         u_int32_t reg;
 895
 896         /*
 897          * Set polling in the DMA configuration register to zero.  0x7 avoids
 898          * resetting the board and zeros out the other fields.
 899          */
 900         WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
 901             HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
 902
 903         /*
 904          * Now that polling has been disabled, we have to wait 1 ms
 905          * before resetting the board.
 906          */
 907         DELAY(1000);
 908
 909         /* Reset the DMA unit */
 910         if (full) {
 911                 WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE);
 912                 DELAY(1000);
 913         } else {
 914                 WRITE_REG_1(sc, HIFN_1_DMA_CNFG,
 915                     HIFN_DMACNFG_MODE | HIFN_DMACNFG_MSTRESET);
 916                 hifn_reset_puc(sc);
 917         }
 918
 919         KASSERT(sc->sc_dma != NULL, ("hifn_reset_board: null DMA tag!"));
 920         bzero(sc->sc_dma, sizeof(*sc->sc_dma));
 921
 922         /* Bring dma unit out of reset */
 923         WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
 924             HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
 925
 926         hifn_puc_wait(sc);
 927         hifn_set_retry(sc);
 928
 929         if (sc->sc_flags & HIFN_IS_7811) {
 930                 for (reg = 0; reg < 1000; reg++) {
 931                         if (READ_REG_1(sc, HIFN_1_7811_MIPSRST) &
 932                             HIFN_MIPSRST_CRAMINIT)
 933                                 break;
 934                         DELAY(1000);
 935                 }
 936                 if (reg == 1000)
 937                         printf(": cram init timeout\n");
 938         } else {
 939           /* set up DMA configuration register #2 */
 940           /* turn off all PK and BAR0 swaps */
 941           WRITE_REG_1(sc, HIFN_1_DMA_CNFG2,
 942                       (3 << HIFN_DMACNFG2_INIT_WRITE_BURST_SHIFT)|
 943                       (3 << HIFN_DMACNFG2_INIT_READ_BURST_SHIFT)|
 944                       (2 << HIFN_DMACNFG2_TGT_WRITE_BURST_SHIFT)|
 945                       (2 << HIFN_DMACNFG2_TGT_READ_BURST_SHIFT));
 946         }
 947
 948 }
 949
 950 static u_int32_t
 951 hifn_next_signature(u_int32_t a, u_int cnt)
 952 {
 953         int i;
 954         u_int32_t v;
 955
 956         for (i = 0; i < cnt; i++) {
 957
 958                 /* get the parity */
 959                 v = a & 0x80080125;
 960                 v ^= v >> 16;
 961                 v ^= v >> 8;
 962                 v ^= v >> 4;
 963                 v ^= v >> 2;
 964                 v ^= v >> 1;
 965
 966                 a = (v & 1) ^ (a << 1);
 967         }
 968
 969         return a;
 970 }
 971
 972 struct pci2id {
 973         u_short         pci_vendor;
 974         u_short         pci_prod;
 975         char            card_id[13];
 976 };
 977 static struct pci2id pci2id[] = {
 978         {
 979                 PCI_VENDOR_HIFN,
 980                 PCI_PRODUCT_HIFN_7951,
 981                 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 982                   0x00, 0x00, 0x00, 0x00, 0x00 }
 983         }, {
 984                 PCI_VENDOR_HIFN,
 985                 PCI_PRODUCT_HIFN_7955,
 986                 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 987                   0x00, 0x00, 0x00, 0x00, 0x00 }
 988         }, {
 989                 PCI_VENDOR_HIFN,
 990                 PCI_PRODUCT_HIFN_7956,
 991                 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 992                   0x00, 0x00, 0x00, 0x00, 0x00 }
 993         }, {
 994                 PCI_VENDOR_NETSEC,
 995                 PCI_PRODUCT_NETSEC_7751,
 996                 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 997                   0x00, 0x00, 0x00, 0x00, 0x00 }
 998         }, {
 999                 PCI_VENDOR_INVERTEX,
1000                 PCI_PRODUCT_INVERTEX_AEON,
1001                 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1002                   0x00, 0x00, 0x00, 0x00, 0x00 }
1003         }, {
1004                 PCI_VENDOR_HIFN,
1005                 PCI_PRODUCT_HIFN_7811,
1006                 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1007                   0x00, 0x00, 0x00, 0x00, 0x00 }
1008         }, {
1009                 /*
1010                  * Other vendors share this PCI ID as well, such as
1011                  * http://www.powercrypt.com, and obviously they also
1012                  * use the same key.
1013                  */
1014                 PCI_VENDOR_HIFN,
1015                 PCI_PRODUCT_HIFN_7751,
1016                 { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1017                   0x00, 0x00, 0x00, 0x00, 0x00 }
1018         },
1019 };
1020
1021 /*
1022  * Checks to see if crypto is already enabled.  If crypto isn't enable,
1023  * "hifn_enable_crypto" is called to enable it.  The check is important,
1024  * as enabling crypto twice will lock the board.
1025  */
1026 static int
1027 hifn_enable_crypto(struct hifn_softc *sc)
1028 {
1029         u_int32_t dmacfg, ramcfg, encl, addr, i;
1030         char *offtbl = NULL;
1031
1032         for (i = 0; i < nitems(pci2id); i++) {
1033                 if (pci2id[i].pci_vendor == pci_get_vendor(sc->sc_dev) &&
1034                     pci2id[i].pci_prod == pci_get_device(sc->sc_dev)) {
1035                         offtbl = pci2id[i].card_id;
1036                         break;
1037                 }
1038         }
1039         if (offtbl == NULL) {
1040                 device_printf(sc->sc_dev, "Unknown card!\n");
1041                 return (1);
1042         }
1043
1044         ramcfg = READ_REG_0(sc, HIFN_0_PUCNFG);
1045         dmacfg = READ_REG_1(sc, HIFN_1_DMA_CNFG);
1046
1047         /*
1048          * The RAM config register's encrypt level bit needs to be set before
1049          * every read performed on the encryption level register.
1050          */
1051         WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg | HIFN_PUCNFG_CHIPID);
1052
1053         encl = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
1054
1055         /*
1056          * Make sure we don't re-unlock.  Two unlocks kills chip until the
1057          * next reboot.
1058          */
1059         if (encl == HIFN_PUSTAT_ENA_1 || encl == HIFN_PUSTAT_ENA_2) {
1060 #ifdef HIFN_DEBUG
1061                 if (hifn_debug)
1062                         device_printf(sc->sc_dev,
1063                             "Strong crypto already enabled!\n");
1064 #endif
1065                 goto report;
1066         }
1067
1068         if (encl != 0 && encl != HIFN_PUSTAT_ENA_0) {
1069 #ifdef HIFN_DEBUG
1070                 if (hifn_debug)
1071                         device_printf(sc->sc_dev,
1072                               "Unknown encryption level 0x%x\n", encl);
1073 #endif
1074                 return 1;
1075         }
1076
1077         WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_UNLOCK |
1078             HIFN_DMACNFG_MSTRESET | HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
1079         DELAY(1000);
1080         addr = READ_REG_1(sc, HIFN_UNLOCK_SECRET1);
1081         DELAY(1000);
1082         WRITE_REG_1(sc, HIFN_UNLOCK_SECRET2, 0);
1083         DELAY(1000);
1084
1085         for (i = 0; i <= 12; i++) {
1086                 addr = hifn_next_signature(addr, offtbl[i] + 0x101);
1087                 WRITE_REG_1(sc, HIFN_UNLOCK_SECRET2, addr);
1088
1089                 DELAY(1000);
1090         }
1091
1092         WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg | HIFN_PUCNFG_CHIPID);
1093         encl = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
1094
1095 #ifdef HIFN_DEBUG
1096         if (hifn_debug) {
1097                 if (encl != HIFN_PUSTAT_ENA_1 && encl != HIFN_PUSTAT_ENA_2)
1098                         device_printf(sc->sc_dev, "Engine is permanently "
1099                                 "locked until next system reset!\n");
1100                 else
1101                         device_printf(sc->sc_dev, "Engine enabled "
1102                                 "successfully!\n");
1103         }
1104 #endif
1105
1106 report:
1107         WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg);
1108         WRITE_REG_1(sc, HIFN_1_DMA_CNFG, dmacfg);
1109
1110         switch (encl) {
1111         case HIFN_PUSTAT_ENA_1:
1112         case HIFN_PUSTAT_ENA_2:
1113                 break;
1114         case HIFN_PUSTAT_ENA_0:
1115         default:
1116                 device_printf(sc->sc_dev, "disabled");
1117                 break;
1118         }
1119
1120         return 0;
1121 }
1122
1123 /*
1124  * Give initial values to the registers listed in the "Register Space"
1125  * section of the HIFN Software Development reference manual.
1126  */
1127 static void
1128 hifn_init_pci_registers(struct hifn_softc *sc)
1129 {
1130         /* write fixed values needed by the Initialization registers */
1131         WRITE_REG_0(sc, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);
1132         WRITE_REG_0(sc, HIFN_0_FIFOCNFG, HIFN_FIFOCNFG_THRESHOLD);
1133         WRITE_REG_0(sc, HIFN_0_PUIER, HIFN_PUIER_DSTOVER);
1134
1135         /* write all 4 ring address registers */
1136         WRITE_REG_1(sc, HIFN_1_DMA_CRAR, sc->sc_dma_physaddr +
1137             offsetof(struct hifn_dma, cmdr[0]));
1138         WRITE_REG_1(sc, HIFN_1_DMA_SRAR, sc->sc_dma_physaddr +
1139             offsetof(struct hifn_dma, srcr[0]));
1140         WRITE_REG_1(sc, HIFN_1_DMA_DRAR, sc->sc_dma_physaddr +
1141             offsetof(struct hifn_dma, dstr[0]));
1142         WRITE_REG_1(sc, HIFN_1_DMA_RRAR, sc->sc_dma_physaddr +
1143             offsetof(struct hifn_dma, resr[0]));
1144
1145         DELAY(2000);
1146
1147         /* write status register */
1148         WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1149             HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS |
1150             HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS |
1151             HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST |
1152             HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER |
1153             HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST |
1154             HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER |
1155             HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST |
1156             HIFN_DMACSR_S_WAIT |
1157             HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST |
1158             HIFN_DMACSR_C_WAIT |
1159             HIFN_DMACSR_ENGINE |
1160             ((sc->sc_flags & HIFN_HAS_PUBLIC) ?
1161                 HIFN_DMACSR_PUBDONE : 0) |
1162             ((sc->sc_flags & HIFN_IS_7811) ?
1163                 HIFN_DMACSR_ILLW | HIFN_DMACSR_ILLR : 0));
1164
1165         sc->sc_d_busy = sc->sc_r_busy = sc->sc_s_busy = sc->sc_c_busy = 0;
1166         sc->sc_dmaier |= HIFN_DMAIER_R_DONE | HIFN_DMAIER_C_ABORT |
1167             HIFN_DMAIER_D_OVER | HIFN_DMAIER_R_OVER |
1168             HIFN_DMAIER_S_ABORT | HIFN_DMAIER_D_ABORT | HIFN_DMAIER_R_ABORT |
1169             ((sc->sc_flags & HIFN_IS_7811) ?
1170                 HIFN_DMAIER_ILLW | HIFN_DMAIER_ILLR : 0);
1171         sc->sc_dmaier &= ~HIFN_DMAIER_C_WAIT;
1172         WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
1173
1174
1175         if (sc->sc_flags & HIFN_IS_7956) {
1176                 u_int32_t pll;
1177
1178                 WRITE_REG_0(sc, HIFN_0_PUCNFG, HIFN_PUCNFG_COMPSING |
1179                     HIFN_PUCNFG_TCALLPHASES |
1180                     HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32);
1181
1182                 /* turn off the clocks and insure bypass is set */
1183                 pll = READ_REG_1(sc, HIFN_1_PLL);
1184                 pll = (pll &~ (HIFN_PLL_PK_CLK_SEL | HIFN_PLL_PE_CLK_SEL))
1185                   | HIFN_PLL_BP | HIFN_PLL_MBSET;
1186                 WRITE_REG_1(sc, HIFN_1_PLL, pll);
1187                 DELAY(10*1000);         /* 10ms */
1188
1189                 /* change configuration */
1190                 pll = (pll &~ HIFN_PLL_CONFIG) | sc->sc_pllconfig;
1191                 WRITE_REG_1(sc, HIFN_1_PLL, pll);
1192                 DELAY(10*1000);         /* 10ms */
1193
1194                 /* disable bypass */
1195                 pll &= ~HIFN_PLL_BP;
1196                 WRITE_REG_1(sc, HIFN_1_PLL, pll);
1197                 /* enable clocks with new configuration */
1198                 pll |= HIFN_PLL_PK_CLK_SEL | HIFN_PLL_PE_CLK_SEL;
1199                 WRITE_REG_1(sc, HIFN_1_PLL, pll);
1200         } else {
1201                 WRITE_REG_0(sc, HIFN_0_PUCNFG, HIFN_PUCNFG_COMPSING |
1202                     HIFN_PUCNFG_DRFR_128 | HIFN_PUCNFG_TCALLPHASES |
1203                     HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32 |
1204                     (sc->sc_drammodel ? HIFN_PUCNFG_DRAM : HIFN_PUCNFG_SRAM));
1205         }
1206
1207         WRITE_REG_0(sc, HIFN_0_PUISR, HIFN_PUISR_DSTOVER);
1208         WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
1209             HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE | HIFN_DMACNFG_LAST |
1210             ((HIFN_POLL_FREQUENCY << 16 ) & HIFN_DMACNFG_POLLFREQ) |
1211             ((HIFN_POLL_SCALAR << 8) & HIFN_DMACNFG_POLLINVAL));
1212 }
1213
1214 /*
1215  * The maximum number of sessions supported by the card
1216  * is dependent on the amount of context ram, which
1217  * encryption algorithms are enabled, and how compression
1218  * is configured.  This should be configured before this
1219  * routine is called.
1220  */
1221 static void
1222 hifn_sessions(struct hifn_softc *sc)
1223 {
1224         u_int32_t pucnfg;
1225         int ctxsize;
1226
1227         pucnfg = READ_REG_0(sc, HIFN_0_PUCNFG);
1228
1229         if (pucnfg & HIFN_PUCNFG_COMPSING) {
1230                 if (pucnfg & HIFN_PUCNFG_ENCCNFG)
1231                         ctxsize = 128;
1232                 else
1233                         ctxsize = 512;
1234                 /*
1235                  * 7955/7956 has internal context memory of 32K
1236                  */
1237                 if (sc->sc_flags & HIFN_IS_7956)
1238                         sc->sc_maxses = 32768 / ctxsize;
1239                 else
1240                         sc->sc_maxses = 1 +
1241                             ((sc->sc_ramsize - 32768) / ctxsize);
1242         } else
1243                 sc->sc_maxses = sc->sc_ramsize / 16384;
1244
1245         if (sc->sc_maxses > 2048)
1246                 sc->sc_maxses = 2048;
1247 }
1248
1249 /*
1250  * Determine ram type (sram or dram).  Board should be just out of a reset
1251  * state when this is called.
1252  */
1253 static int
1254 hifn_ramtype(struct hifn_softc *sc)
1255 {
1256         u_int8_t data[8], dataexpect[8];
1257         int i;
1258
1259         for (i = 0; i < sizeof(data); i++)
1260                 data[i] = dataexpect[i] = 0x55;
1261         if (hifn_writeramaddr(sc, 0, data))
1262                 return (-1);
1263         if (hifn_readramaddr(sc, 0, data))
1264                 return (-1);
1265         if (bcmp(data, dataexpect, sizeof(data)) != 0) {
1266                 sc->sc_drammodel = 1;
1267                 return (0);
1268         }
1269
1270         for (i = 0; i < sizeof(data); i++)
1271                 data[i] = dataexpect[i] = 0xaa;
1272         if (hifn_writeramaddr(sc, 0, data))
1273                 return (-1);
1274         if (hifn_readramaddr(sc, 0, data))
1275                 return (-1);
1276         if (bcmp(data, dataexpect, sizeof(data)) != 0) {
1277                 sc->sc_drammodel = 1;
1278                 return (0);
1279         }
1280
1281         return (0);
1282 }
1283
1284 #define HIFN_SRAM_MAX           (32 << 20)
1285 #define HIFN_SRAM_STEP_SIZE     16384
1286 #define HIFN_SRAM_GRANULARITY   (HIFN_SRAM_MAX / HIFN_SRAM_STEP_SIZE)
1287
1288 static int
1289 hifn_sramsize(struct hifn_softc *sc)
1290 {
1291         u_int32_t a;
1292         u_int8_t data[8];
1293         u_int8_t dataexpect[sizeof(data)];
1294         int32_t i;
1295
1296         for (i = 0; i < sizeof(data); i++)
1297                 data[i] = dataexpect[i] = i ^ 0x5a;
1298
1299         for (i = HIFN_SRAM_GRANULARITY - 1; i >= 0; i--) {
1300                 a = i * HIFN_SRAM_STEP_SIZE;
1301                 bcopy(&i, data, sizeof(i));
1302                 hifn_writeramaddr(sc, a, data);
1303         }
1304
1305         for (i = 0; i < HIFN_SRAM_GRANULARITY; i++) {
1306                 a = i * HIFN_SRAM_STEP_SIZE;
1307                 bcopy(&i, dataexpect, sizeof(i));
1308                 if (hifn_readramaddr(sc, a, data) < 0)
1309                         return (0);
1310                 if (bcmp(data, dataexpect, sizeof(data)) != 0)
1311                         return (0);
1312                 sc->sc_ramsize = a + HIFN_SRAM_STEP_SIZE;
1313         }
1314
1315         return (0);
1316 }
1317
1318 /*
1319  * XXX For dram boards, one should really try all of the
1320  * HIFN_PUCNFG_DSZ_*'s.  This just assumes that PUCNFG
1321  * is already set up correctly.
1322  */
1323 static int
1324 hifn_dramsize(struct hifn_softc *sc)
1325 {
1326         u_int32_t cnfg;
1327
1328         if (sc->sc_flags & HIFN_IS_7956) {
1329                 /*
1330                  * 7955/7956 have a fixed internal ram of only 32K.
1331                  */
1332                 sc->sc_ramsize = 32768;
1333         } else {
1334                 cnfg = READ_REG_0(sc, HIFN_0_PUCNFG) &
1335                     HIFN_PUCNFG_DRAMMASK;
1336                 sc->sc_ramsize = 1 << ((cnfg >> 13) + 18);
1337         }
1338         return (0);
1339 }
1340
1341 static void
1342 hifn_alloc_slot(struct hifn_softc *sc, int *cmdp, int *srcp, int *dstp, int *resp)
1343 {
1344         struct hifn_dma *dma = sc->sc_dma;
1345
1346         if (sc->sc_cmdi == HIFN_D_CMD_RSIZE) {
1347                 sc->sc_cmdi = 0;
1348                 dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID |
1349                     HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1350                 HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE,
1351                     BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1352         }
1353         *cmdp = sc->sc_cmdi++;
1354         sc->sc_cmdk = sc->sc_cmdi;
1355
1356         if (sc->sc_srci == HIFN_D_SRC_RSIZE) {
1357                 sc->sc_srci = 0;
1358                 dma->srcr[HIFN_D_SRC_RSIZE].l = htole32(HIFN_D_VALID |
1359                     HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1360                 HIFN_SRCR_SYNC(sc, HIFN_D_SRC_RSIZE,
1361                     BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1362         }
1363         *srcp = sc->sc_srci++;
1364         sc->sc_srck = sc->sc_srci;
1365
1366         if (sc->sc_dsti == HIFN_D_DST_RSIZE) {
1367                 sc->sc_dsti = 0;
1368                 dma->dstr[HIFN_D_DST_RSIZE].l = htole32(HIFN_D_VALID |
1369                     HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1370                 HIFN_DSTR_SYNC(sc, HIFN_D_DST_RSIZE,
1371                     BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1372         }
1373         *dstp = sc->sc_dsti++;
1374         sc->sc_dstk = sc->sc_dsti;
1375
1376         if (sc->sc_resi == HIFN_D_RES_RSIZE) {
1377                 sc->sc_resi = 0;
1378                 dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID |
1379                     HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1380                 HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE,
1381                     BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1382         }
1383         *resp = sc->sc_resi++;
1384         sc->sc_resk = sc->sc_resi;
1385 }
1386
1387 static int
1388 hifn_writeramaddr(struct hifn_softc *sc, int addr, u_int8_t *data)
1389 {
1390         struct hifn_dma *dma = sc->sc_dma;
1391         hifn_base_command_t wc;
1392         const u_int32_t masks = HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ;
1393         int r, cmdi, resi, srci, dsti;
1394
1395         wc.masks = htole16(3 << 13);
1396         wc.session_num = htole16(addr >> 14);
1397         wc.total_source_count = htole16(8);
1398         wc.total_dest_count = htole16(addr & 0x3fff);
1399
1400         hifn_alloc_slot(sc, &cmdi, &srci, &dsti, &resi);
1401
1402         WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1403             HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA |
1404             HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA);
1405
1406         /* build write command */
1407         bzero(dma->command_bufs[cmdi], HIFN_MAX_COMMAND);
1408         *(hifn_base_command_t *)dma->command_bufs[cmdi] = wc;
1409         bcopy(data, &dma->test_src, sizeof(dma->test_src));
1410
1411         dma->srcr[srci].p = htole32(sc->sc_dma_physaddr
1412             + offsetof(struct hifn_dma, test_src));
1413         dma->dstr[dsti].p = htole32(sc->sc_dma_physaddr
1414             + offsetof(struct hifn_dma, test_dst));
1415
1416         dma->cmdr[cmdi].l = htole32(16 | masks);
1417         dma->srcr[srci].l = htole32(8 | masks);
1418         dma->dstr[dsti].l = htole32(4 | masks);
1419         dma->resr[resi].l = htole32(4 | masks);
1420
1421         bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1422             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1423
1424         for (r = 10000; r >= 0; r--) {
1425                 DELAY(10);
1426                 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1427                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1428                 if ((dma->resr[resi].l & htole32(HIFN_D_VALID)) == 0)
1429                         break;
1430                 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1431                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1432         }
1433         if (r == 0) {
1434                 device_printf(sc->sc_dev, "writeramaddr -- "
1435                     "result[%d](addr %d) still valid\n", resi, addr);
1436                 r = -1;
1437                 return (-1);
1438         } else
1439                 r = 0;
1440
1441         WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1442             HIFN_DMACSR_C_CTRL_DIS | HIFN_DMACSR_S_CTRL_DIS |
1443             HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS);
1444
1445         return (r);
1446 }
1447
1448 static int
1449 hifn_readramaddr(struct hifn_softc *sc, int addr, u_int8_t *data)
1450 {
1451         struct hifn_dma *dma = sc->sc_dma;
1452         hifn_base_command_t rc;
1453         const u_int32_t masks = HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ;
1454         int r, cmdi, srci, dsti, resi;
1455
1456         rc.masks = htole16(2 << 13);
1457         rc.session_num = htole16(addr >> 14);
1458         rc.total_source_count = htole16(addr & 0x3fff);
1459         rc.total_dest_count = htole16(8);
1460
1461         hifn_alloc_slot(sc, &cmdi, &srci, &dsti, &resi);
1462
1463         WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1464             HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA |
1465             HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA);
1466
1467         bzero(dma->command_bufs[cmdi], HIFN_MAX_COMMAND);
1468         *(hifn_base_command_t *)dma->command_bufs[cmdi] = rc;
1469
1470         dma->srcr[srci].p = htole32(sc->sc_dma_physaddr +
1471             offsetof(struct hifn_dma, test_src));
1472         dma->test_src = 0;
1473         dma->dstr[dsti].p =  htole32(sc->sc_dma_physaddr +
1474             offsetof(struct hifn_dma, test_dst));
1475         dma->test_dst = 0;
1476         dma->cmdr[cmdi].l = htole32(8 | masks);
1477         dma->srcr[srci].l = htole32(8 | masks);
1478         dma->dstr[dsti].l = htole32(8 | masks);
1479         dma->resr[resi].l = htole32(HIFN_MAX_RESULT | masks);
1480
1481         bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1482             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1483
1484         for (r = 10000; r >= 0; r--) {
1485                 DELAY(10);
1486                 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1487                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1488                 if ((dma->resr[resi].l & htole32(HIFN_D_VALID)) == 0)
1489                         break;
1490                 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
1491                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1492         }
1493         if (r == 0) {
1494                 device_printf(sc->sc_dev, "readramaddr -- "
1495                     "result[%d](addr %d) still valid\n", resi, addr);
1496                 r = -1;
1497         } else {
1498                 r = 0;
1499                 bcopy(&dma->test_dst, data, sizeof(dma->test_dst));
1500         }
1501
1502         WRITE_REG_1(sc, HIFN_1_DMA_CSR,
1503             HIFN_DMACSR_C_CTRL_DIS | HIFN_DMACSR_S_CTRL_DIS |
1504             HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS);
1505
1506         return (r);
1507 }
1508
1509 /*
1510  * Initialize the descriptor rings.
1511  */
1512 static void
1513 hifn_init_dma(struct hifn_softc *sc)
1514 {
1515         struct hifn_dma *dma = sc->sc_dma;
1516         int i;
1517
1518         hifn_set_retry(sc);
1519
1520         /* initialize static pointer values */
1521         for (i = 0; i < HIFN_D_CMD_RSIZE; i++)
1522                 dma->cmdr[i].p = htole32(sc->sc_dma_physaddr +
1523                     offsetof(struct hifn_dma, command_bufs[i][0]));
1524         for (i = 0; i < HIFN_D_RES_RSIZE; i++)
1525                 dma->resr[i].p = htole32(sc->sc_dma_physaddr +
1526                     offsetof(struct hifn_dma, result_bufs[i][0]));
1527
1528         dma->cmdr[HIFN_D_CMD_RSIZE].p =
1529             htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, cmdr[0]));
1530         dma->srcr[HIFN_D_SRC_RSIZE].p =
1531             htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, srcr[0]));
1532         dma->dstr[HIFN_D_DST_RSIZE].p =
1533             htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, dstr[0]));
1534         dma->resr[HIFN_D_RES_RSIZE].p =
1535             htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, resr[0]));
1536
1537         sc->sc_cmdu = sc->sc_srcu = sc->sc_dstu = sc->sc_resu = 0;
1538         sc->sc_cmdi = sc->sc_srci = sc->sc_dsti = sc->sc_resi = 0;
1539         sc->sc_cmdk = sc->sc_srck = sc->sc_dstk = sc->sc_resk = 0;
1540 }
1541
1542 /*
1543  * Writes out the raw command buffer space.  Returns the
1544  * command buffer size.
1545  */
1546 static u_int
1547 hifn_write_command(struct hifn_command *cmd, u_int8_t *buf)
1548 {
1549         u_int8_t *buf_pos;
1550         hifn_base_command_t *base_cmd;
1551         hifn_mac_command_t *mac_cmd;
1552         hifn_crypt_command_t *cry_cmd;
1553         int using_mac, using_crypt, len, ivlen;
1554         u_int32_t dlen, slen;
1555
1556         buf_pos = buf;
1557         using_mac = cmd->base_masks & HIFN_BASE_CMD_MAC;
1558         using_crypt = cmd->base_masks & HIFN_BASE_CMD_CRYPT;
1559
1560         base_cmd = (hifn_base_command_t *)buf_pos;
1561         base_cmd->masks = htole16(cmd->base_masks);
1562         slen = cmd->src_mapsize;
1563         if (cmd->sloplen)
1564                 dlen = cmd->dst_mapsize - cmd->sloplen + sizeof(u_int32_t);
1565         else
1566                 dlen = cmd->dst_mapsize;
1567         base_cmd->total_source_count = htole16(slen & HIFN_BASE_CMD_LENMASK_LO);
1568         base_cmd->total_dest_count = htole16(dlen & HIFN_BASE_CMD_LENMASK_LO);
1569         dlen >>= 16;
1570         slen >>= 16;
1571         base_cmd->session_num = htole16(
1572             ((slen << HIFN_BASE_CMD_SRCLEN_S) & HIFN_BASE_CMD_SRCLEN_M) |
1573             ((dlen << HIFN_BASE_CMD_DSTLEN_S) & HIFN_BASE_CMD_DSTLEN_M));
1574         buf_pos += sizeof(hifn_base_command_t);
1575
1576         if (using_mac) {
1577                 mac_cmd = (hifn_mac_command_t *)buf_pos;
1578                 dlen = cmd->maccrd->crd_len;
1579                 mac_cmd->source_count = htole16(dlen & 0xffff);
1580                 dlen >>= 16;
1581                 mac_cmd->masks = htole16(cmd->mac_masks |
1582                     ((dlen << HIFN_MAC_CMD_SRCLEN_S) & HIFN_MAC_CMD_SRCLEN_M));
1583                 mac_cmd->header_skip = htole16(cmd->maccrd->crd_skip);
1584                 mac_cmd->reserved = 0;
1585                 buf_pos += sizeof(hifn_mac_command_t);
1586         }
1587
1588         if (using_crypt) {
1589                 cry_cmd = (hifn_crypt_command_t *)buf_pos;
1590                 dlen = cmd->enccrd->crd_len;
1591                 cry_cmd->source_count = htole16(dlen & 0xffff);
1592                 dlen >>= 16;
1593                 cry_cmd->masks = htole16(cmd->cry_masks |
1594                     ((dlen << HIFN_CRYPT_CMD_SRCLEN_S) & HIFN_CRYPT_CMD_SRCLEN_M));
1595                 cry_cmd->header_skip = htole16(cmd->enccrd->crd_skip);
1596                 cry_cmd->reserved = 0;
1597                 buf_pos += sizeof(hifn_crypt_command_t);
1598         }
1599
1600         if (using_mac && cmd->mac_masks & HIFN_MAC_CMD_NEW_KEY) {
1601                 bcopy(cmd->mac, buf_pos, HIFN_MAC_KEY_LENGTH);
1602                 buf_pos += HIFN_MAC_KEY_LENGTH;
1603         }
1604
1605         if (using_crypt && cmd->cry_masks & HIFN_CRYPT_CMD_NEW_KEY) {
1606                 switch (cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) {
1607                 case HIFN_CRYPT_CMD_ALG_3DES:
1608                         bcopy(cmd->ck, buf_pos, HIFN_3DES_KEY_LENGTH);
1609                         buf_pos += HIFN_3DES_KEY_LENGTH;
1610                         break;
1611                 case HIFN_CRYPT_CMD_ALG_DES:
1612                         bcopy(cmd->ck, buf_pos, HIFN_DES_KEY_LENGTH);
1613                         buf_pos += HIFN_DES_KEY_LENGTH;
1614                         break;
1615                 case HIFN_CRYPT_CMD_ALG_RC4:
1616                         len = 256;
1617                         do {
1618                                 int clen;
1619
1620                                 clen = MIN(cmd->cklen, len);
1621                                 bcopy(cmd->ck, buf_pos, clen);
1622                                 len -= clen;
1623                                 buf_pos += clen;
1624                         } while (len > 0);
1625                         bzero(buf_pos, 4);
1626                         buf_pos += 4;
1627                         break;
1628                 case HIFN_CRYPT_CMD_ALG_AES:
1629                         /*
1630                          * AES keys are variable 128, 192 and
1631                          * 256 bits (16, 24 and 32 bytes).
1632                          */
1633                         bcopy(cmd->ck, buf_pos, cmd->cklen);
1634                         buf_pos += cmd->cklen;
1635                         break;
1636                 }
1637         }
1638
1639         if (using_crypt && cmd->cry_masks & HIFN_CRYPT_CMD_NEW_IV) {
1640                 switch (cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) {
1641                 case HIFN_CRYPT_CMD_ALG_AES:
1642                         ivlen = HIFN_AES_IV_LENGTH;
1643                         break;
1644                 default:
1645                         ivlen = HIFN_IV_LENGTH;
1646                         break;
1647                 }
1648                 bcopy(cmd->iv, buf_pos, ivlen);
1649                 buf_pos += ivlen;
1650         }
1651
1652         if ((cmd->base_masks & (HIFN_BASE_CMD_MAC|HIFN_BASE_CMD_CRYPT)) == 0) {
1653                 bzero(buf_pos, 8);
1654                 buf_pos += 8;
1655         }
1656
1657         return (buf_pos - buf);
1658 }
1659
1660 static int
1661 hifn_dmamap_aligned(struct hifn_operand *op)
1662 {
1663         int i;
1664
1665         for (i = 0; i < op->nsegs; i++) {
1666                 if (op->segs[i].ds_addr & 3)
1667                         return (0);
1668                 if ((i != (op->nsegs - 1)) && (op->segs[i].ds_len & 3))
1669                         return (0);
1670         }
1671         return (1);
1672 }
1673
1674 static __inline int
1675 hifn_dmamap_dstwrap(struct hifn_softc *sc, int idx)
1676 {
1677         struct hifn_dma *dma = sc->sc_dma;
1678
1679         if (++idx == HIFN_D_DST_RSIZE) {
1680                 dma->dstr[idx].l = htole32(HIFN_D_VALID | HIFN_D_JUMP |
1681                     HIFN_D_MASKDONEIRQ);
1682                 HIFN_DSTR_SYNC(sc, idx,
1683                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1684                 idx = 0;
1685         }
1686         return (idx);
1687 }
1688
1689 static int
1690 hifn_dmamap_load_dst(struct hifn_softc *sc, struct hifn_command *cmd)
1691 {
1692         struct hifn_dma *dma = sc->sc_dma;
1693         struct hifn_operand *dst = &cmd->dst;
1694         u_int32_t p, l;
1695         int idx, used = 0, i;
1696
1697         idx = sc->sc_dsti;
1698         for (i = 0; i < dst->nsegs - 1; i++) {
1699                 dma->dstr[idx].p = htole32(dst->segs[i].ds_addr);
1700                 dma->dstr[idx].l = htole32(HIFN_D_VALID |
1701                     HIFN_D_MASKDONEIRQ | dst->segs[i].ds_len);
1702                 HIFN_DSTR_SYNC(sc, idx,
1703                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1704                 used++;
1705
1706                 idx = hifn_dmamap_dstwrap(sc, idx);
1707         }
1708
1709         if (cmd->sloplen == 0) {
1710                 p = dst->segs[i].ds_addr;
1711                 l = HIFN_D_VALID | HIFN_D_MASKDONEIRQ | HIFN_D_LAST |
1712                     dst->segs[i].ds_len;
1713         } else {
1714                 p = sc->sc_dma_physaddr +
1715                     offsetof(struct hifn_dma, slop[cmd->slopidx]);
1716                 l = HIFN_D_VALID | HIFN_D_MASKDONEIRQ | HIFN_D_LAST |
1717                     sizeof(u_int32_t);
1718
1719                 if ((dst->segs[i].ds_len - cmd->sloplen) != 0) {
1720                         dma->dstr[idx].p = htole32(dst->segs[i].ds_addr);
1721                         dma->dstr[idx].l = htole32(HIFN_D_VALID |
1722                             HIFN_D_MASKDONEIRQ |
1723                             (dst->segs[i].ds_len - cmd->sloplen));
1724                         HIFN_DSTR_SYNC(sc, idx,
1725                             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1726                         used++;
1727
1728                         idx = hifn_dmamap_dstwrap(sc, idx);
1729                 }
1730         }
1731         dma->dstr[idx].p = htole32(p);
1732         dma->dstr[idx].l = htole32(l);
1733         HIFN_DSTR_SYNC(sc, idx, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1734         used++;
1735
1736         idx = hifn_dmamap_dstwrap(sc, idx);
1737
1738         sc->sc_dsti = idx;
1739         sc->sc_dstu += used;
1740         return (idx);
1741 }
1742
1743 static __inline int
1744 hifn_dmamap_srcwrap(struct hifn_softc *sc, int idx)
1745 {
1746         struct hifn_dma *dma = sc->sc_dma;
1747
1748         if (++idx == HIFN_D_SRC_RSIZE) {
1749                 dma->srcr[idx].l = htole32(HIFN_D_VALID |
1750                     HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
1751                 HIFN_SRCR_SYNC(sc, HIFN_D_SRC_RSIZE,
1752                     BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1753                 idx = 0;
1754         }
1755         return (idx);
1756 }
1757
1758 static int
1759 hifn_dmamap_load_src(struct hifn_softc *sc, struct hifn_command *cmd)
1760 {
1761         struct hifn_dma *dma = sc->sc_dma;
1762         struct hifn_operand *src = &cmd->src;
1763         int idx, i;
1764         u_int32_t last = 0;
1765
1766         idx = sc->sc_srci;
1767         for (i = 0; i < src->nsegs; i++) {
1768                 if (i == src->nsegs - 1)
1769                         last = HIFN_D_LAST;
1770
1771                 dma->srcr[idx].p = htole32(src->segs[i].ds_addr);
1772                 dma->srcr[idx].l = htole32(src->segs[i].ds_len |
1773                     HIFN_D_VALID | HIFN_D_MASKDONEIRQ | last);
1774                 HIFN_SRCR_SYNC(sc, idx,
1775                     BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
1776
1777                 idx = hifn_dmamap_srcwrap(sc, idx);
1778         }
1779         sc->sc_srci = idx;
1780         sc->sc_srcu += src->nsegs;
1781         return (idx);
1782 }
1783
1784 static void
1785 hifn_op_cb(void* arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error)
1786 {
1787         struct hifn_operand *op = arg;
1788
1789         KASSERT(nsegs <= MAX_SCATTER,
1790                 ("hifn_op_cb: too many DMA segments (%u > %u) "
1791                  "returned when mapping operand", nsegs, MAX_SCATTER));
1792         op->mapsize = mapsize;
1793         op->nsegs = nsegs;
1794         bcopy(seg, op->segs, nsegs * sizeof (seg[0]));
1795 }
1796
1797 static int
1798 hifn_crypto(
1799         struct hifn_softc *sc,
1800         struct hifn_command *cmd,
1801         struct cryptop *crp,
1802         int hint)
1803 {
1804         struct  hifn_dma *dma = sc->sc_dma;
1805         u_int32_t cmdlen, csr;
1806         int cmdi, resi, err = 0;
1807
1808         /*
1809          * need 1 cmd, and 1 res
1810          *
1811          * NB: check this first since it's easy.
1812          */
1813         HIFN_LOCK(sc);
1814         if ((sc->sc_cmdu + 1) > HIFN_D_CMD_RSIZE ||
1815             (sc->sc_resu + 1) > HIFN_D_RES_RSIZE) {
1816 #ifdef HIFN_DEBUG
1817                 if (hifn_debug) {
1818                         device_printf(sc->sc_dev,
1819                                 "cmd/result exhaustion, cmdu %u resu %u\n",
1820                                 sc->sc_cmdu, sc->sc_resu);
1821                 }
1822 #endif
1823                 hifnstats.hst_nomem_cr++;
1824                 HIFN_UNLOCK(sc);
1825                 return (ERESTART);
1826         }
1827
1828         if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &cmd->src_map)) {
1829                 hifnstats.hst_nomem_map++;
1830                 HIFN_UNLOCK(sc);
1831                 return (ENOMEM);
1832         }
1833
1834         if (crp->crp_flags & CRYPTO_F_IMBUF) {
1835                 if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->src_map,
1836                     cmd->src_m, hifn_op_cb, &cmd->src, BUS_DMA_NOWAIT)) {
1837                         hifnstats.hst_nomem_load++;
1838                         err = ENOMEM;
1839                         goto err_srcmap1;
1840                 }
1841         } else if (crp->crp_flags & CRYPTO_F_IOV) {
1842                 if (bus_dmamap_load_uio(sc->sc_dmat, cmd->src_map,
1843                     cmd->src_io, hifn_op_cb, &cmd->src, BUS_DMA_NOWAIT)) {
1844                         hifnstats.hst_nomem_load++;
1845                         err = ENOMEM;
1846                         goto err_srcmap1;
1847                 }
1848         } else {
1849                 err = EINVAL;
1850                 goto err_srcmap1;
1851         }
1852
1853         if (hifn_dmamap_aligned(&cmd->src)) {
1854                 cmd->sloplen = cmd->src_mapsize & 3;
1855                 cmd->dst = cmd->src;
1856         } else {
1857                 if (crp->crp_flags & CRYPTO_F_IOV) {
1858                         err = EINVAL;
1859                         goto err_srcmap;
1860                 } else if (crp->crp_flags & CRYPTO_F_IMBUF) {
1861                         int totlen, len;
1862                         struct mbuf *m, *m0, *mlast;
1863
1864                         KASSERT(cmd->dst_m == cmd->src_m,
1865                                 ("hifn_crypto: dst_m initialized improperly"));
1866                         hifnstats.hst_unaligned++;
1867                         /*
1868                          * Source is not aligned on a longword boundary.
1869                          * Copy the data to insure alignment.  If we fail
1870                          * to allocate mbufs or clusters while doing this
1871                          * we return ERESTART so the operation is requeued
1872                          * at the crypto later, but only if there are
1873                          * ops already posted to the hardware; otherwise we
1874                          * have no guarantee that we'll be re-entered.
1875                          */
1876                         totlen = cmd->src_mapsize;
1877                         if (cmd->src_m->m_flags & M_PKTHDR) {
1878                                 len = MHLEN;
1879                                 MGETHDR(m0, M_NOWAIT, MT_DATA);
1880                                 if (m0 && !m_dup_pkthdr(m0, cmd->src_m, M_NOWAIT)) {
1881                                         m_free(m0);
1882                                         m0 = NULL;
1883                                 }
1884                         } else {
1885                                 len = MLEN;
1886                                 MGET(m0, M_NOWAIT, MT_DATA);
1887                         }
1888                         if (m0 == NULL) {
1889                                 hifnstats.hst_nomem_mbuf++;
1890                                 err = sc->sc_cmdu ? ERESTART : ENOMEM;
1891                                 goto err_srcmap;
1892                         }
1893                         if (totlen >= MINCLSIZE) {
1894                                 if (!(MCLGET(m0, M_NOWAIT))) {
1895                                         hifnstats.hst_nomem_mcl++;
1896                                         err = sc->sc_cmdu ? ERESTART : ENOMEM;
1897                                         m_freem(m0);
1898                                         goto err_srcmap;
1899                                 }
1900                                 len = MCLBYTES;
1901                         }
1902                         totlen -= len;
1903                         m0->m_pkthdr.len = m0->m_len = len;
1904                         mlast = m0;
1905
1906                         while (totlen > 0) {
1907                                 MGET(m, M_NOWAIT, MT_DATA);
1908                                 if (m == NULL) {
1909                                         hifnstats.hst_nomem_mbuf++;
1910                                         err = sc->sc_cmdu ? ERESTART : ENOMEM;
1911                                         m_freem(m0);
1912                                         goto err_srcmap;
1913                                 }
1914                                 len = MLEN;
1915                                 if (totlen >= MINCLSIZE) {
1916                                         if (!(MCLGET(m, M_NOWAIT))) {
1917                                                 hifnstats.hst_nomem_mcl++;
1918                                                 err = sc->sc_cmdu ? ERESTART : ENOMEM;
1919                                                 mlast->m_next = m;
1920                                                 m_freem(m0);
1921                                                 goto err_srcmap;
1922                                         }
1923                                         len = MCLBYTES;
1924                                 }
1925
1926                                 m->m_len = len;
1927                                 m0->m_pkthdr.len += len;
1928                                 totlen -= len;
1929
1930                                 mlast->m_next = m;
1931                                 mlast = m;
1932                         }
1933                         cmd->dst_m = m0;
1934                 }
1935         }
1936
1937         if (cmd->dst_map == NULL) {
1938                 if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &cmd->dst_map)) {
1939                         hifnstats.hst_nomem_map++;
1940                         err = ENOMEM;
1941                         goto err_srcmap;
1942                 }
1943                 if (crp->crp_flags & CRYPTO_F_IMBUF) {
1944                         if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->dst_map,
1945                             cmd->dst_m, hifn_op_cb, &cmd->dst, BUS_DMA_NOWAIT)) {
1946                                 hifnstats.hst_nomem_map++;
1947                                 err = ENOMEM;
1948                                 goto err_dstmap1;
1949                         }
1950                 } else if (crp->crp_flags & CRYPTO_F_IOV) {
1951                         if (bus_dmamap_load_uio(sc->sc_dmat, cmd->dst_map,
1952                             cmd->dst_io, hifn_op_cb, &cmd->dst, BUS_DMA_NOWAIT)) {
1953                                 hifnstats.hst_nomem_load++;
1954                                 err = ENOMEM;
1955                                 goto err_dstmap1;
1956                         }
1957                 }
1958         }
1959
1960 #ifdef HIFN_DEBUG
1961         if (hifn_debug) {
1962                 device_printf(sc->sc_dev,
1963                     "Entering cmd: stat %8x ien %8x u %d/%d/%d/%d n %d/%d\n",
1964                     READ_REG_1(sc, HIFN_1_DMA_CSR),
1965                     READ_REG_1(sc, HIFN_1_DMA_IER),
1966                     sc->sc_cmdu, sc->sc_srcu, sc->sc_dstu, sc->sc_resu,
1967                     cmd->src_nsegs, cmd->dst_nsegs);
1968         }
1969 #endif
1970
1971         if (cmd->src_map == cmd->dst_map) {
1972                 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
1973                     BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
1974         } else {
1975                 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
1976                     BUS_DMASYNC_PREWRITE);
1977                 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
1978                     BUS_DMASYNC_PREREAD);
1979         }
1980
1981         /*
1982          * need N src, and N dst
1983          */
1984         if ((sc->sc_srcu + cmd->src_nsegs) > HIFN_D_SRC_RSIZE ||
1985             (sc->sc_dstu + cmd->dst_nsegs + 1) > HIFN_D_DST_RSIZE) {
1986 #ifdef HIFN_DEBUG
1987                 if (hifn_debug) {
1988                         device_printf(sc->sc_dev,
1989                                 "src/dst exhaustion, srcu %u+%u dstu %u+%u\n",
1990                                 sc->sc_srcu, cmd->src_nsegs,
1991                                 sc->sc_dstu, cmd->dst_nsegs);
1992                 }
1993 #endif
1994                 hifnstats.hst_nomem_sd++;
1995                 err = ERESTART;
1996                 goto err_dstmap;
1997         }
1998
1999         if (sc->sc_cmdi == HIFN_D_CMD_RSIZE) {
2000                 sc->sc_cmdi = 0;
2001                 dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID |
2002                     HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
2003                 HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE,
2004                     BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2005         }
2006         cmdi = sc->sc_cmdi++;
2007         cmdlen = hifn_write_command(cmd, dma->command_bufs[cmdi]);
2008         HIFN_CMD_SYNC(sc, cmdi, BUS_DMASYNC_PREWRITE);
2009
2010         /* .p for command/result already set */
2011         dma->cmdr[cmdi].l = htole32(cmdlen | HIFN_D_VALID | HIFN_D_LAST |
2012             HIFN_D_MASKDONEIRQ);
2013         HIFN_CMDR_SYNC(sc, cmdi,
2014             BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
2015         sc->sc_cmdu++;
2016
2017         /*
2018          * We don't worry about missing an interrupt (which a "command wait"
2019          * interrupt salvages us from), unless there is more than one command
2020          * in the queue.
2021          */
2022         if (sc->sc_cmdu > 1) {
2023                 sc->sc_dmaier |= HIFN_DMAIER_C_WAIT;
2024                 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
2025         }
2026
2027         hifnstats.hst_ipackets++;
2028         hifnstats.hst_ibytes += cmd->src_mapsize;
2029
2030         hifn_dmamap_load_src(sc, cmd);
2031
2032         /*
2033          * Unlike other descriptors, we don't mask done interrupt from
2034          * result descriptor.
2035          */
2036 #ifdef HIFN_DEBUG
2037         if (hifn_debug)
2038                 printf("load res\n");
2039 #endif
2040         if (sc->sc_resi == HIFN_D_RES_RSIZE) {
2041                 sc->sc_resi = 0;
2042                 dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID |
2043                     HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
2044                 HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE,
2045                     BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2046         }
2047         resi = sc->sc_resi++;
2048         KASSERT(sc->sc_hifn_commands[resi] == NULL,
2049                 ("hifn_crypto: command slot %u busy", resi));
2050         sc->sc_hifn_commands[resi] = cmd;
2051         HIFN_RES_SYNC(sc, resi, BUS_DMASYNC_PREREAD);
2052         if ((hint & CRYPTO_HINT_MORE) && sc->sc_curbatch < hifn_maxbatch) {
2053                 dma->resr[resi].l = htole32(HIFN_MAX_RESULT |
2054                     HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ);
2055                 sc->sc_curbatch++;
2056                 if (sc->sc_curbatch > hifnstats.hst_maxbatch)
2057                         hifnstats.hst_maxbatch = sc->sc_curbatch;
2058                 hifnstats.hst_totbatch++;
2059         } else {
2060                 dma->resr[resi].l = htole32(HIFN_MAX_RESULT |
2061                     HIFN_D_VALID | HIFN_D_LAST);
2062                 sc->sc_curbatch = 0;
2063         }
2064         HIFN_RESR_SYNC(sc, resi,
2065             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2066         sc->sc_resu++;
2067
2068         if (cmd->sloplen)
2069                 cmd->slopidx = resi;
2070
2071         hifn_dmamap_load_dst(sc, cmd);
2072
2073         csr = 0;
2074         if (sc->sc_c_busy == 0) {
2075                 csr |= HIFN_DMACSR_C_CTRL_ENA;
2076                 sc->sc_c_busy = 1;
2077         }
2078         if (sc->sc_s_busy == 0) {
2079                 csr |= HIFN_DMACSR_S_CTRL_ENA;
2080                 sc->sc_s_busy = 1;
2081         }
2082         if (sc->sc_r_busy == 0) {
2083                 csr |= HIFN_DMACSR_R_CTRL_ENA;
2084                 sc->sc_r_busy = 1;
2085         }
2086         if (sc->sc_d_busy == 0) {
2087                 csr |= HIFN_DMACSR_D_CTRL_ENA;
2088                 sc->sc_d_busy = 1;
2089         }
2090         if (csr)
2091                 WRITE_REG_1(sc, HIFN_1_DMA_CSR, csr);
2092
2093 #ifdef HIFN_DEBUG
2094         if (hifn_debug) {
2095                 device_printf(sc->sc_dev, "command: stat %8x ier %8x\n",
2096                     READ_REG_1(sc, HIFN_1_DMA_CSR),
2097                     READ_REG_1(sc, HIFN_1_DMA_IER));
2098         }
2099 #endif
2100
2101         sc->sc_active = 5;
2102         HIFN_UNLOCK(sc);
2103         KASSERT(err == 0, ("hifn_crypto: success with error %u", err));
2104         return (err);           /* success */
2105
2106 err_dstmap:
2107         if (cmd->src_map != cmd->dst_map)
2108                 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
2109 err_dstmap1:
2110         if (cmd->src_map != cmd->dst_map)
2111                 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
2112 err_srcmap:
2113         if (crp->crp_flags & CRYPTO_F_IMBUF) {
2114                 if (cmd->src_m != cmd->dst_m)
2115                         m_freem(cmd->dst_m);
2116         }
2117         bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
2118 err_srcmap1:
2119         bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
2120         HIFN_UNLOCK(sc);
2121         return (err);
2122 }
2123
2124 static void
2125 hifn_tick(void* vsc)
2126 {
2127         struct hifn_softc *sc = vsc;
2128
2129         HIFN_LOCK(sc);
2130         if (sc->sc_active == 0) {
2131                 u_int32_t r = 0;
2132
2133                 if (sc->sc_cmdu == 0 && sc->sc_c_busy) {
2134                         sc->sc_c_busy = 0;
2135                         r |= HIFN_DMACSR_C_CTRL_DIS;
2136                 }
2137                 if (sc->sc_srcu == 0 && sc->sc_s_busy) {
2138                         sc->sc_s_busy = 0;
2139                         r |= HIFN_DMACSR_S_CTRL_DIS;
2140                 }
2141                 if (sc->sc_dstu == 0 && sc->sc_d_busy) {
2142                         sc->sc_d_busy = 0;
2143                         r |= HIFN_DMACSR_D_CTRL_DIS;
2144                 }
2145                 if (sc->sc_resu == 0 && sc->sc_r_busy) {
2146                         sc->sc_r_busy = 0;
2147                         r |= HIFN_DMACSR_R_CTRL_DIS;
2148                 }
2149                 if (r)
2150                         WRITE_REG_1(sc, HIFN_1_DMA_CSR, r);
2151         } else
2152                 sc->sc_active--;
2153         HIFN_UNLOCK(sc);
2154         callout_reset(&sc->sc_tickto, hz, hifn_tick, sc);
2155 }
2156
2157 static void
2158 hifn_intr(void *arg)
2159 {
2160         struct hifn_softc *sc = arg;
2161         struct hifn_dma *dma;
2162         u_int32_t dmacsr, restart;
2163         int i, u;
2164
2165         dmacsr = READ_REG_1(sc, HIFN_1_DMA_CSR);
2166
2167         /* Nothing in the DMA unit interrupted */
2168         if ((dmacsr & sc->sc_dmaier) == 0)
2169                 return;
2170
2171         HIFN_LOCK(sc);
2172
2173         dma = sc->sc_dma;
2174
2175 #ifdef HIFN_DEBUG
2176         if (hifn_debug) {
2177                 device_printf(sc->sc_dev,
2178                     "irq: stat %08x ien %08x damier %08x i %d/%d/%d/%d k %d/%d/%d/%d u %d/%d/%d/%d\n",
2179                     dmacsr, READ_REG_1(sc, HIFN_1_DMA_IER), sc->sc_dmaier,
2180                     sc->sc_cmdi, sc->sc_srci, sc->sc_dsti, sc->sc_resi,
2181                     sc->sc_cmdk, sc->sc_srck, sc->sc_dstk, sc->sc_resk,
2182                     sc->sc_cmdu, sc->sc_srcu, sc->sc_dstu, sc->sc_resu);
2183         }
2184 #endif
2185
2186         WRITE_REG_1(sc, HIFN_1_DMA_CSR, dmacsr & sc->sc_dmaier);
2187
2188         if ((sc->sc_flags & HIFN_HAS_PUBLIC) &&
2189             (dmacsr & HIFN_DMACSR_PUBDONE))
2190                 WRITE_REG_1(sc, HIFN_1_PUB_STATUS,
2191                     READ_REG_1(sc, HIFN_1_PUB_STATUS) | HIFN_PUBSTS_DONE);
2192
2193         restart = dmacsr & (HIFN_DMACSR_D_OVER | HIFN_DMACSR_R_OVER);
2194         if (restart)
2195                 device_printf(sc->sc_dev, "overrun %x\n", dmacsr);
2196
2197         if (sc->sc_flags & HIFN_IS_7811) {
2198                 if (dmacsr & HIFN_DMACSR_ILLR)
2199                         device_printf(sc->sc_dev, "illegal read\n");
2200                 if (dmacsr & HIFN_DMACSR_ILLW)
2201                         device_printf(sc->sc_dev, "illegal write\n");
2202         }
2203
2204         restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT |
2205             HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT);
2206         if (restart) {
2207                 device_printf(sc->sc_dev, "abort, resetting.\n");
2208                 hifnstats.hst_abort++;
2209                 hifn_abort(sc);
2210                 HIFN_UNLOCK(sc);
2211                 return;
2212         }
2213
2214         if ((dmacsr & HIFN_DMACSR_C_WAIT) && (sc->sc_cmdu == 0)) {
2215                 /*
2216                  * If no slots to process and we receive a "waiting on
2217                  * command" interrupt, we disable the "waiting on command"
2218                  * (by clearing it).
2219                  */
2220                 sc->sc_dmaier &= ~HIFN_DMAIER_C_WAIT;
2221                 WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
2222         }
2223
2224         /* clear the rings */
2225         i = sc->sc_resk; u = sc->sc_resu;
2226         while (u != 0) {
2227                 HIFN_RESR_SYNC(sc, i,
2228                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2229                 if (dma->resr[i].l & htole32(HIFN_D_VALID)) {
2230                         HIFN_RESR_SYNC(sc, i,
2231                             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2232                         break;
2233                 }
2234
2235                 if (i != HIFN_D_RES_RSIZE) {
2236                         struct hifn_command *cmd;
2237                         u_int8_t *macbuf = NULL;
2238
2239                         HIFN_RES_SYNC(sc, i, BUS_DMASYNC_POSTREAD);
2240                         cmd = sc->sc_hifn_commands[i];
2241                         KASSERT(cmd != NULL,
2242                                 ("hifn_intr: null command slot %u", i));
2243                         sc->sc_hifn_commands[i] = NULL;
2244
2245                         if (cmd->base_masks & HIFN_BASE_CMD_MAC) {
2246                                 macbuf = dma->result_bufs[i];
2247                                 macbuf += 12;
2248                         }
2249
2250                         hifn_callback(sc, cmd, macbuf);
2251                         hifnstats.hst_opackets++;
2252                         u--;
2253                 }
2254
2255                 if (++i == (HIFN_D_RES_RSIZE + 1))
2256                         i = 0;
2257         }
2258         sc->sc_resk = i; sc->sc_resu = u;
2259
2260         i = sc->sc_srck; u = sc->sc_srcu;
2261         while (u != 0) {
2262                 if (i == HIFN_D_SRC_RSIZE)
2263                         i = 0;
2264                 HIFN_SRCR_SYNC(sc, i,
2265                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2266                 if (dma->srcr[i].l & htole32(HIFN_D_VALID)) {
2267                         HIFN_SRCR_SYNC(sc, i,
2268                             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2269                         break;
2270                 }
2271                 i++, u--;
2272         }
2273         sc->sc_srck = i; sc->sc_srcu = u;
2274
2275         i = sc->sc_cmdk; u = sc->sc_cmdu;
2276         while (u != 0) {
2277                 HIFN_CMDR_SYNC(sc, i,
2278                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2279                 if (dma->cmdr[i].l & htole32(HIFN_D_VALID)) {
2280                         HIFN_CMDR_SYNC(sc, i,
2281                             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2282                         break;
2283                 }
2284                 if (i != HIFN_D_CMD_RSIZE) {
2285                         u--;
2286                         HIFN_CMD_SYNC(sc, i, BUS_DMASYNC_POSTWRITE);
2287                 }
2288                 if (++i == (HIFN_D_CMD_RSIZE + 1))
2289                         i = 0;
2290         }
2291         sc->sc_cmdk = i; sc->sc_cmdu = u;
2292
2293         HIFN_UNLOCK(sc);
2294
2295         if (sc->sc_needwakeup) {                /* XXX check high watermark */
2296                 int wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ);
2297 #ifdef HIFN_DEBUG
2298                 if (hifn_debug)
2299                         device_printf(sc->sc_dev,
2300                                 "wakeup crypto (%x) u %d/%d/%d/%d\n",
2301                                 sc->sc_needwakeup,
2302                                 sc->sc_cmdu, sc->sc_srcu, sc->sc_dstu, sc->sc_resu);
2303 #endif
2304                 sc->sc_needwakeup &= ~wakeup;
2305                 crypto_unblock(sc->sc_cid, wakeup);
2306         }
2307 }
2308
2309 /*
2310  * Allocate a new 'session' and return an encoded session id.  'sidp'
2311  * contains our registration id, and should contain an encoded session
2312  * id on successful allocation.
2313  */
2314 static int
2315 hifn_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri)
2316 {
2317         struct hifn_softc *sc = device_get_softc(dev);
2318         struct cryptoini *c;
2319         int mac = 0, cry = 0, sesn;
2320         struct hifn_session *ses = NULL;
2321
2322         KASSERT(sc != NULL, ("hifn_newsession: null softc"));
2323         if (sidp == NULL || cri == NULL || sc == NULL)
2324                 return (EINVAL);
2325
2326         HIFN_LOCK(sc);
2327         if (sc->sc_sessions == NULL) {
2328                 ses = sc->sc_sessions = (struct hifn_session *)malloc(
2329                     sizeof(*ses), M_DEVBUF, M_NOWAIT);
2330                 if (ses == NULL) {
2331                         HIFN_UNLOCK(sc);
2332                         return (ENOMEM);
2333                 }
2334                 sesn = 0;
2335                 sc->sc_nsessions = 1;
2336         } else {
2337                 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
2338                         if (!sc->sc_sessions[sesn].hs_used) {
2339                                 ses = &sc->sc_sessions[sesn];
2340                                 break;
2341                         }
2342                 }
2343
2344                 if (ses == NULL) {
2345                         sesn = sc->sc_nsessions;
2346                         ses = (struct hifn_session *)malloc((sesn + 1) *
2347                             sizeof(*ses), M_DEVBUF, M_NOWAIT);
2348                         if (ses == NULL) {
2349                                 HIFN_UNLOCK(sc);
2350                                 return (ENOMEM);
2351                         }
2352                         bcopy(sc->sc_sessions, ses, sesn * sizeof(*ses));
2353                         bzero(sc->sc_sessions, sesn * sizeof(*ses));
2354                         free(sc->sc_sessions, M_DEVBUF);
2355                         sc->sc_sessions = ses;
2356                         ses = &sc->sc_sessions[sesn];
2357                         sc->sc_nsessions++;
2358                 }
2359         }
2360         HIFN_UNLOCK(sc);
2361
2362         bzero(ses, sizeof(*ses));
2363         ses->hs_used = 1;
2364
2365         for (c = cri; c != NULL; c = c->cri_next) {
2366                 switch (c->cri_alg) {
2367                 case CRYPTO_MD5:
2368                 case CRYPTO_SHA1:
2369                 case CRYPTO_MD5_HMAC:
2370                 case CRYPTO_SHA1_HMAC:
2371                         if (mac)
2372                                 return (EINVAL);
2373                         mac = 1;
2374                         ses->hs_mlen = c->cri_mlen;
2375                         if (ses->hs_mlen == 0) {
2376                                 switch (c->cri_alg) {
2377                                 case CRYPTO_MD5:
2378                                 case CRYPTO_MD5_HMAC:
2379                                         ses->hs_mlen = 16;
2380                                         break;
2381                                 case CRYPTO_SHA1:
2382                                 case CRYPTO_SHA1_HMAC:
2383                                         ses->hs_mlen = 20;
2384                                         break;
2385                                 }
2386                         }
2387                         break;
2388                 case CRYPTO_DES_CBC:
2389                 case CRYPTO_3DES_CBC:
2390                 case CRYPTO_AES_CBC:
2391                         /* XXX this may read fewer, does it matter? */
2392                         read_random(ses->hs_iv,
2393                                 c->cri_alg == CRYPTO_AES_CBC ?
2394                                         HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
2395                         /*FALLTHROUGH*/
2396                 case CRYPTO_ARC4:
2397                         if (cry)
2398                                 return (EINVAL);
2399                         cry = 1;
2400                         break;
2401                 default:
2402                         return (EINVAL);
2403                 }
2404         }
2405         if (mac == 0 && cry == 0)
2406                 return (EINVAL);
2407
2408         *sidp = HIFN_SID(device_get_unit(sc->sc_dev), sesn);
2409
2410         return (0);
2411 }
2412
2413 /*
2414  * Deallocate a session.
2415  * XXX this routine should run a zero'd mac/encrypt key into context ram.
2416  * XXX to blow away any keys already stored there.
2417  */
2418 static int
2419 hifn_freesession(device_t dev, u_int64_t tid)
2420 {
2421         struct hifn_softc *sc = device_get_softc(dev);
2422         int session, error;
2423         u_int32_t sid = CRYPTO_SESID2LID(tid);
2424
2425         KASSERT(sc != NULL, ("hifn_freesession: null softc"));
2426         if (sc == NULL)
2427                 return (EINVAL);
2428
2429         HIFN_LOCK(sc);
2430         session = HIFN_SESSION(sid);
2431         if (session < sc->sc_nsessions) {
2432                 bzero(&sc->sc_sessions[session], sizeof(struct hifn_session));
2433                 error = 0;
2434         } else
2435                 error = EINVAL;
2436         HIFN_UNLOCK(sc);
2437
2438         return (error);
2439 }
2440
2441 static int
2442 hifn_process(device_t dev, struct cryptop *crp, int hint)
2443 {
2444         struct hifn_softc *sc = device_get_softc(dev);
2445         struct hifn_command *cmd = NULL;
2446         int session, err, ivlen;
2447         struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
2448
2449         if (crp == NULL || crp->crp_callback == NULL) {
2450                 hifnstats.hst_invalid++;
2451                 return (EINVAL);
2452         }
2453         session = HIFN_SESSION(crp->crp_sid);
2454
2455         if (sc == NULL || session >= sc->sc_nsessions) {
2456                 err = EINVAL;
2457                 goto errout;
2458         }
2459
2460         cmd = malloc(sizeof(struct hifn_command), M_DEVBUF, M_NOWAIT | M_ZERO);
2461         if (cmd == NULL) {
2462                 hifnstats.hst_nomem++;
2463                 err = ENOMEM;
2464                 goto errout;
2465         }
2466
2467         if (crp->crp_flags & CRYPTO_F_IMBUF) {
2468                 cmd->src_m = (struct mbuf *)crp->crp_buf;
2469                 cmd->dst_m = (struct mbuf *)crp->crp_buf;
2470         } else if (crp->crp_flags & CRYPTO_F_IOV) {
2471                 cmd->src_io = (struct uio *)crp->crp_buf;
2472                 cmd->dst_io = (struct uio *)crp->crp_buf;
2473         } else {
2474                 err = EINVAL;
2475                 goto errout;    /* XXX we don't handle contiguous buffers! */
2476         }
2477
2478         crd1 = crp->crp_desc;
2479         if (crd1 == NULL) {
2480                 err = EINVAL;
2481                 goto errout;
2482         }
2483         crd2 = crd1->crd_next;
2484
2485         if (crd2 == NULL) {
2486                 if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
2487                     crd1->crd_alg == CRYPTO_SHA1_HMAC ||
2488                     crd1->crd_alg == CRYPTO_SHA1 ||
2489                     crd1->crd_alg == CRYPTO_MD5) {
2490                         maccrd = crd1;
2491                         enccrd = NULL;
2492                 } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
2493                     crd1->crd_alg == CRYPTO_3DES_CBC ||
2494                     crd1->crd_alg == CRYPTO_AES_CBC ||
2495                     crd1->crd_alg == CRYPTO_ARC4) {
2496                         if ((crd1->crd_flags & CRD_F_ENCRYPT) == 0)
2497                                 cmd->base_masks |= HIFN_BASE_CMD_DECODE;
2498                         maccrd = NULL;
2499                         enccrd = crd1;
2500                 } else {
2501                         err = EINVAL;
2502                         goto errout;
2503                 }
2504         } else {
2505                 if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
2506                      crd1->crd_alg == CRYPTO_SHA1_HMAC ||
2507                      crd1->crd_alg == CRYPTO_MD5 ||
2508                      crd1->crd_alg == CRYPTO_SHA1) &&
2509                     (crd2->crd_alg == CRYPTO_DES_CBC ||
2510                      crd2->crd_alg == CRYPTO_3DES_CBC ||
2511                      crd2->crd_alg == CRYPTO_AES_CBC ||
2512                      crd2->crd_alg == CRYPTO_ARC4) &&
2513                     ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
2514                         cmd->base_masks = HIFN_BASE_CMD_DECODE;
2515                         maccrd = crd1;
2516                         enccrd = crd2;
2517                 } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
2518                      crd1->crd_alg == CRYPTO_ARC4 ||
2519                      crd1->crd_alg == CRYPTO_3DES_CBC ||
2520                      crd1->crd_alg == CRYPTO_AES_CBC) &&
2521                     (crd2->crd_alg == CRYPTO_MD5_HMAC ||
2522                      crd2->crd_alg == CRYPTO_SHA1_HMAC ||
2523                      crd2->crd_alg == CRYPTO_MD5 ||
2524                      crd2->crd_alg == CRYPTO_SHA1) &&
2525                     (crd1->crd_flags & CRD_F_ENCRYPT)) {
2526                         enccrd = crd1;
2527                         maccrd = crd2;
2528                 } else {
2529                         /*
2530                          * We cannot order the 7751 as requested
2531                          */
2532                         err = EINVAL;
2533                         goto errout;
2534                 }
2535         }
2536
2537         if (enccrd) {
2538                 cmd->enccrd = enccrd;
2539                 cmd->base_masks |= HIFN_BASE_CMD_CRYPT;
2540                 switch (enccrd->crd_alg) {
2541                 case CRYPTO_ARC4:
2542                         cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_RC4;
2543                         break;
2544                 case CRYPTO_DES_CBC:
2545                         cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_DES |
2546                             HIFN_CRYPT_CMD_MODE_CBC |
2547                             HIFN_CRYPT_CMD_NEW_IV;
2548                         break;
2549                 case CRYPTO_3DES_CBC:
2550                         cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_3DES |
2551                             HIFN_CRYPT_CMD_MODE_CBC |
2552                             HIFN_CRYPT_CMD_NEW_IV;
2553                         break;
2554                 case CRYPTO_AES_CBC:
2555                         cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_AES |
2556                             HIFN_CRYPT_CMD_MODE_CBC |
2557                             HIFN_CRYPT_CMD_NEW_IV;
2558                         break;
2559                 default:
2560                         err = EINVAL;
2561                         goto errout;
2562                 }
2563                 if (enccrd->crd_alg != CRYPTO_ARC4) {
2564                         ivlen = ((enccrd->crd_alg == CRYPTO_AES_CBC) ?
2565                                 HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
2566                         if (enccrd->crd_flags & CRD_F_ENCRYPT) {
2567                                 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
2568                                         bcopy(enccrd->crd_iv, cmd->iv, ivlen);
2569                                 else
2570                                         bcopy(sc->sc_sessions[session].hs_iv,
2571                                             cmd->iv, ivlen);
2572
2573                                 if ((enccrd->crd_flags & CRD_F_IV_PRESENT)
2574                                     == 0) {
2575                                         crypto_copyback(crp->crp_flags,
2576                                             crp->crp_buf, enccrd->crd_inject,
2577                                             ivlen, cmd->iv);
2578                                 }
2579                         } else {
2580                                 if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
2581                                         bcopy(enccrd->crd_iv, cmd->iv, ivlen);
2582                                 else {
2583                                         crypto_copydata(crp->crp_flags,
2584                                             crp->crp_buf, enccrd->crd_inject,
2585                                             ivlen, cmd->iv);
2586                                 }
2587                         }
2588                 }
2589
2590                 if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT)
2591                         cmd->cry_masks |= HIFN_CRYPT_CMD_NEW_KEY;
2592                 cmd->ck = enccrd->crd_key;
2593                 cmd->cklen = enccrd->crd_klen >> 3;
2594                 cmd->cry_masks |= HIFN_CRYPT_CMD_NEW_KEY;
2595
2596                 /*
2597                  * Need to specify the size for the AES key in the masks.
2598                  */
2599                 if ((cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) ==
2600                     HIFN_CRYPT_CMD_ALG_AES) {
2601                         switch (cmd->cklen) {
2602                         case 16:
2603                                 cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_128;
2604                                 break;
2605                         case 24:
2606                                 cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_192;
2607                                 break;
2608                         case 32:
2609                                 cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_256;
2610                                 break;
2611                         default:
2612                                 err = EINVAL;
2613                                 goto errout;
2614                         }
2615                 }
2616         }
2617
2618         if (maccrd) {
2619                 cmd->maccrd = maccrd;
2620                 cmd->base_masks |= HIFN_BASE_CMD_MAC;
2621
2622                 switch (maccrd->crd_alg) {
2623                 case CRYPTO_MD5:
2624                         cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 |
2625                             HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH |
2626                             HIFN_MAC_CMD_POS_IPSEC;
2627                        break;
2628                 case CRYPTO_MD5_HMAC:
2629                         cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 |
2630                             HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HMAC |
2631                             HIFN_MAC_CMD_POS_IPSEC | HIFN_MAC_CMD_TRUNC;
2632                         break;
2633                 case CRYPTO_SHA1:
2634                         cmd->mac_masks |= HIFN_MAC_CMD_ALG_SHA1 |
2635                             HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH |
2636                             HIFN_MAC_CMD_POS_IPSEC;
2637                         break;
2638                 case CRYPTO_SHA1_HMAC:
2639                         cmd->mac_masks |= HIFN_MAC_CMD_ALG_SHA1 |
2640                             HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HMAC |
2641                             HIFN_MAC_CMD_POS_IPSEC | HIFN_MAC_CMD_TRUNC;
2642                         break;
2643                 }
2644
2645                 if (maccrd->crd_alg == CRYPTO_SHA1_HMAC ||
2646                      maccrd->crd_alg == CRYPTO_MD5_HMAC) {
2647                         cmd->mac_masks |= HIFN_MAC_CMD_NEW_KEY;
2648                         bcopy(maccrd->crd_key, cmd->mac, maccrd->crd_klen >> 3);
2649                         bzero(cmd->mac + (maccrd->crd_klen >> 3),
2650                             HIFN_MAC_KEY_LENGTH - (maccrd->crd_klen >> 3));
2651                 }
2652         }
2653
2654         cmd->crp = crp;
2655         cmd->session_num = session;
2656         cmd->softc = sc;
2657
2658         err = hifn_crypto(sc, cmd, crp, hint);
2659         if (!err) {
2660                 return 0;
2661         } else if (err == ERESTART) {
2662                 /*
2663                  * There weren't enough resources to dispatch the request
2664                  * to the part.  Notify the caller so they'll requeue this
2665                  * request and resubmit it again soon.
2666                  */
2667 #ifdef HIFN_DEBUG
2668                 if (hifn_debug)
2669                         device_printf(sc->sc_dev, "requeue request\n");
2670 #endif
2671                 free(cmd, M_DEVBUF);
2672                 sc->sc_needwakeup |= CRYPTO_SYMQ;
2673                 return (err);
2674         }
2675
2676 errout:
2677         if (cmd != NULL)
2678                 free(cmd, M_DEVBUF);
2679         if (err == EINVAL)
2680                 hifnstats.hst_invalid++;
2681         else
2682                 hifnstats.hst_nomem++;
2683         crp->crp_etype = err;
2684         crypto_done(crp);
2685         return (err);
2686 }
2687
2688 static void
2689 hifn_abort(struct hifn_softc *sc)
2690 {
2691         struct hifn_dma *dma = sc->sc_dma;
2692         struct hifn_command *cmd;
2693         struct cryptop *crp;
2694         int i, u;
2695
2696         i = sc->sc_resk; u = sc->sc_resu;
2697         while (u != 0) {
2698                 cmd = sc->sc_hifn_commands[i];
2699                 KASSERT(cmd != NULL, ("hifn_abort: null command slot %u", i));
2700                 sc->sc_hifn_commands[i] = NULL;
2701                 crp = cmd->crp;
2702
2703                 if ((dma->resr[i].l & htole32(HIFN_D_VALID)) == 0) {
2704                         /* Salvage what we can. */
2705                         u_int8_t *macbuf;
2706
2707                         if (cmd->base_masks & HIFN_BASE_CMD_MAC) {
2708                                 macbuf = dma->result_bufs[i];
2709                                 macbuf += 12;
2710                         } else
2711                                 macbuf = NULL;
2712                         hifnstats.hst_opackets++;
2713                         hifn_callback(sc, cmd, macbuf);
2714                 } else {
2715                         if (cmd->src_map == cmd->dst_map) {
2716                                 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2717                                     BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
2718                         } else {
2719                                 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2720                                     BUS_DMASYNC_POSTWRITE);
2721                                 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
2722                                     BUS_DMASYNC_POSTREAD);
2723                         }
2724
2725                         if (cmd->src_m != cmd->dst_m) {
2726                                 m_freem(cmd->src_m);
2727                                 crp->crp_buf = (caddr_t)cmd->dst_m;
2728                         }
2729
2730                         /* non-shared buffers cannot be restarted */
2731                         if (cmd->src_map != cmd->dst_map) {
2732                                 /*
2733                                  * XXX should be EAGAIN, delayed until
2734                                  * after the reset.
2735                                  */
2736                                 crp->crp_etype = ENOMEM;
2737                                 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
2738                                 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
2739                         } else
2740                                 crp->crp_etype = ENOMEM;
2741
2742                         bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
2743                         bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
2744
2745                         free(cmd, M_DEVBUF);
2746                         if (crp->crp_etype != EAGAIN)
2747                                 crypto_done(crp);
2748                 }
2749
2750                 if (++i == HIFN_D_RES_RSIZE)
2751                         i = 0;
2752                 u--;
2753         }
2754         sc->sc_resk = i; sc->sc_resu = u;
2755
2756         hifn_reset_board(sc, 1);
2757         hifn_init_dma(sc);
2758         hifn_init_pci_registers(sc);
2759 }
2760
2761 static void
2762 hifn_callback(struct hifn_softc *sc, struct hifn_command *cmd, u_int8_t *macbuf)
2763 {
2764         struct hifn_dma *dma = sc->sc_dma;
2765         struct cryptop *crp = cmd->crp;
2766         struct cryptodesc *crd;
2767         struct mbuf *m;
2768         int totlen, i, u, ivlen;
2769
2770         if (cmd->src_map == cmd->dst_map) {
2771                 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2772                     BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
2773         } else {
2774                 bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
2775                     BUS_DMASYNC_POSTWRITE);
2776                 bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
2777                     BUS_DMASYNC_POSTREAD);
2778         }
2779
2780         if (crp->crp_flags & CRYPTO_F_IMBUF) {
2781                 if (cmd->src_m != cmd->dst_m) {
2782                         crp->crp_buf = (caddr_t)cmd->dst_m;
2783                         totlen = cmd->src_mapsize;
2784                         for (m = cmd->dst_m; m != NULL; m = m->m_next) {
2785                                 if (totlen < m->m_len) {
2786                                         m->m_len = totlen;
2787                                         totlen = 0;
2788                                 } else
2789                                         totlen -= m->m_len;
2790                         }
2791                         cmd->dst_m->m_pkthdr.len = cmd->src_m->m_pkthdr.len;
2792                         m_freem(cmd->src_m);
2793                 }
2794         }
2795
2796         if (cmd->sloplen != 0) {
2797                 crypto_copyback(crp->crp_flags, crp->crp_buf,
2798                     cmd->src_mapsize - cmd->sloplen, cmd->sloplen,
2799                     (caddr_t)&dma->slop[cmd->slopidx]);
2800         }
2801
2802         i = sc->sc_dstk; u = sc->sc_dstu;
2803         while (u != 0) {
2804                 if (i == HIFN_D_DST_RSIZE)
2805                         i = 0;
2806                 bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
2807                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2808                 if (dma->dstr[i].l & htole32(HIFN_D_VALID)) {
2809                         bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
2810                             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2811                         break;
2812                 }
2813                 i++, u--;
2814         }
2815         sc->sc_dstk = i; sc->sc_dstu = u;
2816
2817         hifnstats.hst_obytes += cmd->dst_mapsize;
2818
2819         if ((cmd->base_masks & (HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE)) ==
2820             HIFN_BASE_CMD_CRYPT) {
2821                 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
2822                         if (crd->crd_alg != CRYPTO_DES_CBC &&
2823                             crd->crd_alg != CRYPTO_3DES_CBC &&
2824                             crd->crd_alg != CRYPTO_AES_CBC)
2825                                 continue;
2826                         ivlen = ((crd->crd_alg == CRYPTO_AES_CBC) ?
2827                                 HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
2828                         crypto_copydata(crp->crp_flags, crp->crp_buf,
2829                             crd->crd_skip + crd->crd_len - ivlen, ivlen,
2830                             cmd->softc->sc_sessions[cmd->session_num].hs_iv);
2831                         break;
2832                 }
2833         }
2834
2835         if (macbuf != NULL) {
2836                 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
2837                         int len;
2838
2839                         if (crd->crd_alg != CRYPTO_MD5 &&
2840                             crd->crd_alg != CRYPTO_SHA1 &&
2841                             crd->crd_alg != CRYPTO_MD5_HMAC &&
2842                             crd->crd_alg != CRYPTO_SHA1_HMAC) {
2843                                 continue;
2844                         }
2845                         len = cmd->softc->sc_sessions[cmd->session_num].hs_mlen;
2846                         crypto_copyback(crp->crp_flags, crp->crp_buf,
2847                             crd->crd_inject, len, macbuf);
2848                         break;
2849                 }
2850         }
2851
2852         if (cmd->src_map != cmd->dst_map) {
2853                 bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
2854                 bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
2855         }
2856         bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
2857         bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
2858         free(cmd, M_DEVBUF);
2859         crypto_done(crp);
2860 }
2861
2862 /*
2863  * 7811 PB3 rev/2 parts lock-up on burst writes to Group 0
2864  * and Group 1 registers; avoid conditions that could create
2865  * burst writes by doing a read in between the writes.
2866  *
2867  * NB: The read we interpose is always to the same register;
2868  *     we do this because reading from an arbitrary (e.g. last)
2869  *     register may not always work.
2870  */
2871 static void
2872 hifn_write_reg_0(struct hifn_softc *sc, bus_size_t reg, u_int32_t val)
2873 {
2874         if (sc->sc_flags & HIFN_IS_7811) {
2875                 if (sc->sc_bar0_lastreg == reg - 4)
2876                         bus_space_read_4(sc->sc_st0, sc->sc_sh0, HIFN_0_PUCNFG);
2877                 sc->sc_bar0_lastreg = reg;
2878         }
2879         bus_space_write_4(sc->sc_st0, sc->sc_sh0, reg, val);
2880 }
2881
2882 static void
2883 hifn_write_reg_1(struct hifn_softc *sc, bus_size_t reg, u_int32_t val)
2884 {
2885         if (sc->sc_flags & HIFN_IS_7811) {
2886                 if (sc->sc_bar1_lastreg == reg - 4)
2887                         bus_space_read_4(sc->sc_st1, sc->sc_sh1, HIFN_1_REVID);
2888                 sc->sc_bar1_lastreg = reg;
2889         }
2890         bus_space_write_4(sc->sc_st1, sc->sc_sh1, reg, val);
2891 }
2892
2893 #ifdef HIFN_VULCANDEV
2894 /*
2895  * this code provides support for mapping the PK engine's register
2896  * into a userspace program.
2897  *
2898  */
2899 static int
2900 vulcanpk_mmap(struct cdev *dev, vm_ooffset_t offset,
2901               vm_paddr_t *paddr, int nprot, vm_memattr_t *memattr)
2902 {
2903         struct hifn_softc *sc;
2904         vm_paddr_t pd;
2905         void *b;
2906
2907         sc = dev->si_drv1;
2908
2909         pd = rman_get_start(sc->sc_bar1res);
2910         b = rman_get_virtual(sc->sc_bar1res);
2911
2912 #if 0
2913         printf("vpk mmap: %p(%016llx) offset=%lld\n", b,
2914             (unsigned long long)pd, offset);
2915         hexdump(b, HIFN_1_PUB_MEMEND, "vpk", 0);
2916 #endif
2917
2918         if (offset == 0) {
2919                 *paddr = pd;
2920                 return (0);
2921         }
2922         return (-1);
2923 }
2924
2925 static struct cdevsw vulcanpk_cdevsw = {
2926         .d_version =    D_VERSION,
2927         .d_mmap =       vulcanpk_mmap,
2928         .d_name =       "vulcanpk",
2929 };
2930 #endif /* HIFN_VULCANDEV */