sys/dev/safexcel/safexcel.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2020 Rubicon Communications, LLC (Netgate)
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  */
  26
  27 #include <sys/cdefs.h>
  28 __FBSDID("$FreeBSD$");
  29
  30 #include <sys/param.h>
  31 #include <sys/bus.h>
  32 #include <sys/endian.h>
  33 #include <sys/kernel.h>
  34 #include <sys/lock.h>
  35 #include <sys/malloc.h>
  36 #include <sys/module.h>
  37 #include <sys/mutex.h>
  38 #include <sys/rman.h>
  39 #include <sys/sglist.h>
  40 #include <sys/sysctl.h>
  41
  42 #include <machine/atomic.h>
  43 #include <machine/bus.h>
  44
  45 #include <crypto/rijndael/rijndael.h>
  46 #include <opencrypto/cryptodev.h>
  47 #include <opencrypto/xform.h>
  48
  49 #include <dev/ofw/ofw_bus.h>
  50 #include <dev/ofw/ofw_bus_subr.h>
  51
  52 #include "cryptodev_if.h"
  53
  54 #include "safexcel_reg.h"
  55 #include "safexcel_var.h"
  56
  57 static MALLOC_DEFINE(M_SAFEXCEL, "safexcel_req", "safexcel request buffers");
  58
  59 /*
  60  * We only support the EIP97 for now.
  61  */
  62 static struct ofw_compat_data safexcel_compat[] = {
  63         { "inside-secure,safexcel-eip97ies",    (uintptr_t)97 },
  64         { "inside-secure,safexcel-eip97",       (uintptr_t)97 },
  65         { NULL,                                 0 }
  66 };
  67
  68 const struct safexcel_reg_offsets eip97_regs_offset = {
  69         .hia_aic        = SAFEXCEL_EIP97_HIA_AIC_BASE,
  70         .hia_aic_g      = SAFEXCEL_EIP97_HIA_AIC_G_BASE,
  71         .hia_aic_r      = SAFEXCEL_EIP97_HIA_AIC_R_BASE,
  72         .hia_aic_xdr    = SAFEXCEL_EIP97_HIA_AIC_xDR_BASE,
  73         .hia_dfe        = SAFEXCEL_EIP97_HIA_DFE_BASE,
  74         .hia_dfe_thr    = SAFEXCEL_EIP97_HIA_DFE_THR_BASE,
  75         .hia_dse        = SAFEXCEL_EIP97_HIA_DSE_BASE,
  76         .hia_dse_thr    = SAFEXCEL_EIP97_HIA_DSE_THR_BASE,
  77         .hia_gen_cfg    = SAFEXCEL_EIP97_HIA_GEN_CFG_BASE,
  78         .pe             = SAFEXCEL_EIP97_PE_BASE,
  79 };
  80
  81 const struct safexcel_reg_offsets eip197_regs_offset = {
  82         .hia_aic        = SAFEXCEL_EIP197_HIA_AIC_BASE,
  83         .hia_aic_g      = SAFEXCEL_EIP197_HIA_AIC_G_BASE,
  84         .hia_aic_r      = SAFEXCEL_EIP197_HIA_AIC_R_BASE,
  85         .hia_aic_xdr    = SAFEXCEL_EIP197_HIA_AIC_xDR_BASE,
  86         .hia_dfe        = SAFEXCEL_EIP197_HIA_DFE_BASE,
  87         .hia_dfe_thr    = SAFEXCEL_EIP197_HIA_DFE_THR_BASE,
  88         .hia_dse        = SAFEXCEL_EIP197_HIA_DSE_BASE,
  89         .hia_dse_thr    = SAFEXCEL_EIP197_HIA_DSE_THR_BASE,
  90         .hia_gen_cfg    = SAFEXCEL_EIP197_HIA_GEN_CFG_BASE,
  91         .pe             = SAFEXCEL_EIP197_PE_BASE,
  92 };
  93
  94 static struct safexcel_cmd_descr *
  95 safexcel_cmd_descr_next(struct safexcel_cmd_descr_ring *ring)
  96 {
  97         struct safexcel_cmd_descr *cdesc;
  98
  99         if (ring->write == ring->read)
 100                 return (NULL);
 101         cdesc = &ring->desc[ring->read];
 102         ring->read = (ring->read + 1) % SAFEXCEL_RING_SIZE;
 103         return (cdesc);
 104 }
 105
 106 static struct safexcel_res_descr *
 107 safexcel_res_descr_next(struct safexcel_res_descr_ring *ring)
 108 {
 109         struct safexcel_res_descr *rdesc;
 110
 111         if (ring->write == ring->read)
 112                 return (NULL);
 113         rdesc = &ring->desc[ring->read];
 114         ring->read = (ring->read + 1) % SAFEXCEL_RING_SIZE;
 115         return (rdesc);
 116 }
 117
 118 static struct safexcel_request *
 119 safexcel_alloc_request(struct safexcel_softc *sc, struct safexcel_ring *ring)
 120 {
 121         struct safexcel_request *req;
 122
 123         mtx_assert(&ring->mtx, MA_OWNED);
 124
 125         if ((req = STAILQ_FIRST(&ring->free_requests)) != NULL)
 126                 STAILQ_REMOVE_HEAD(&ring->free_requests, link);
 127         return (req);
 128 }
 129
 130 static void
 131 safexcel_free_request(struct safexcel_ring *ring, struct safexcel_request *req)
 132 {
 133         struct safexcel_context_record *ctx;
 134
 135         mtx_assert(&ring->mtx, MA_OWNED);
 136
 137         if (req->dmap_loaded) {
 138                 bus_dmamap_unload(ring->data_dtag, req->dmap);
 139                 req->dmap_loaded = false;
 140         }
 141         ctx = (struct safexcel_context_record *)req->ctx.vaddr;
 142         explicit_bzero(ctx->data, sizeof(ctx->data));
 143         explicit_bzero(req->iv, sizeof(req->iv));
 144         STAILQ_INSERT_TAIL(&ring->free_requests, req, link);
 145 }
 146
 147 static void
 148 safexcel_enqueue_request(struct safexcel_softc *sc, struct safexcel_ring *ring,
 149     struct safexcel_request *req)
 150 {
 151         mtx_assert(&ring->mtx, MA_OWNED);
 152
 153         STAILQ_INSERT_TAIL(&ring->ready_requests, req, link);
 154 }
 155
 156 static void
 157 safexcel_rdr_intr(struct safexcel_softc *sc, int ringidx)
 158 {
 159         struct safexcel_cmd_descr *cdesc;
 160         struct safexcel_res_descr *rdesc;
 161         struct safexcel_request *req;
 162         struct safexcel_ring *ring;
 163         uint32_t error, i, ncdescs, nrdescs, nreqs;
 164
 165         ring = &sc->sc_ring[ringidx];
 166
 167         mtx_lock(&ring->mtx);
 168         nreqs = SAFEXCEL_READ(sc,
 169             SAFEXCEL_HIA_RDR(sc, ringidx) + SAFEXCEL_HIA_xDR_PROC_COUNT);
 170         nreqs >>= SAFEXCEL_xDR_PROC_xD_PKT_OFFSET;
 171         nreqs &= SAFEXCEL_xDR_PROC_xD_PKT_MASK;
 172         if (nreqs == 0) {
 173                 SAFEXCEL_DPRINTF(sc, 1,
 174                     "zero pending requests on ring %d\n", ringidx);
 175                 goto out;
 176         }
 177
 178         ring = &sc->sc_ring[ringidx];
 179         bus_dmamap_sync(ring->rdr.dma.tag, ring->rdr.dma.map,
 180             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 181         bus_dmamap_sync(ring->cdr.dma.tag, ring->cdr.dma.map,
 182             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 183         bus_dmamap_sync(ring->dma_atok.tag, ring->dma_atok.map,
 184             BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 185
 186         ncdescs = nrdescs = 0;
 187         for (i = 0; i < nreqs; i++) {
 188                 req = STAILQ_FIRST(&ring->queued_requests);
 189                 KASSERT(req != NULL, ("%s: expected %d pending requests",
 190                     __func__, nreqs));
 191                 STAILQ_REMOVE_HEAD(&ring->queued_requests, link);
 192                 mtx_unlock(&ring->mtx);
 193
 194                 bus_dmamap_sync(req->ctx.tag, req->ctx.map,
 195                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 196                 bus_dmamap_sync(ring->data_dtag, req->dmap,
 197                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 198
 199                 ncdescs += req->cdescs;
 200                 while (req->cdescs-- > 0) {
 201                         cdesc = safexcel_cmd_descr_next(&ring->cdr);
 202                         KASSERT(cdesc != NULL,
 203                             ("%s: missing control descriptor", __func__));
 204                         if (req->cdescs == 0)
 205                                 KASSERT(cdesc->last_seg,
 206                                     ("%s: chain is not terminated", __func__));
 207                 }
 208                 nrdescs += req->rdescs;
 209                 while (req->rdescs-- > 0) {
 210                         rdesc = safexcel_res_descr_next(&ring->rdr);
 211                         error = rdesc->result_data.error_code;
 212                         if (error != 0) {
 213                                 if (error == SAFEXCEL_RESULT_ERR_AUTH_FAILED &&
 214                                     req->crp->crp_etype == 0) {
 215                                         req->crp->crp_etype = EBADMSG;
 216                                 } else {
 217                                         SAFEXCEL_DPRINTF(sc, 1,
 218                                             "error code %#x\n", error);
 219                                         req->crp->crp_etype = EIO;
 220                                 }
 221                         }
 222                 }
 223
 224                 crypto_done(req->crp);
 225                 mtx_lock(&ring->mtx);
 226                 safexcel_free_request(ring, req);
 227         }
 228
 229         if (nreqs != 0) {
 230                 SAFEXCEL_WRITE(sc,
 231                     SAFEXCEL_HIA_RDR(sc, ringidx) + SAFEXCEL_HIA_xDR_PROC_COUNT,
 232                     SAFEXCEL_xDR_PROC_xD_PKT(nreqs) |
 233                     (sc->sc_config.rd_offset * nrdescs * sizeof(uint32_t)));
 234         }
 235 out:
 236         if (!STAILQ_EMPTY(&ring->queued_requests)) {
 237                 SAFEXCEL_WRITE(sc,
 238                     SAFEXCEL_HIA_RDR(sc, ringidx) + SAFEXCEL_HIA_xDR_THRESH,
 239                     SAFEXCEL_HIA_CDR_THRESH_PKT_MODE | 1);
 240         }
 241         mtx_unlock(&ring->mtx);
 242 }
 243
 244 static void
 245 safexcel_ring_intr(void *arg)
 246 {
 247         struct safexcel_softc *sc;
 248         struct safexcel_intr_handle *ih;
 249         uint32_t status, stat;
 250         int ring;
 251         bool blocked, rdrpending;
 252
 253         ih = arg;
 254         sc = ih->sc;
 255         ring = ih->ring;
 256
 257         status = SAFEXCEL_READ(sc, SAFEXCEL_HIA_AIC_R(sc) +
 258             SAFEXCEL_HIA_AIC_R_ENABLED_STAT(ring));
 259         /* CDR interrupts */
 260         if (status & SAFEXCEL_CDR_IRQ(ring)) {
 261                 stat = SAFEXCEL_READ(sc,
 262                     SAFEXCEL_HIA_CDR(sc, ring) + SAFEXCEL_HIA_xDR_STAT);
 263                 SAFEXCEL_WRITE(sc,
 264                     SAFEXCEL_HIA_CDR(sc, ring) + SAFEXCEL_HIA_xDR_STAT,
 265                     stat & SAFEXCEL_CDR_INTR_MASK);
 266         }
 267         /* RDR interrupts */
 268         rdrpending = false;
 269         if (status & SAFEXCEL_RDR_IRQ(ring)) {
 270                 stat = SAFEXCEL_READ(sc,
 271                     SAFEXCEL_HIA_RDR(sc, ring) + SAFEXCEL_HIA_xDR_STAT);
 272                 if ((stat & SAFEXCEL_xDR_ERR) == 0)
 273                         rdrpending = true;
 274                 SAFEXCEL_WRITE(sc,
 275                     SAFEXCEL_HIA_RDR(sc, ring) + SAFEXCEL_HIA_xDR_STAT,
 276                     stat & SAFEXCEL_RDR_INTR_MASK);
 277         }
 278         SAFEXCEL_WRITE(sc,
 279             SAFEXCEL_HIA_AIC_R(sc) + SAFEXCEL_HIA_AIC_R_ACK(ring),
 280             status);
 281
 282         if (rdrpending)
 283                 safexcel_rdr_intr(sc, ring);
 284
 285         mtx_lock(&sc->sc_mtx);
 286         blocked = sc->sc_blocked;
 287         sc->sc_blocked = 0;
 288         mtx_unlock(&sc->sc_mtx);
 289
 290         if (blocked)
 291                 crypto_unblock(sc->sc_cid, blocked);
 292 }
 293
 294 static int
 295 safexcel_configure(struct safexcel_softc *sc)
 296 {
 297         uint32_t i, mask, pemask, reg;
 298         device_t dev;
 299
 300         if (sc->sc_type == 197) {
 301                 sc->sc_offsets = eip197_regs_offset;
 302                 pemask = SAFEXCEL_N_PES_MASK;
 303         } else {
 304                 sc->sc_offsets = eip97_regs_offset;
 305                 pemask = EIP97_N_PES_MASK;
 306         }
 307
 308         dev = sc->sc_dev;
 309
 310         /* Scan for valid ring interrupt controllers. */
 311         for (i = 0; i < SAFEXCEL_MAX_RING_AIC; i++) {
 312                 reg = SAFEXCEL_READ(sc, SAFEXCEL_HIA_AIC_R(sc) +
 313                     SAFEXCEL_HIA_AIC_R_VERSION(i));
 314                 if (SAFEXCEL_REG_LO16(reg) != EIP201_VERSION_LE)
 315                         break;
 316         }
 317         sc->sc_config.aic_rings = i;
 318         if (sc->sc_config.aic_rings == 0)
 319                 return (-1);
 320
 321         reg = SAFEXCEL_READ(sc, SAFEXCEL_HIA_AIC_G(sc) + SAFEXCEL_HIA_OPTIONS);
 322         /* Check for 64bit addressing. */
 323         if ((reg & SAFEXCEL_OPT_ADDR_64) == 0)
 324                 return (-1);
 325         /* Check alignment constraints (which we do not support). */
 326         if (((reg & SAFEXCEL_OPT_TGT_ALIGN_MASK) >>
 327             SAFEXCEL_OPT_TGT_ALIGN_OFFSET) != 0)
 328                 return (-1);
 329
 330         sc->sc_config.hdw =
 331             (reg & SAFEXCEL_xDR_HDW_MASK) >> SAFEXCEL_xDR_HDW_OFFSET;
 332         mask = (1 << sc->sc_config.hdw) - 1;
 333
 334         sc->sc_config.rings = reg & SAFEXCEL_N_RINGS_MASK;
 335         /* Limit the number of rings to the number of the AIC Rings. */
 336         sc->sc_config.rings = MIN(sc->sc_config.rings, sc->sc_config.aic_rings);
 337
 338         sc->sc_config.pes = (reg & pemask) >> SAFEXCEL_N_PES_OFFSET;
 339
 340         sc->sc_config.cd_size =
 341             sizeof(struct safexcel_cmd_descr) / sizeof(uint32_t);
 342         sc->sc_config.cd_offset = (sc->sc_config.cd_size + mask) & ~mask;
 343
 344         sc->sc_config.rd_size =
 345             sizeof(struct safexcel_res_descr) / sizeof(uint32_t);
 346         sc->sc_config.rd_offset = (sc->sc_config.rd_size + mask) & ~mask;
 347
 348         sc->sc_config.atok_offset =
 349             (SAFEXCEL_MAX_ATOKENS * sizeof(struct safexcel_instr) + mask) &
 350             ~mask;
 351
 352         return (0);
 353 }
 354
 355 static void
 356 safexcel_init_hia_bus_access(struct safexcel_softc *sc)
 357 {
 358         uint32_t version, val;
 359
 360         /* Determine endianness and configure byte swap. */
 361         version = SAFEXCEL_READ(sc,
 362             SAFEXCEL_HIA_AIC(sc) + SAFEXCEL_HIA_VERSION);
 363         val = SAFEXCEL_READ(sc, SAFEXCEL_HIA_AIC(sc) + SAFEXCEL_HIA_MST_CTRL);
 364         if (SAFEXCEL_REG_HI16(version) == SAFEXCEL_HIA_VERSION_BE) {
 365                 val = SAFEXCEL_READ(sc,
 366                     SAFEXCEL_HIA_AIC(sc) + SAFEXCEL_HIA_MST_CTRL);
 367                 val = val ^ (SAFEXCEL_MST_CTRL_NO_BYTE_SWAP >> 24);
 368                 SAFEXCEL_WRITE(sc,
 369                     SAFEXCEL_HIA_AIC(sc) + SAFEXCEL_HIA_MST_CTRL,
 370                     val);
 371         }
 372
 373         /* Configure wr/rd cache values. */
 374         SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_GEN_CFG(sc) + SAFEXCEL_HIA_MST_CTRL,
 375             SAFEXCEL_MST_CTRL_RD_CACHE(RD_CACHE_4BITS) |
 376             SAFEXCEL_MST_CTRL_WD_CACHE(WR_CACHE_4BITS));
 377 }
 378
 379 static void
 380 safexcel_disable_global_interrupts(struct safexcel_softc *sc)
 381 {
 382         /* Disable and clear pending interrupts. */
 383         SAFEXCEL_WRITE(sc,
 384             SAFEXCEL_HIA_AIC_G(sc) + SAFEXCEL_HIA_AIC_G_ENABLE_CTRL, 0);
 385         SAFEXCEL_WRITE(sc,
 386             SAFEXCEL_HIA_AIC_G(sc) + SAFEXCEL_HIA_AIC_G_ACK,
 387             SAFEXCEL_AIC_G_ACK_ALL_MASK);
 388 }
 389
 390 /*
 391  * Configure the data fetch engine.  This component parses command descriptors
 392  * and sets up DMA transfers from host memory to the corresponding processing
 393  * engine.
 394  */
 395 static void
 396 safexcel_configure_dfe_engine(struct safexcel_softc *sc, int pe)
 397 {
 398         /* Reset all DFE threads. */
 399         SAFEXCEL_WRITE(sc,
 400             SAFEXCEL_HIA_DFE_THR(sc) + SAFEXCEL_HIA_DFE_THR_CTRL(pe),
 401             SAFEXCEL_DxE_THR_CTRL_RESET_PE);
 402
 403         /* Deassert the DFE reset. */
 404         SAFEXCEL_WRITE(sc,
 405             SAFEXCEL_HIA_DFE_THR(sc) + SAFEXCEL_HIA_DFE_THR_CTRL(pe), 0);
 406
 407         /* DMA transfer size to use. */
 408         SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DFE(sc) + SAFEXCEL_HIA_DFE_CFG(pe),
 409             SAFEXCEL_HIA_DFE_CFG_DIS_DEBUG |
 410             SAFEXCEL_HIA_DxE_CFG_MIN_DATA_SIZE(6) |
 411             SAFEXCEL_HIA_DxE_CFG_MAX_DATA_SIZE(9) |
 412             SAFEXCEL_HIA_DxE_CFG_MIN_CTRL_SIZE(6) |
 413             SAFEXCEL_HIA_DxE_CFG_MAX_CTRL_SIZE(7) |
 414             SAFEXCEL_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS) |
 415             SAFEXCEL_HIA_DxE_CFG_CTRL_CACHE_CTRL(RD_CACHE_3BITS));
 416
 417         /* Configure the PE DMA transfer thresholds. */
 418         SAFEXCEL_WRITE(sc, SAFEXCEL_PE(sc) + SAFEXCEL_PE_IN_DBUF_THRES(pe),
 419             SAFEXCEL_PE_IN_xBUF_THRES_MIN(6) |
 420             SAFEXCEL_PE_IN_xBUF_THRES_MAX(9));
 421         SAFEXCEL_WRITE(sc, SAFEXCEL_PE(sc) + SAFEXCEL_PE_IN_TBUF_THRES(pe),
 422             SAFEXCEL_PE_IN_xBUF_THRES_MIN(6) |
 423             SAFEXCEL_PE_IN_xBUF_THRES_MAX(7));
 424 }
 425
 426 /*
 427  * Configure the data store engine.  This component parses result descriptors
 428  * and sets up DMA transfers from the processing engine to host memory.
 429  */
 430 static int
 431 safexcel_configure_dse(struct safexcel_softc *sc, int pe)
 432 {
 433         uint32_t val;
 434         int count;
 435
 436         /* Disable and reset all DSE threads. */
 437         SAFEXCEL_WRITE(sc,
 438             SAFEXCEL_HIA_DSE_THR(sc) + SAFEXCEL_HIA_DSE_THR_CTRL(pe),
 439             SAFEXCEL_DxE_THR_CTRL_RESET_PE);
 440
 441         /* Wait for a second for threads to go idle. */
 442         for (count = 0;;) {
 443                 val = SAFEXCEL_READ(sc,
 444                     SAFEXCEL_HIA_DSE_THR(sc) + SAFEXCEL_HIA_DSE_THR_STAT(pe));
 445                 if ((val & SAFEXCEL_DSE_THR_RDR_ID_MASK) ==
 446                     SAFEXCEL_DSE_THR_RDR_ID_MASK)
 447                         break;
 448                 if (count++ > 10000) {
 449                         device_printf(sc->sc_dev, "DSE reset timeout\n");
 450                         return (-1);
 451                 }
 452                 DELAY(100);
 453         }
 454
 455         /* Exit the reset state. */
 456         SAFEXCEL_WRITE(sc,
 457             SAFEXCEL_HIA_DSE_THR(sc) + SAFEXCEL_HIA_DSE_THR_CTRL(pe), 0);
 458
 459         /* DMA transfer size to use */
 460         SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DSE(sc) + SAFEXCEL_HIA_DSE_CFG(pe),
 461             SAFEXCEL_HIA_DSE_CFG_DIS_DEBUG |
 462             SAFEXCEL_HIA_DxE_CFG_MIN_DATA_SIZE(7) |
 463             SAFEXCEL_HIA_DxE_CFG_MAX_DATA_SIZE(8) |
 464             SAFEXCEL_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS) |
 465             SAFEXCEL_HIA_DSE_CFG_ALLWAYS_BUFFERABLE);
 466
 467         /* Configure the procesing engine thresholds */
 468         SAFEXCEL_WRITE(sc,
 469             SAFEXCEL_PE(sc) + SAFEXCEL_PE_OUT_DBUF_THRES(pe),
 470             SAFEXCEL_PE_OUT_DBUF_THRES_MIN(7) |
 471             SAFEXCEL_PE_OUT_DBUF_THRES_MAX(8));
 472
 473         return (0);
 474 }
 475
 476 static void
 477 safexcel_hw_prepare_rings(struct safexcel_softc *sc)
 478 {
 479         int i;
 480
 481         for (i = 0; i < sc->sc_config.rings; i++) {
 482                 /*
 483                  * Command descriptors.
 484                  */
 485
 486                 /* Clear interrupts for this ring. */
 487                 SAFEXCEL_WRITE(sc,
 488                     SAFEXCEL_HIA_AIC_R(sc) + SAFEXCEL_HIA_AIC_R_ENABLE_CLR(i),
 489                     SAFEXCEL_HIA_AIC_R_ENABLE_CLR_ALL_MASK);
 490
 491                 /* Disable external triggering. */
 492                 SAFEXCEL_WRITE(sc,
 493                     SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_CFG, 0);
 494
 495                 /* Clear the pending prepared counter. */
 496                 SAFEXCEL_WRITE(sc,
 497                     SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_COUNT,
 498                     SAFEXCEL_xDR_PREP_CLR_COUNT);
 499
 500                 /* Clear the pending processed counter. */
 501                 SAFEXCEL_WRITE(sc,
 502                     SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_COUNT,
 503                     SAFEXCEL_xDR_PROC_CLR_COUNT);
 504
 505                 SAFEXCEL_WRITE(sc,
 506                     SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_PNTR, 0);
 507                 SAFEXCEL_WRITE(sc,
 508                     SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_PNTR, 0);
 509
 510                 SAFEXCEL_WRITE(sc,
 511                     SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_RING_SIZE,
 512                     SAFEXCEL_RING_SIZE * sc->sc_config.cd_offset *
 513                     sizeof(uint32_t));
 514
 515                 /*
 516                  * Result descriptors.
 517                  */
 518
 519                 /* Disable external triggering. */
 520                 SAFEXCEL_WRITE(sc,
 521                     SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_CFG, 0);
 522
 523                 /* Clear the pending prepared counter. */
 524                 SAFEXCEL_WRITE(sc,
 525                     SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_COUNT,
 526                     SAFEXCEL_xDR_PREP_CLR_COUNT);
 527
 528                 /* Clear the pending processed counter. */
 529                 SAFEXCEL_WRITE(sc,
 530                     SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_COUNT,
 531                     SAFEXCEL_xDR_PROC_CLR_COUNT);
 532
 533                 SAFEXCEL_WRITE(sc,
 534                     SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_PNTR, 0);
 535                 SAFEXCEL_WRITE(sc,
 536                     SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_PNTR, 0);
 537
 538                 /* Ring size. */
 539                 SAFEXCEL_WRITE(sc,
 540                     SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_RING_SIZE,
 541                     SAFEXCEL_RING_SIZE * sc->sc_config.rd_offset *
 542                     sizeof(uint32_t));
 543         }
 544 }
 545
 546 static void
 547 safexcel_hw_setup_rings(struct safexcel_softc *sc)
 548 {
 549         struct safexcel_ring *ring;
 550         uint32_t cd_size_rnd, mask, rd_size_rnd, val;
 551         int i;
 552
 553         mask = (1 << sc->sc_config.hdw) - 1;
 554         cd_size_rnd = (sc->sc_config.cd_size + mask) >> sc->sc_config.hdw;
 555         val = (sizeof(struct safexcel_res_descr) -
 556             sizeof(struct safexcel_res_data)) / sizeof(uint32_t);
 557         rd_size_rnd = (val + mask) >> sc->sc_config.hdw;
 558
 559         for (i = 0; i < sc->sc_config.rings; i++) {
 560                 ring = &sc->sc_ring[i];
 561
 562                 /*
 563                  * Command descriptors.
 564                  */
 565
 566                 /* Ring base address. */
 567                 SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) +
 568                     SAFEXCEL_HIA_xDR_RING_BASE_ADDR_LO,
 569                     SAFEXCEL_ADDR_LO(ring->cdr.dma.paddr));
 570                 SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) +
 571                     SAFEXCEL_HIA_xDR_RING_BASE_ADDR_HI,
 572                     SAFEXCEL_ADDR_HI(ring->cdr.dma.paddr));
 573
 574                 SAFEXCEL_WRITE(sc,
 575                     SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_DESC_SIZE,
 576                     SAFEXCEL_xDR_DESC_MODE_64BIT | SAFEXCEL_CDR_DESC_MODE_ADCP |
 577                     (sc->sc_config.cd_offset << SAFEXCEL_xDR_DESC_xD_OFFSET) |
 578                     sc->sc_config.cd_size);
 579
 580                 SAFEXCEL_WRITE(sc,
 581                     SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_CFG,
 582                     ((SAFEXCEL_FETCH_COUNT * (cd_size_rnd << sc->sc_config.hdw)) <<
 583                       SAFEXCEL_xDR_xD_FETCH_THRESH) |
 584                     (SAFEXCEL_FETCH_COUNT * sc->sc_config.cd_offset));
 585
 586                 /* Configure DMA tx control. */
 587                 SAFEXCEL_WRITE(sc,
 588                     SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_DMA_CFG,
 589                     SAFEXCEL_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS) |
 590                     SAFEXCEL_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS));
 591
 592                 /* Clear any pending interrupt. */
 593                 SAFEXCEL_WRITE(sc,
 594                     SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_STAT,
 595                     SAFEXCEL_CDR_INTR_MASK);
 596
 597                 /*
 598                  * Result descriptors.
 599                  */
 600
 601                 /* Ring base address. */
 602                 SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) +
 603                     SAFEXCEL_HIA_xDR_RING_BASE_ADDR_LO,
 604                     SAFEXCEL_ADDR_LO(ring->rdr.dma.paddr));
 605                 SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) +
 606                     SAFEXCEL_HIA_xDR_RING_BASE_ADDR_HI,
 607                     SAFEXCEL_ADDR_HI(ring->rdr.dma.paddr));
 608
 609                 SAFEXCEL_WRITE(sc,
 610                     SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_DESC_SIZE,
 611                     SAFEXCEL_xDR_DESC_MODE_64BIT |
 612                     (sc->sc_config.rd_offset << SAFEXCEL_xDR_DESC_xD_OFFSET) |
 613                     sc->sc_config.rd_size);
 614
 615                 SAFEXCEL_WRITE(sc,
 616                     SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_CFG,
 617                     ((SAFEXCEL_FETCH_COUNT * (rd_size_rnd << sc->sc_config.hdw)) <<
 618                     SAFEXCEL_xDR_xD_FETCH_THRESH) |
 619                     (SAFEXCEL_FETCH_COUNT * sc->sc_config.rd_offset));
 620
 621                 /* Configure DMA tx control. */
 622                 SAFEXCEL_WRITE(sc,
 623                     SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_DMA_CFG,
 624                     SAFEXCEL_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS) |
 625                     SAFEXCEL_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS) |
 626                     SAFEXCEL_HIA_xDR_WR_RES_BUF | SAFEXCEL_HIA_xDR_WR_CTRL_BUF);
 627
 628                 /* Clear any pending interrupt. */
 629                 SAFEXCEL_WRITE(sc,
 630                     SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_STAT,
 631                     SAFEXCEL_RDR_INTR_MASK);
 632
 633                 /* Enable ring interrupt. */
 634                 SAFEXCEL_WRITE(sc,
 635                     SAFEXCEL_HIA_AIC_R(sc) + SAFEXCEL_HIA_AIC_R_ENABLE_CTRL(i),
 636                     SAFEXCEL_RDR_IRQ(i));
 637         }
 638 }
 639
 640 /* Reset the command and result descriptor rings. */
 641 static void
 642 safexcel_hw_reset_rings(struct safexcel_softc *sc)
 643 {
 644         int i;
 645
 646         for (i = 0; i < sc->sc_config.rings; i++) {
 647                 /*
 648                  * Result descriptor ring operations.
 649                  */
 650
 651                 /* Reset ring base address. */
 652                 SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) +
 653                     SAFEXCEL_HIA_xDR_RING_BASE_ADDR_LO, 0);
 654                 SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_RDR(sc, i) +
 655                     SAFEXCEL_HIA_xDR_RING_BASE_ADDR_HI, 0);
 656
 657                 /* Clear the pending prepared counter. */
 658                 SAFEXCEL_WRITE(sc,
 659                     SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_COUNT,
 660                     SAFEXCEL_xDR_PREP_CLR_COUNT);
 661
 662                 /* Clear the pending processed counter. */
 663                 SAFEXCEL_WRITE(sc,
 664                     SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_COUNT,
 665                     SAFEXCEL_xDR_PROC_CLR_COUNT);
 666
 667                 SAFEXCEL_WRITE(sc,
 668                     SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_PNTR, 0);
 669                 SAFEXCEL_WRITE(sc,
 670                     SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_PNTR, 0);
 671
 672                 SAFEXCEL_WRITE(sc,
 673                     SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_RING_SIZE, 0);
 674
 675                 /* Clear any pending interrupt. */
 676                 SAFEXCEL_WRITE(sc,
 677                     SAFEXCEL_HIA_RDR(sc, i) + SAFEXCEL_HIA_xDR_STAT,
 678                     SAFEXCEL_RDR_INTR_MASK);
 679
 680                 /* Disable ring interrupt. */
 681                 SAFEXCEL_WRITE(sc,
 682                     SAFEXCEL_HIA_AIC_R(sc) + SAFEXCEL_HIA_AIC_R_ENABLE_CLR(i),
 683                     SAFEXCEL_RDR_IRQ(i));
 684
 685                 /*
 686                  * Command descriptor ring operations.
 687                  */
 688
 689                 /* Reset ring base address. */
 690                 SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) +
 691                     SAFEXCEL_HIA_xDR_RING_BASE_ADDR_LO, 0);
 692                 SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_CDR(sc, i) +
 693                     SAFEXCEL_HIA_xDR_RING_BASE_ADDR_HI, 0);
 694
 695                 /* Clear the pending prepared counter. */
 696                 SAFEXCEL_WRITE(sc,
 697                     SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_COUNT,
 698                     SAFEXCEL_xDR_PREP_CLR_COUNT);
 699
 700                 /* Clear the pending processed counter. */
 701                 SAFEXCEL_WRITE(sc,
 702                     SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_COUNT,
 703                     SAFEXCEL_xDR_PROC_CLR_COUNT);
 704
 705                 SAFEXCEL_WRITE(sc,
 706                     SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PREP_PNTR, 0);
 707                 SAFEXCEL_WRITE(sc,
 708                     SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_PROC_PNTR, 0);
 709
 710                 SAFEXCEL_WRITE(sc,
 711                     SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_RING_SIZE, 0);
 712
 713                 /* Clear any pending interrupt. */
 714                 SAFEXCEL_WRITE(sc,
 715                     SAFEXCEL_HIA_CDR(sc, i) + SAFEXCEL_HIA_xDR_STAT,
 716                     SAFEXCEL_CDR_INTR_MASK);
 717         }
 718 }
 719
 720 static void
 721 safexcel_enable_pe_engine(struct safexcel_softc *sc, int pe)
 722 {
 723         int i, ring_mask;
 724
 725         for (ring_mask = 0, i = 0; i < sc->sc_config.rings; i++) {
 726                 ring_mask <<= 1;
 727                 ring_mask |= 1;
 728         }
 729
 730         /* Enable command descriptor rings. */
 731         SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DFE_THR(sc) + SAFEXCEL_HIA_DFE_THR_CTRL(pe),
 732             SAFEXCEL_DxE_THR_CTRL_EN | ring_mask);
 733
 734         /* Enable result descriptor rings. */
 735         SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_DSE_THR(sc) + SAFEXCEL_HIA_DSE_THR_CTRL(pe),
 736             SAFEXCEL_DxE_THR_CTRL_EN | ring_mask);
 737
 738         /* Clear any HIA interrupt. */
 739         SAFEXCEL_WRITE(sc, SAFEXCEL_HIA_AIC_G(sc) + SAFEXCEL_HIA_AIC_G_ACK,
 740             SAFEXCEL_AIC_G_ACK_HIA_MASK);
 741 }
 742
 743 static void
 744 safexcel_execute(struct safexcel_softc *sc, struct safexcel_ring *ring,
 745     struct safexcel_request *req)
 746 {
 747         uint32_t ncdescs, nrdescs, nreqs;
 748         int ringidx;
 749         bool busy;
 750
 751         mtx_assert(&ring->mtx, MA_OWNED);
 752
 753         ringidx = req->sess->ringidx;
 754         if (STAILQ_EMPTY(&ring->ready_requests))
 755                 return;
 756         busy = !STAILQ_EMPTY(&ring->queued_requests);
 757         ncdescs = nrdescs = nreqs = 0;
 758         while ((req = STAILQ_FIRST(&ring->ready_requests)) != NULL &&
 759             req->cdescs + ncdescs <= SAFEXCEL_MAX_BATCH_SIZE &&
 760             req->rdescs + nrdescs <= SAFEXCEL_MAX_BATCH_SIZE) {
 761                 STAILQ_REMOVE_HEAD(&ring->ready_requests, link);
 762                 STAILQ_INSERT_TAIL(&ring->queued_requests, req, link);
 763                 ncdescs += req->cdescs;
 764                 nrdescs += req->rdescs;
 765                 nreqs++;
 766         }
 767
 768         if (!busy) {
 769                 SAFEXCEL_WRITE(sc,
 770                     SAFEXCEL_HIA_RDR(sc, ringidx) + SAFEXCEL_HIA_xDR_THRESH,
 771                     SAFEXCEL_HIA_CDR_THRESH_PKT_MODE | nreqs);
 772         }
 773         SAFEXCEL_WRITE(sc,
 774             SAFEXCEL_HIA_RDR(sc, ringidx) + SAFEXCEL_HIA_xDR_PREP_COUNT,
 775             nrdescs * sc->sc_config.rd_offset * sizeof(uint32_t));
 776         SAFEXCEL_WRITE(sc,
 777             SAFEXCEL_HIA_CDR(sc, ringidx) + SAFEXCEL_HIA_xDR_PREP_COUNT,
 778             ncdescs * sc->sc_config.cd_offset * sizeof(uint32_t));
 779 }
 780
 781 static void
 782 safexcel_init_rings(struct safexcel_softc *sc)
 783 {
 784         struct safexcel_cmd_descr *cdesc;
 785         struct safexcel_ring *ring;
 786         char buf[32];
 787         uint64_t atok;
 788         int i, j;
 789
 790         for (i = 0; i < sc->sc_config.rings; i++) {
 791                 ring = &sc->sc_ring[i];
 792
 793                 snprintf(buf, sizeof(buf), "safexcel_ring%d", i);
 794                 mtx_init(&ring->mtx, buf, NULL, MTX_DEF);
 795                 STAILQ_INIT(&ring->free_requests);
 796                 STAILQ_INIT(&ring->ready_requests);
 797                 STAILQ_INIT(&ring->queued_requests);
 798
 799                 ring->cdr.read = ring->cdr.write = 0;
 800                 ring->rdr.read = ring->rdr.write = 0;
 801                 for (j = 0; j < SAFEXCEL_RING_SIZE; j++) {
 802                         cdesc = &ring->cdr.desc[j];
 803                         atok = ring->dma_atok.paddr +
 804                             sc->sc_config.atok_offset * j;
 805                         cdesc->atok_lo = SAFEXCEL_ADDR_LO(atok);
 806                         cdesc->atok_hi = SAFEXCEL_ADDR_HI(atok);
 807                 }
 808         }
 809 }
 810
 811 static void
 812 safexcel_dma_alloc_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg,
 813     int error)
 814 {
 815         struct safexcel_dma_mem *sdm;
 816
 817         if (error != 0)
 818                 return;
 819
 820         KASSERT(nseg == 1, ("%s: nsegs is %d", __func__, nseg));
 821         sdm = arg;
 822         sdm->paddr = segs->ds_addr;
 823 }
 824
 825 static int
 826 safexcel_dma_alloc_mem(struct safexcel_softc *sc, struct safexcel_dma_mem *sdm,
 827     bus_size_t size)
 828 {
 829         int error;
 830
 831         KASSERT(sdm->vaddr == NULL,
 832             ("%s: DMA memory descriptor in use.", __func__));
 833
 834         error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), /* parent */
 835             PAGE_SIZE, 0,               /* alignment, boundary */
 836             BUS_SPACE_MAXADDR_32BIT,    /* lowaddr */
 837             BUS_SPACE_MAXADDR,          /* highaddr */
 838             NULL, NULL,                 /* filtfunc, filtfuncarg */
 839             size, 1,                    /* maxsize, nsegments */
 840             size, BUS_DMA_COHERENT,     /* maxsegsz, flags */
 841             NULL, NULL,                 /* lockfunc, lockfuncarg */
 842             &sdm->tag);                 /* dmat */
 843         if (error != 0) {
 844                 device_printf(sc->sc_dev,
 845                     "failed to allocate busdma tag, error %d\n", error);
 846                 goto err1;
 847         }
 848
 849         error = bus_dmamem_alloc(sdm->tag, (void **)&sdm->vaddr,
 850             BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT, &sdm->map);
 851         if (error != 0) {
 852                 device_printf(sc->sc_dev,
 853                     "failed to allocate DMA safe memory, error %d\n", error);
 854                 goto err2;
 855         }
 856
 857         error = bus_dmamap_load(sdm->tag, sdm->map, sdm->vaddr, size,
 858             safexcel_dma_alloc_mem_cb, sdm, BUS_DMA_NOWAIT);
 859         if (error != 0) {
 860                 device_printf(sc->sc_dev,
 861                     "cannot get address of the DMA memory, error %d\n", error);
 862                 goto err3;
 863         }
 864
 865         return (0);
 866 err3:
 867         bus_dmamem_free(sdm->tag, sdm->vaddr, sdm->map);
 868 err2:
 869         bus_dma_tag_destroy(sdm->tag);
 870 err1:
 871         sdm->vaddr = NULL;
 872
 873         return (error);
 874 }
 875
 876 static void
 877 safexcel_dma_free_mem(struct safexcel_dma_mem *sdm)
 878 {
 879         bus_dmamap_unload(sdm->tag, sdm->map);
 880         bus_dmamem_free(sdm->tag, sdm->vaddr, sdm->map);
 881         bus_dma_tag_destroy(sdm->tag);
 882 }
 883
 884 static void
 885 safexcel_dma_free_rings(struct safexcel_softc *sc)
 886 {
 887         struct safexcel_ring *ring;
 888         int i;
 889
 890         for (i = 0; i < sc->sc_config.rings; i++) {
 891                 ring = &sc->sc_ring[i];
 892                 safexcel_dma_free_mem(&ring->cdr.dma);
 893                 safexcel_dma_free_mem(&ring->dma_atok);
 894                 safexcel_dma_free_mem(&ring->rdr.dma);
 895                 bus_dma_tag_destroy(ring->data_dtag);
 896                 mtx_destroy(&ring->mtx);
 897         }
 898 }
 899
 900 static int
 901 safexcel_dma_init(struct safexcel_softc *sc)
 902 {
 903         struct safexcel_ring *ring;
 904         bus_size_t size;
 905         int error, i;
 906
 907         for (i = 0; i < sc->sc_config.rings; i++) {
 908                 ring = &sc->sc_ring[i];
 909
 910                 error = bus_dma_tag_create(
 911                     bus_get_dma_tag(sc->sc_dev),/* parent */
 912                     1, 0,                       /* alignment, boundary */
 913                     BUS_SPACE_MAXADDR_32BIT,    /* lowaddr */
 914                     BUS_SPACE_MAXADDR,          /* highaddr */
 915                     NULL, NULL,                 /* filtfunc, filtfuncarg */
 916                     SAFEXCEL_MAX_REQUEST_SIZE,  /* maxsize */
 917                     SAFEXCEL_MAX_FRAGMENTS,     /* nsegments */
 918                     SAFEXCEL_MAX_REQUEST_SIZE,  /* maxsegsz */
 919                     BUS_DMA_COHERENT,           /* flags */
 920                     NULL, NULL,                 /* lockfunc, lockfuncarg */
 921                     &ring->data_dtag);          /* dmat */
 922                 if (error != 0) {
 923                         device_printf(sc->sc_dev,
 924                             "bus_dma_tag_create main failed; error %d\n", error);
 925                         return (error);
 926                 }
 927
 928                 size = sizeof(uint32_t) * sc->sc_config.cd_offset *
 929                     SAFEXCEL_RING_SIZE;
 930                 error = safexcel_dma_alloc_mem(sc, &ring->cdr.dma, size);
 931                 if (error != 0) {
 932                         device_printf(sc->sc_dev,
 933                             "failed to allocate CDR DMA memory, error %d\n",
 934                             error);
 935                         goto err;
 936                 }
 937                 ring->cdr.desc =
 938                     (struct safexcel_cmd_descr *)ring->cdr.dma.vaddr;
 939
 940                 /* Allocate additional CDR token memory. */
 941                 size = (bus_size_t)sc->sc_config.atok_offset *
 942                     SAFEXCEL_RING_SIZE;
 943                 error = safexcel_dma_alloc_mem(sc, &ring->dma_atok, size);
 944                 if (error != 0) {
 945                         device_printf(sc->sc_dev,
 946                             "failed to allocate atoken DMA memory, error %d\n",
 947                             error);
 948                         goto err;
 949                 }
 950
 951                 size = sizeof(uint32_t) * sc->sc_config.rd_offset *
 952                     SAFEXCEL_RING_SIZE;
 953                 error = safexcel_dma_alloc_mem(sc, &ring->rdr.dma, size);
 954                 if (error) {
 955                         device_printf(sc->sc_dev,
 956                             "failed to allocate RDR DMA memory, error %d\n",
 957                             error);
 958                         goto err;
 959                 }
 960                 ring->rdr.desc =
 961                     (struct safexcel_res_descr *)ring->rdr.dma.vaddr;
 962         }
 963
 964         return (0);
 965 err:
 966         safexcel_dma_free_rings(sc);
 967         return (error);
 968 }
 969
 970 static void
 971 safexcel_deinit_hw(struct safexcel_softc *sc)
 972 {
 973         safexcel_hw_reset_rings(sc);
 974         safexcel_dma_free_rings(sc);
 975 }
 976
 977 static int
 978 safexcel_init_hw(struct safexcel_softc *sc)
 979 {
 980         int pe;
 981
 982         /* 23.3.7 Initialization */
 983         if (safexcel_configure(sc) != 0)
 984                 return (EINVAL);
 985
 986         if (safexcel_dma_init(sc) != 0)
 987                 return (ENOMEM);
 988
 989         safexcel_init_rings(sc);
 990
 991         safexcel_init_hia_bus_access(sc);
 992
 993         /* 23.3.7.2 Disable EIP-97 global Interrupts */
 994         safexcel_disable_global_interrupts(sc);
 995
 996         for (pe = 0; pe < sc->sc_config.pes; pe++) {
 997                 /* 23.3.7.3 Configure Data Fetch Engine */
 998                 safexcel_configure_dfe_engine(sc, pe);
 999
1000                 /* 23.3.7.4 Configure Data Store Engine */
1001                 if (safexcel_configure_dse(sc, pe)) {
1002                         safexcel_deinit_hw(sc);
1003                         return (-1);
1004                 }
1005
1006                 /* 23.3.7.5 1. Protocol enables */
1007                 SAFEXCEL_WRITE(sc,
1008                     SAFEXCEL_PE(sc) + SAFEXCEL_PE_EIP96_FUNCTION_EN(pe),
1009                     0xffffffff);
1010                 SAFEXCEL_WRITE(sc,
1011                     SAFEXCEL_PE(sc) + SAFEXCEL_PE_EIP96_FUNCTION2_EN(pe),
1012                     0xffffffff);
1013         }
1014
1015         safexcel_hw_prepare_rings(sc);
1016
1017         /* 23.3.7.5 Configure the Processing Engine(s). */
1018         for (pe = 0; pe < sc->sc_config.pes; pe++)
1019                 safexcel_enable_pe_engine(sc, pe);
1020
1021         safexcel_hw_setup_rings(sc);
1022
1023         return (0);
1024 }
1025
1026 static int
1027 safexcel_setup_dev_interrupts(struct safexcel_softc *sc)
1028 {
1029         int i, j;
1030
1031         for (i = 0; i < SAFEXCEL_MAX_RINGS && sc->sc_intr[i] != NULL; i++) {
1032                 sc->sc_ih[i].sc = sc;
1033                 sc->sc_ih[i].ring = i;
1034
1035                 if (bus_setup_intr(sc->sc_dev, sc->sc_intr[i],
1036                     INTR_TYPE_NET | INTR_MPSAFE, NULL, safexcel_ring_intr,
1037                     &sc->sc_ih[i], &sc->sc_ih[i].handle)) {
1038                         device_printf(sc->sc_dev,
1039                             "couldn't setup interrupt %d\n", i);
1040                         goto err;
1041                 }
1042         }
1043
1044         return (0);
1045
1046 err:
1047         for (j = 0; j < i; j++)
1048                 bus_teardown_intr(sc->sc_dev, sc->sc_intr[j],
1049                     sc->sc_ih[j].handle);
1050
1051         return (ENXIO);
1052 }
1053
1054 static void
1055 safexcel_teardown_dev_interrupts(struct safexcel_softc *sc)
1056 {
1057         int i;
1058
1059         for (i = 0; i < SAFEXCEL_MAX_RINGS; i++)
1060                 bus_teardown_intr(sc->sc_dev, sc->sc_intr[i],
1061                     sc->sc_ih[i].handle);
1062 }
1063
1064 static int
1065 safexcel_alloc_dev_resources(struct safexcel_softc *sc)
1066 {
1067         char name[16];
1068         device_t dev;
1069         phandle_t node;
1070         int error, i, rid;
1071
1072         dev = sc->sc_dev;
1073         node = ofw_bus_get_node(dev);
1074
1075         rid = 0;
1076         sc->sc_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
1077             RF_ACTIVE);
1078         if (sc->sc_res == NULL) {
1079                 device_printf(dev, "couldn't allocate memory resources\n");
1080                 return (ENXIO);
1081         }
1082
1083         for (i = 0; i < SAFEXCEL_MAX_RINGS; i++) {
1084                 (void)snprintf(name, sizeof(name), "ring%d", i);
1085                 error = ofw_bus_find_string_index(node, "interrupt-names", name,
1086                     &rid);
1087                 if (error != 0)
1088                         break;
1089
1090                 sc->sc_intr[i] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
1091                     RF_ACTIVE | RF_SHAREABLE);
1092                 if (sc->sc_intr[i] == NULL) {
1093                         error = ENXIO;
1094                         goto out;
1095                 }
1096         }
1097         if (i == 0) {
1098                 device_printf(dev, "couldn't allocate interrupt resources\n");
1099                 error = ENXIO;
1100                 goto out;
1101         }
1102
1103         mtx_init(&sc->sc_mtx, "safexcel softc", NULL, MTX_DEF);
1104
1105         return (0);
1106
1107 out:
1108         for (i = 0; i < SAFEXCEL_MAX_RINGS && sc->sc_intr[i] != NULL; i++)
1109                 bus_release_resource(dev, SYS_RES_IRQ,
1110                     rman_get_rid(sc->sc_intr[i]), sc->sc_intr[i]);
1111         bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(sc->sc_res),
1112             sc->sc_res);
1113         return (error);
1114 }
1115
1116 static void
1117 safexcel_free_dev_resources(struct safexcel_softc *sc)
1118 {
1119         int i;
1120
1121         mtx_destroy(&sc->sc_mtx);
1122
1123         for (i = 0; i < SAFEXCEL_MAX_RINGS && sc->sc_intr[i] != NULL; i++)
1124                 bus_release_resource(sc->sc_dev, SYS_RES_IRQ,
1125                     rman_get_rid(sc->sc_intr[i]), sc->sc_intr[i]);
1126         if (sc->sc_res != NULL)
1127                 bus_release_resource(sc->sc_dev, SYS_RES_MEMORY,
1128                     rman_get_rid(sc->sc_res), sc->sc_res);
1129 }
1130
1131 static int
1132 safexcel_probe(device_t dev)
1133 {
1134         struct safexcel_softc *sc;
1135
1136         if (!ofw_bus_status_okay(dev))
1137                 return (ENXIO);
1138
1139         sc = device_get_softc(dev);
1140         sc->sc_type = ofw_bus_search_compatible(dev, safexcel_compat)->ocd_data;
1141         if (sc->sc_type == 0)
1142                 return (ENXIO);
1143
1144         device_set_desc(dev, "SafeXcel EIP-97 crypto accelerator");
1145
1146         return (BUS_PROBE_DEFAULT);
1147 }
1148
1149 static int
1150 safexcel_attach(device_t dev)
1151 {
1152         struct sysctl_ctx_list *sctx;
1153         struct safexcel_softc *sc;
1154         struct safexcel_request *req;
1155         struct safexcel_ring *ring;
1156         int i, j, ringidx;
1157
1158         sc = device_get_softc(dev);
1159         sc->sc_dev = dev;
1160         sc->sc_blocked = 0;
1161         sc->sc_cid = -1;
1162
1163         if (safexcel_alloc_dev_resources(sc))
1164                 goto err;
1165
1166         if (safexcel_setup_dev_interrupts(sc))
1167                 goto err1;
1168
1169         if (safexcel_init_hw(sc))
1170                 goto err2;
1171
1172         for (ringidx = 0; ringidx < sc->sc_config.rings; ringidx++) {
1173                 ring = &sc->sc_ring[ringidx];
1174
1175                 ring->cmd_data = sglist_alloc(SAFEXCEL_MAX_FRAGMENTS, M_WAITOK);
1176                 ring->res_data = sglist_alloc(SAFEXCEL_MAX_FRAGMENTS, M_WAITOK);
1177
1178                 ring->requests = mallocarray(SAFEXCEL_REQUESTS_PER_RING,
1179                     sizeof(struct safexcel_request), M_SAFEXCEL,
1180                     M_WAITOK | M_ZERO);
1181
1182                 for (i = 0; i < SAFEXCEL_REQUESTS_PER_RING; i++) {
1183                         req = &ring->requests[i];
1184                         req->sc = sc;
1185                         if (bus_dmamap_create(ring->data_dtag,
1186                             BUS_DMA_COHERENT, &req->dmap) != 0) {
1187                                 for (j = 0; j < i; j++)
1188                                         bus_dmamap_destroy(ring->data_dtag,
1189                                             ring->requests[j].dmap);
1190                                 goto err2;
1191                         }
1192                         if (safexcel_dma_alloc_mem(sc, &req->ctx,
1193                             sizeof(struct safexcel_context_record)) != 0) {
1194                                 for (j = 0; j < i; j++) {
1195                                         bus_dmamap_destroy(ring->data_dtag,
1196                                             ring->requests[j].dmap);
1197                                         safexcel_dma_free_mem(
1198                                             &ring->requests[j].ctx);
1199                                 }
1200                                 goto err2;
1201                         }
1202                         STAILQ_INSERT_TAIL(&ring->free_requests, req, link);
1203                 }
1204         }
1205
1206         sctx = device_get_sysctl_ctx(dev);
1207         SYSCTL_ADD_INT(sctx, SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
1208             OID_AUTO, "debug", CTLFLAG_RWTUN, &sc->sc_debug, 0,
1209             "Debug message verbosity");
1210
1211         sc->sc_cid = crypto_get_driverid(dev, sizeof(struct safexcel_session),
1212             CRYPTOCAP_F_HARDWARE);
1213         if (sc->sc_cid < 0)
1214                 goto err2;
1215
1216         return (0);
1217
1218 err2:
1219         safexcel_teardown_dev_interrupts(sc);
1220 err1:
1221         safexcel_free_dev_resources(sc);
1222 err:
1223         return (ENXIO);
1224 }
1225
1226 static int
1227 safexcel_detach(device_t dev)
1228 {
1229         struct safexcel_ring *ring;
1230         struct safexcel_softc *sc;
1231         int i, ringidx;
1232
1233         sc = device_get_softc(dev);
1234
1235         if (sc->sc_cid >= 0)
1236                 crypto_unregister_all(sc->sc_cid);
1237         for (ringidx = 0; ringidx < sc->sc_config.rings; ringidx++) {
1238                 ring = &sc->sc_ring[ringidx];
1239                 for (i = 0; i < SAFEXCEL_REQUESTS_PER_RING; i++) {
1240                         bus_dmamap_destroy(ring->data_dtag,
1241                             ring->requests[i].dmap);
1242                         safexcel_dma_free_mem(&ring->requests[i].ctx);
1243                 }
1244                 free(ring->requests, M_SAFEXCEL);
1245                 sglist_free(ring->cmd_data);
1246                 sglist_free(ring->res_data);
1247         }
1248         safexcel_deinit_hw(sc);
1249         safexcel_teardown_dev_interrupts(sc);
1250         safexcel_free_dev_resources(sc);
1251
1252         return (0);
1253 }
1254
1255 /*
1256  * Populate the request's context record with pre-computed key material.
1257  */
1258 static int
1259 safexcel_set_context(struct safexcel_request *req)
1260 {
1261         const struct crypto_session_params *csp;
1262         struct cryptop *crp;
1263         struct safexcel_context_record *ctx;
1264         struct safexcel_session *sess;
1265         uint8_t *data;
1266         int off;
1267
1268         crp = req->crp;
1269         csp = crypto_get_params(crp->crp_session);
1270         sess = req->sess;
1271
1272         ctx = (struct safexcel_context_record *)req->ctx.vaddr;
1273         data = (uint8_t *)ctx->data;
1274         if (csp->csp_cipher_alg != 0) {
1275                 if (crp->crp_cipher_key != NULL)
1276                         memcpy(data, crp->crp_cipher_key, sess->klen);
1277                 else
1278                         memcpy(data, csp->csp_cipher_key, sess->klen);
1279                 off = sess->klen;
1280         } else if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC) {
1281                 if (crp->crp_auth_key != NULL)
1282                         memcpy(data, crp->crp_auth_key, sess->klen);
1283                 else
1284                         memcpy(data, csp->csp_auth_key, sess->klen);
1285                 off = sess->klen;
1286         } else {
1287                 off = 0;
1288         }
1289
1290         switch (csp->csp_cipher_alg) {
1291         case CRYPTO_AES_NIST_GCM_16:
1292                 memcpy(data + off, sess->ghash_key, GMAC_BLOCK_LEN);
1293                 off += GMAC_BLOCK_LEN;
1294                 break;
1295         case CRYPTO_AES_CCM_16:
1296                 memcpy(data + off, sess->xcbc_key,
1297                     AES_BLOCK_LEN * 2 + sess->klen);
1298                 off += AES_BLOCK_LEN * 2 + sess->klen;
1299                 break;
1300         case CRYPTO_AES_XTS:
1301                 memcpy(data + off, sess->tweak_key, sess->klen);
1302                 off += sess->klen;
1303                 break;
1304         }
1305
1306         switch (csp->csp_auth_alg) {
1307         case CRYPTO_AES_NIST_GMAC:
1308                 memcpy(data + off, sess->ghash_key, GMAC_BLOCK_LEN);
1309                 off += GMAC_BLOCK_LEN;
1310                 break;
1311         case CRYPTO_SHA1_HMAC:
1312         case CRYPTO_SHA2_224_HMAC:
1313         case CRYPTO_SHA2_256_HMAC:
1314         case CRYPTO_SHA2_384_HMAC:
1315         case CRYPTO_SHA2_512_HMAC:
1316                 memcpy(data + off, sess->hmac_ipad, sess->statelen);
1317                 off += sess->statelen;
1318                 memcpy(data + off, sess->hmac_opad, sess->statelen);
1319                 off += sess->statelen;
1320                 break;
1321         }
1322
1323         return (off);
1324 }
1325
1326 /*
1327  * Populate fields in the first command descriptor of the chain used to encode
1328  * the specified request.  These fields indicate the algorithms used, the size
1329  * of the key material stored in the associated context record, the primitive
1330  * operations to be performed on input data, and the location of the IV if any.
1331  */
1332 static void
1333 safexcel_set_command(struct safexcel_request *req,
1334     struct safexcel_cmd_descr *cdesc)
1335 {
1336         const struct crypto_session_params *csp;
1337         struct cryptop *crp;
1338         struct safexcel_session *sess;
1339         uint32_t ctrl0, ctrl1, ctxr_len;
1340         int alg;
1341
1342         crp = req->crp;
1343         csp = crypto_get_params(crp->crp_session);
1344         sess = req->sess;
1345
1346         ctrl0 = sess->alg | sess->digest | sess->hash;
1347         ctrl1 = sess->mode;
1348
1349         ctxr_len = safexcel_set_context(req) / sizeof(uint32_t);
1350         ctrl0 |= SAFEXCEL_CONTROL0_SIZE(ctxr_len);
1351
1352         alg = csp->csp_cipher_alg;
1353         if (alg == 0)
1354                 alg = csp->csp_auth_alg;
1355
1356         switch (alg) {
1357         case CRYPTO_AES_CCM_16:
1358                 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
1359                         ctrl0 |= SAFEXCEL_CONTROL0_TYPE_HASH_ENCRYPT_OUT |
1360                             SAFEXCEL_CONTROL0_KEY_EN;
1361                 } else {
1362                         ctrl0 |= SAFEXCEL_CONTROL0_TYPE_DECRYPT_HASH_IN |
1363                             SAFEXCEL_CONTROL0_KEY_EN;
1364                 }
1365                 ctrl1 |= SAFEXCEL_CONTROL1_IV0 | SAFEXCEL_CONTROL1_IV1 |
1366                     SAFEXCEL_CONTROL1_IV2 | SAFEXCEL_CONTROL1_IV3;
1367                 break;
1368         case CRYPTO_AES_CBC:
1369         case CRYPTO_AES_ICM:
1370         case CRYPTO_AES_XTS:
1371                 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
1372                         ctrl0 |= SAFEXCEL_CONTROL0_TYPE_CRYPTO_OUT |
1373                             SAFEXCEL_CONTROL0_KEY_EN;
1374                         if (csp->csp_auth_alg != 0)
1375                                 ctrl0 |=
1376                                     SAFEXCEL_CONTROL0_TYPE_ENCRYPT_HASH_OUT;
1377                 } else {
1378                         ctrl0 |= SAFEXCEL_CONTROL0_TYPE_CRYPTO_IN |
1379                             SAFEXCEL_CONTROL0_KEY_EN;
1380                         if (csp->csp_auth_alg != 0)
1381                                 ctrl0 |= SAFEXCEL_CONTROL0_TYPE_HASH_DECRYPT_IN;
1382                 }
1383                 break;
1384         case CRYPTO_AES_NIST_GCM_16:
1385         case CRYPTO_AES_NIST_GMAC:
1386                 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op) ||
1387                     csp->csp_auth_alg != 0) {
1388                         ctrl0 |= SAFEXCEL_CONTROL0_TYPE_CRYPTO_OUT |
1389                             SAFEXCEL_CONTROL0_KEY_EN |
1390                             SAFEXCEL_CONTROL0_TYPE_HASH_OUT;
1391                 } else {
1392                         ctrl0 |= SAFEXCEL_CONTROL0_TYPE_CRYPTO_IN |
1393                             SAFEXCEL_CONTROL0_KEY_EN |
1394                             SAFEXCEL_CONTROL0_TYPE_HASH_DECRYPT_IN;
1395                 }
1396                 if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16) {
1397                         ctrl1 |= SAFEXCEL_CONTROL1_COUNTER_MODE |
1398                             SAFEXCEL_CONTROL1_IV0 | SAFEXCEL_CONTROL1_IV1 |
1399                             SAFEXCEL_CONTROL1_IV2;
1400                 }
1401                 break;
1402         case CRYPTO_SHA1:
1403         case CRYPTO_SHA2_224:
1404         case CRYPTO_SHA2_256:
1405         case CRYPTO_SHA2_384:
1406         case CRYPTO_SHA2_512:
1407                 ctrl0 |= SAFEXCEL_CONTROL0_RESTART_HASH;
1408                 /* FALLTHROUGH */
1409         case CRYPTO_SHA1_HMAC:
1410         case CRYPTO_SHA2_224_HMAC:
1411         case CRYPTO_SHA2_256_HMAC:
1412         case CRYPTO_SHA2_384_HMAC:
1413         case CRYPTO_SHA2_512_HMAC:
1414                 ctrl0 |= SAFEXCEL_CONTROL0_TYPE_HASH_OUT;
1415                 break;
1416         }
1417
1418         cdesc->control_data.control0 = ctrl0;
1419         cdesc->control_data.control1 = ctrl1;
1420 }
1421
1422 /*
1423  * Construct a no-op instruction, used to pad input tokens.
1424  */
1425 static void
1426 safexcel_instr_nop(struct safexcel_instr **instrp)
1427 {
1428         struct safexcel_instr *instr;
1429
1430         instr = *instrp;
1431         instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT;
1432         instr->length = (1 << 2);
1433         instr->status = 0;
1434         instr->instructions = 0;
1435
1436         *instrp = instr + 1;
1437 }
1438
1439 /*
1440  * Insert the digest of the input payload.  This is typically the last
1441  * instruction of a sequence.
1442  */
1443 static void
1444 safexcel_instr_insert_digest(struct safexcel_instr **instrp, int len)
1445 {
1446         struct safexcel_instr *instr;
1447
1448         instr = *instrp;
1449         instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT;
1450         instr->length = len;
1451         instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH |
1452             SAFEXCEL_INSTR_STATUS_LAST_PACKET;
1453         instr->instructions = SAFEXCEL_INSTR_DEST_OUTPUT |
1454             SAFEXCEL_INSTR_INSERT_HASH_DIGEST;
1455
1456         *instrp = instr + 1;
1457 }
1458
1459 /*
1460  * Retrieve and verify a digest.
1461  */
1462 static void
1463 safexcel_instr_retrieve_digest(struct safexcel_instr **instrp, int len)
1464 {
1465         struct safexcel_instr *instr;
1466
1467         instr = *instrp;
1468         instr->opcode = SAFEXCEL_INSTR_OPCODE_RETRIEVE;
1469         instr->length = len;
1470         instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH |
1471             SAFEXCEL_INSTR_STATUS_LAST_PACKET;
1472         instr->instructions = SAFEXCEL_INSTR_INSERT_HASH_DIGEST;
1473         instr++;
1474
1475         instr->opcode = SAFEXCEL_INSTR_OPCODE_VERIFY_FIELDS;
1476         instr->length = len | SAFEXCEL_INSTR_VERIFY_HASH;
1477         instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH |
1478             SAFEXCEL_INSTR_STATUS_LAST_PACKET;
1479         instr->instructions = SAFEXCEL_INSTR_VERIFY_PADDING;
1480
1481         *instrp = instr + 1;
1482 }
1483
1484 static void
1485 safexcel_instr_temp_aes_block(struct safexcel_instr **instrp)
1486 {
1487         struct safexcel_instr *instr;
1488
1489         instr = *instrp;
1490         instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT_REMOVE_RESULT;
1491         instr->length = 0;
1492         instr->status = 0;
1493         instr->instructions = AES_BLOCK_LEN;
1494         instr++;
1495
1496         instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT;
1497         instr->length = AES_BLOCK_LEN;
1498         instr->status = 0;
1499         instr->instructions = SAFEXCEL_INSTR_DEST_OUTPUT |
1500             SAFEXCEL_INSTR_DEST_CRYPTO;
1501
1502         *instrp = instr + 1;
1503 }
1504
1505 /*
1506  * Handle a request for an unauthenticated block cipher.
1507  */
1508 static void
1509 safexcel_instr_cipher(struct safexcel_request *req,
1510     struct safexcel_instr *instr, struct safexcel_cmd_descr *cdesc)
1511 {
1512         struct cryptop *crp;
1513
1514         crp = req->crp;
1515
1516         /* Insert the payload. */
1517         instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION;
1518         instr->length = crp->crp_payload_length;
1519         instr->status = SAFEXCEL_INSTR_STATUS_LAST_PACKET |
1520             SAFEXCEL_INSTR_STATUS_LAST_HASH;
1521         instr->instructions = SAFEXCEL_INSTR_INS_LAST |
1522             SAFEXCEL_INSTR_DEST_CRYPTO | SAFEXCEL_INSTR_DEST_OUTPUT;
1523
1524         cdesc->additional_cdata_size = 1;
1525 }
1526
1527 static void
1528 safexcel_instr_eta(struct safexcel_request *req, struct safexcel_instr *instr,
1529     struct safexcel_cmd_descr *cdesc)
1530 {
1531         const struct crypto_session_params *csp;
1532         struct cryptop *crp;
1533         struct safexcel_instr *start;
1534
1535         crp = req->crp;
1536         csp = crypto_get_params(crp->crp_session);
1537         start = instr;
1538
1539         /* Insert the AAD. */
1540         instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION;
1541         instr->length = crp->crp_aad_length;
1542         instr->status = crp->crp_payload_length == 0 ?
1543             SAFEXCEL_INSTR_STATUS_LAST_HASH : 0;
1544         instr->instructions = SAFEXCEL_INSTR_INS_LAST |
1545             SAFEXCEL_INSTR_DEST_HASH;
1546         instr++;
1547
1548         /* Encrypt any data left in the request. */
1549         if (crp->crp_payload_length > 0) {
1550                 instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION;
1551                 instr->length = crp->crp_payload_length;
1552                 instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH;
1553                 instr->instructions = SAFEXCEL_INSTR_INS_LAST |
1554                     SAFEXCEL_INSTR_DEST_CRYPTO |
1555                     SAFEXCEL_INSTR_DEST_HASH |
1556                     SAFEXCEL_INSTR_DEST_OUTPUT;
1557                 instr++;
1558         }
1559
1560         /*
1561          * Compute the digest, or extract it and place it in the output stream.
1562          */
1563         if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
1564                 safexcel_instr_insert_digest(&instr, req->sess->digestlen);
1565         else
1566                 safexcel_instr_retrieve_digest(&instr, req->sess->digestlen);
1567         cdesc->additional_cdata_size = instr - start;
1568 }
1569
1570 static void
1571 safexcel_instr_sha_hash(struct safexcel_request *req,
1572     struct safexcel_instr *instr)
1573 {
1574         struct cryptop *crp;
1575         struct safexcel_instr *start;
1576
1577         crp = req->crp;
1578         start = instr;
1579
1580         /* Pass the input data to the hash engine. */
1581         instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION;
1582         instr->length = crp->crp_payload_length;
1583         instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH;
1584         instr->instructions = SAFEXCEL_INSTR_DEST_HASH;
1585         instr++;
1586
1587         /* Insert the hash result into the output stream. */
1588         safexcel_instr_insert_digest(&instr, req->sess->digestlen);
1589
1590         /* Pad the rest of the inline instruction space. */
1591         while (instr != start + SAFEXCEL_MAX_ITOKENS)
1592                 safexcel_instr_nop(&instr);
1593 }
1594
1595 static void
1596 safexcel_instr_ccm(struct safexcel_request *req, struct safexcel_instr *instr,
1597     struct safexcel_cmd_descr *cdesc)
1598 {
1599         struct cryptop *crp;
1600         struct safexcel_instr *start;
1601         uint8_t *a0, *b0, *alenp, L;
1602         int aalign, blen;
1603
1604         crp = req->crp;
1605         start = instr;
1606
1607         /*
1608          * Construct two blocks, A0 and B0, used in encryption and
1609          * authentication, respectively.  A0 is embedded in the token
1610          * descriptor, and B0 is inserted directly into the data stream using
1611          * instructions below.
1612          *
1613          * OCF seems to assume a 12-byte IV, fixing L (the payload length size)
1614          * at 3 bytes due to the layout of B0.  This is fine since the driver
1615          * has a maximum of 65535 bytes anyway.
1616          */
1617         blen = AES_BLOCK_LEN;
1618         L = 3;
1619
1620         a0 = (uint8_t *)&cdesc->control_data.token[0];
1621         memset(a0, 0, blen);
1622         a0[0] = L - 1;
1623         memcpy(&a0[1], req->iv, AES_CCM_IV_LEN);
1624
1625         /*
1626          * Insert B0 and the AAD length into the input stream.
1627          */
1628         instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT;
1629         instr->length = blen + (crp->crp_aad_length > 0 ? 2 : 0);
1630         instr->status = 0;
1631         instr->instructions = SAFEXCEL_INSTR_DEST_HASH |
1632             SAFEXCEL_INSTR_INSERT_IMMEDIATE;
1633         instr++;
1634
1635         b0 = (uint8_t *)instr;
1636         memset(b0, 0, blen);
1637         b0[0] =
1638             (L - 1) | /* payload length size */
1639             ((CCM_CBC_MAX_DIGEST_LEN - 2) / 2) << 3 /* digest length */ |
1640             (crp->crp_aad_length > 0 ? 1 : 0) << 6 /* AAD present bit */;
1641         memcpy(&b0[1], req->iv, AES_CCM_IV_LEN);
1642         b0[14] = crp->crp_payload_length >> 8;
1643         b0[15] = crp->crp_payload_length & 0xff;
1644         instr += blen / sizeof(*instr);
1645
1646         /* Insert the AAD length and data into the input stream. */
1647         if (crp->crp_aad_length > 0) {
1648                 alenp = (uint8_t *)instr;
1649                 alenp[0] = crp->crp_aad_length >> 8;
1650                 alenp[1] = crp->crp_aad_length & 0xff;
1651                 alenp[2] = 0;
1652                 alenp[3] = 0;
1653                 instr++;
1654
1655                 instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION;
1656                 instr->length = crp->crp_aad_length;
1657                 instr->status = 0;
1658                 instr->instructions = SAFEXCEL_INSTR_DEST_HASH;
1659                 instr++;
1660
1661                 /* Insert zero padding. */
1662                 aalign = (crp->crp_aad_length + 2) & (blen - 1);
1663                 instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT;
1664                 instr->length = aalign == 0 ? 0 :
1665                     blen - ((crp->crp_aad_length + 2) & (blen - 1));
1666                 instr->status = crp->crp_payload_length == 0 ?
1667                     SAFEXCEL_INSTR_STATUS_LAST_HASH : 0;
1668                 instr->instructions = SAFEXCEL_INSTR_DEST_HASH;
1669                 instr++;
1670         }
1671
1672         safexcel_instr_temp_aes_block(&instr);
1673
1674         /* Insert the cipher payload into the input stream. */
1675         if (crp->crp_payload_length > 0) {
1676                 instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION;
1677                 instr->length = crp->crp_payload_length;
1678                 instr->status = (crp->crp_payload_length & (blen - 1)) == 0 ?
1679                     SAFEXCEL_INSTR_STATUS_LAST_HASH : 0;
1680                 instr->instructions = SAFEXCEL_INSTR_DEST_OUTPUT |
1681                     SAFEXCEL_INSTR_DEST_CRYPTO |
1682                     SAFEXCEL_INSTR_DEST_HASH |
1683                     SAFEXCEL_INSTR_INS_LAST;
1684                 instr++;
1685
1686                 /* Insert zero padding. */
1687                 if (crp->crp_payload_length & (blen - 1)) {
1688                         instr->opcode = SAFEXCEL_INSTR_OPCODE_INSERT;
1689                         instr->length = blen -
1690                             (crp->crp_payload_length & (blen - 1));
1691                         instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH;
1692                         instr->instructions = SAFEXCEL_INSTR_DEST_HASH;
1693                         instr++;
1694                 }
1695         }
1696
1697         /*
1698          * Compute the digest, or extract it and place it in the output stream.
1699          */
1700         if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
1701                 safexcel_instr_insert_digest(&instr, req->sess->digestlen);
1702         else
1703                 safexcel_instr_retrieve_digest(&instr, req->sess->digestlen);
1704
1705         cdesc->additional_cdata_size = instr - start;
1706 }
1707
1708 static void
1709 safexcel_instr_gcm(struct safexcel_request *req, struct safexcel_instr *instr,
1710     struct safexcel_cmd_descr *cdesc)
1711 {
1712         struct cryptop *crp;
1713         struct safexcel_instr *start;
1714
1715         memcpy(cdesc->control_data.token, req->iv, AES_GCM_IV_LEN);
1716         cdesc->control_data.token[3] = htobe32(1);
1717
1718         crp = req->crp;
1719         start = instr;
1720
1721         /* Insert the AAD into the input stream. */
1722         instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION;
1723         instr->length = crp->crp_aad_length;
1724         instr->status = crp->crp_payload_length == 0 ?
1725             SAFEXCEL_INSTR_STATUS_LAST_HASH : 0;
1726         instr->instructions = SAFEXCEL_INSTR_INS_LAST |
1727             SAFEXCEL_INSTR_DEST_HASH;
1728         instr++;
1729
1730         safexcel_instr_temp_aes_block(&instr);
1731
1732         /* Insert the cipher payload into the input stream. */
1733         if (crp->crp_payload_length > 0) {
1734                 instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION;
1735                 instr->length = crp->crp_payload_length;
1736                 instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH;
1737                 instr->instructions = SAFEXCEL_INSTR_DEST_OUTPUT |
1738                     SAFEXCEL_INSTR_DEST_CRYPTO | SAFEXCEL_INSTR_DEST_HASH |
1739                     SAFEXCEL_INSTR_INS_LAST;
1740                 instr++;
1741         }
1742
1743         /*
1744          * Compute the digest, or extract it and place it in the output stream.
1745          */
1746         if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
1747                 safexcel_instr_insert_digest(&instr, req->sess->digestlen);
1748         else
1749                 safexcel_instr_retrieve_digest(&instr, req->sess->digestlen);
1750
1751         cdesc->additional_cdata_size = instr - start;
1752 }
1753
1754 static void
1755 safexcel_instr_gmac(struct safexcel_request *req, struct safexcel_instr *instr,
1756     struct safexcel_cmd_descr *cdesc)
1757 {
1758         struct cryptop *crp;
1759         struct safexcel_instr *start;
1760
1761         memcpy(cdesc->control_data.token, req->iv, AES_GCM_IV_LEN);
1762         cdesc->control_data.token[3] = htobe32(1);
1763
1764         crp = req->crp;
1765         start = instr;
1766
1767         instr->opcode = SAFEXCEL_INSTR_OPCODE_DIRECTION;
1768         instr->length = crp->crp_payload_length;
1769         instr->status = SAFEXCEL_INSTR_STATUS_LAST_HASH;
1770         instr->instructions = SAFEXCEL_INSTR_INS_LAST |
1771             SAFEXCEL_INSTR_DEST_HASH;
1772         instr++;
1773
1774         safexcel_instr_temp_aes_block(&instr);
1775
1776         safexcel_instr_insert_digest(&instr, req->sess->digestlen);
1777
1778         cdesc->additional_cdata_size = instr - start;
1779 }
1780
1781 static void
1782 safexcel_set_token(struct safexcel_request *req)
1783 {
1784         const struct crypto_session_params *csp;
1785         struct safexcel_cmd_descr *cdesc;
1786         struct safexcel_instr *instr;
1787         struct safexcel_softc *sc;
1788         int ringidx;
1789
1790         csp = crypto_get_params(req->crp->crp_session);
1791         cdesc = req->cdesc;
1792         sc = req->sc;
1793         ringidx = req->sess->ringidx;
1794
1795         safexcel_set_command(req, cdesc);
1796
1797         /*
1798          * For keyless hash operations, the token instructions can be embedded
1799          * in the token itself.  Otherwise we use an additional token descriptor
1800          * and the embedded instruction space is used to store the IV.
1801          */
1802         if (csp->csp_cipher_alg == 0 &&
1803             csp->csp_auth_alg != CRYPTO_AES_NIST_GMAC) {
1804                 instr = (void *)cdesc->control_data.token;
1805         } else {
1806                 instr = (void *)(sc->sc_ring[ringidx].dma_atok.vaddr +
1807                     sc->sc_config.atok_offset *
1808                     (cdesc - sc->sc_ring[ringidx].cdr.desc));
1809                 cdesc->control_data.options |= SAFEXCEL_OPTION_4_TOKEN_IV_CMD;
1810         }
1811
1812         switch (csp->csp_cipher_alg) {
1813         case CRYPTO_AES_NIST_GCM_16:
1814                 safexcel_instr_gcm(req, instr, cdesc);
1815                 break;
1816         case CRYPTO_AES_CCM_16:
1817                 safexcel_instr_ccm(req, instr, cdesc);
1818                 break;
1819         case CRYPTO_AES_XTS:
1820                 memcpy(cdesc->control_data.token, req->iv, AES_XTS_IV_LEN);
1821                 memset(cdesc->control_data.token +
1822                     AES_XTS_IV_LEN / sizeof(uint32_t), 0, AES_XTS_IV_LEN);
1823
1824                 safexcel_instr_cipher(req, instr, cdesc);
1825                 break;
1826         case CRYPTO_AES_CBC:
1827         case CRYPTO_AES_ICM:
1828                 memcpy(cdesc->control_data.token, req->iv, AES_BLOCK_LEN);
1829                 if (csp->csp_auth_alg != 0)
1830                         safexcel_instr_eta(req, instr, cdesc);
1831                 else
1832                         safexcel_instr_cipher(req, instr, cdesc);
1833                 break;
1834         default:
1835                 switch (csp->csp_auth_alg) {
1836                 case CRYPTO_SHA1:
1837                 case CRYPTO_SHA1_HMAC:
1838                 case CRYPTO_SHA2_224:
1839                 case CRYPTO_SHA2_224_HMAC:
1840                 case CRYPTO_SHA2_256:
1841                 case CRYPTO_SHA2_256_HMAC:
1842                 case CRYPTO_SHA2_384:
1843                 case CRYPTO_SHA2_384_HMAC:
1844                 case CRYPTO_SHA2_512:
1845                 case CRYPTO_SHA2_512_HMAC:
1846                         safexcel_instr_sha_hash(req, instr);
1847                         break;
1848                 case CRYPTO_AES_NIST_GMAC:
1849                         safexcel_instr_gmac(req, instr, cdesc);
1850                         break;
1851                 default:
1852                         panic("unhandled auth request %d", csp->csp_auth_alg);
1853                 }
1854                 break;
1855         }
1856 }
1857
1858 static struct safexcel_res_descr *
1859 safexcel_res_descr_add(struct safexcel_ring *ring, bool first, bool last,
1860     bus_addr_t data, uint32_t len)
1861 {
1862         struct safexcel_res_descr *rdesc;
1863         struct safexcel_res_descr_ring *rring;
1864
1865         mtx_assert(&ring->mtx, MA_OWNED);
1866
1867         rring = &ring->rdr;
1868         if ((rring->write + 1) % SAFEXCEL_RING_SIZE == rring->read)
1869                 return (NULL);
1870
1871         rdesc = &rring->desc[rring->write];
1872         rring->write = (rring->write + 1) % SAFEXCEL_RING_SIZE;
1873
1874         rdesc->particle_size = len;
1875         rdesc->rsvd0 = 0;
1876         rdesc->descriptor_overflow = 0;
1877         rdesc->buffer_overflow = 0;
1878         rdesc->last_seg = last;
1879         rdesc->first_seg = first;
1880         rdesc->result_size =
1881             sizeof(struct safexcel_res_data) / sizeof(uint32_t);
1882         rdesc->rsvd1 = 0;
1883         rdesc->data_lo = SAFEXCEL_ADDR_LO(data);
1884         rdesc->data_hi = SAFEXCEL_ADDR_HI(data);
1885
1886         if (first) {
1887                 rdesc->result_data.packet_length = 0;
1888                 rdesc->result_data.error_code = 0;
1889         }
1890
1891         return (rdesc);
1892 }
1893
1894 static struct safexcel_cmd_descr *
1895 safexcel_cmd_descr_add(struct safexcel_ring *ring, bool first, bool last,
1896     bus_addr_t data, uint32_t seglen, uint32_t reqlen, bus_addr_t context)
1897 {
1898         struct safexcel_cmd_descr *cdesc;
1899         struct safexcel_cmd_descr_ring *cring;
1900
1901         KASSERT(reqlen <= SAFEXCEL_MAX_REQUEST_SIZE,
1902             ("%s: request length %u too long", __func__, reqlen));
1903         mtx_assert(&ring->mtx, MA_OWNED);
1904
1905         cring = &ring->cdr;
1906         if ((cring->write + 1) % SAFEXCEL_RING_SIZE == cring->read)
1907                 return (NULL);
1908
1909         cdesc = &cring->desc[cring->write];
1910         cring->write = (cring->write + 1) % SAFEXCEL_RING_SIZE;
1911
1912         cdesc->particle_size = seglen;
1913         cdesc->rsvd0 = 0;
1914         cdesc->last_seg = last;
1915         cdesc->first_seg = first;
1916         cdesc->additional_cdata_size = 0;
1917         cdesc->rsvd1 = 0;
1918         cdesc->data_lo = SAFEXCEL_ADDR_LO(data);
1919         cdesc->data_hi = SAFEXCEL_ADDR_HI(data);
1920         if (first) {
1921                 cdesc->control_data.packet_length = reqlen;
1922                 cdesc->control_data.options = SAFEXCEL_OPTION_IP |
1923                     SAFEXCEL_OPTION_CP | SAFEXCEL_OPTION_CTX_CTRL_IN_CMD |
1924                     SAFEXCEL_OPTION_RC_AUTO;
1925                 cdesc->control_data.type = SAFEXCEL_TOKEN_TYPE_BYPASS;
1926                 cdesc->control_data.context_lo = SAFEXCEL_ADDR_LO(context) |
1927                     SAFEXCEL_CONTEXT_SMALL;
1928                 cdesc->control_data.context_hi = SAFEXCEL_ADDR_HI(context);
1929         }
1930
1931         return (cdesc);
1932 }
1933
1934 static void
1935 safexcel_cmd_descr_rollback(struct safexcel_ring *ring, int count)
1936 {
1937         struct safexcel_cmd_descr_ring *cring;
1938
1939         mtx_assert(&ring->mtx, MA_OWNED);
1940
1941         cring = &ring->cdr;
1942         cring->write -= count;
1943         if (cring->write < 0)
1944                 cring->write += SAFEXCEL_RING_SIZE;
1945 }
1946
1947 static void
1948 safexcel_res_descr_rollback(struct safexcel_ring *ring, int count)
1949 {
1950         struct safexcel_res_descr_ring *rring;
1951
1952         mtx_assert(&ring->mtx, MA_OWNED);
1953
1954         rring = &ring->rdr;
1955         rring->write -= count;
1956         if (rring->write < 0)
1957                 rring->write += SAFEXCEL_RING_SIZE;
1958 }
1959
1960 static void
1961 safexcel_append_segs(bus_dma_segment_t *segs, int nseg, struct sglist *sg,
1962     int start, int len)
1963 {
1964         bus_dma_segment_t *seg;
1965         size_t seglen;
1966         int error, i;
1967
1968         for (i = 0; i < nseg && len > 0; i++) {
1969                 seg = &segs[i];
1970
1971                 if (seg->ds_len <= start) {
1972                         start -= seg->ds_len;
1973                         continue;
1974                 }
1975
1976                 seglen = MIN(len, seg->ds_len - start);
1977                 error = sglist_append_phys(sg, seg->ds_addr + start, seglen);
1978                 if (error != 0)
1979                         panic("%s: ran out of segments: %d", __func__, error);
1980                 len -= seglen;
1981                 start = 0;
1982         }
1983 }
1984
1985 static void
1986 safexcel_create_chain_cb(void *arg, bus_dma_segment_t *segs, int nseg,
1987     int error)
1988 {
1989         const struct crypto_session_params *csp;
1990         struct cryptop *crp;
1991         struct safexcel_cmd_descr *cdesc;
1992         struct safexcel_request *req;
1993         struct safexcel_ring *ring;
1994         struct safexcel_session *sess;
1995         struct sglist *sg;
1996         size_t inlen;
1997         int i;
1998         bool first, last;
1999
2000         req = arg;
2001         if (error != 0) {
2002                 req->error = error;
2003                 return;
2004         }
2005
2006         crp = req->crp;
2007         csp = crypto_get_params(crp->crp_session);
2008         sess = req->sess;
2009         ring = &req->sc->sc_ring[sess->ringidx];
2010
2011         mtx_assert(&ring->mtx, MA_OWNED);
2012
2013         /*
2014          * Set up descriptors for input and output data.
2015          *
2016          * The processing engine programs require that any AAD comes first,
2017          * followed by the cipher plaintext, followed by the digest.  Some
2018          * consumers place the digest first in the input buffer, in which case
2019          * we have to create an extra descriptor.
2020          *
2021          * As an optimization, unmodified data is not passed to the output
2022          * stream.
2023          */
2024         sglist_reset(ring->cmd_data);
2025         sglist_reset(ring->res_data);
2026         if (crp->crp_aad_length != 0) {
2027                 safexcel_append_segs(segs, nseg, ring->cmd_data,
2028                     crp->crp_aad_start, crp->crp_aad_length);
2029         }
2030         safexcel_append_segs(segs, nseg, ring->cmd_data,
2031             crp->crp_payload_start, crp->crp_payload_length);
2032         if (csp->csp_cipher_alg != 0) {
2033                 safexcel_append_segs(segs, nseg, ring->res_data,
2034                     crp->crp_payload_start, crp->crp_payload_length);
2035         }
2036         if (sess->digestlen > 0) {
2037                 if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) != 0) {
2038                         safexcel_append_segs(segs, nseg, ring->cmd_data,
2039                             crp->crp_digest_start, sess->digestlen);
2040                 } else {
2041                         safexcel_append_segs(segs, nseg, ring->res_data,
2042                             crp->crp_digest_start, sess->digestlen);
2043                 }
2044         }
2045
2046         sg = ring->cmd_data;
2047         if (sg->sg_nseg == 0) {
2048                 /*
2049                  * Fake a segment for the command descriptor if the input has
2050                  * length zero.  The EIP97 apparently does not handle
2051                  * zero-length packets properly since subsequent requests return
2052                  * bogus errors, so provide a dummy segment using the context
2053                  * descriptor.
2054                  */
2055                 (void)sglist_append_phys(sg, req->ctx.paddr, 1);
2056         }
2057         for (i = 0, inlen = 0; i < sg->sg_nseg; i++)
2058                 inlen += sg->sg_segs[i].ss_len;
2059         for (i = 0; i < sg->sg_nseg; i++) {
2060                 first = i == 0;
2061                 last = i == sg->sg_nseg - 1;
2062
2063                 cdesc = safexcel_cmd_descr_add(ring, first, last,
2064                     sg->sg_segs[i].ss_paddr, sg->sg_segs[i].ss_len,
2065                     (uint32_t)inlen, req->ctx.paddr);
2066                 if (cdesc == NULL) {
2067                         safexcel_cmd_descr_rollback(ring, i);
2068                         req->error = EAGAIN;
2069                         return;
2070                 }
2071                 if (i == 0)
2072                         req->cdesc = cdesc;
2073         }
2074         req->cdescs = sg->sg_nseg;
2075
2076         sg = ring->res_data;
2077         if (sg->sg_nseg == 0) {
2078                 /*
2079                  * We need a result descriptor even if the output stream will be
2080                  * empty, for example when verifying an AAD digest.
2081                  */
2082                 sg->sg_segs[0].ss_paddr = 0;
2083                 sg->sg_segs[0].ss_len = 0;
2084                 sg->sg_nseg = 1;
2085         }
2086         for (i = 0; i < sg->sg_nseg; i++) {
2087                 first = i == 0;
2088                 last = i == sg->sg_nseg - 1;
2089
2090                 if (safexcel_res_descr_add(ring, first, last,
2091                     sg->sg_segs[i].ss_paddr, sg->sg_segs[i].ss_len) == NULL) {
2092                         safexcel_cmd_descr_rollback(ring,
2093                             ring->cmd_data->sg_nseg);
2094                         safexcel_res_descr_rollback(ring, i);
2095                         req->error = EAGAIN;
2096                         return;
2097                 }
2098         }
2099         req->rdescs = sg->sg_nseg;
2100 }
2101
2102 static int
2103 safexcel_create_chain(struct safexcel_ring *ring, struct safexcel_request *req)
2104 {
2105         int error;
2106
2107         req->error = 0;
2108         req->cdescs = req->rdescs = 0;
2109
2110         error = bus_dmamap_load_crp(ring->data_dtag, req->dmap, req->crp,
2111             safexcel_create_chain_cb, req, BUS_DMA_NOWAIT);
2112         if (error == 0)
2113                 req->dmap_loaded = true;
2114
2115         if (req->error != 0)
2116                 error = req->error;
2117
2118         return (error);
2119 }
2120
2121 static bool
2122 safexcel_probe_cipher(const struct crypto_session_params *csp)
2123 {
2124         switch (csp->csp_cipher_alg) {
2125         case CRYPTO_AES_CBC:
2126         case CRYPTO_AES_ICM:
2127                 if (csp->csp_ivlen != AES_BLOCK_LEN)
2128                         return (false);
2129                 break;
2130         case CRYPTO_AES_XTS:
2131                 if (csp->csp_ivlen != AES_XTS_IV_LEN)
2132                         return (false);
2133                 break;
2134         default:
2135                 return (false);
2136         }
2137
2138         return (true);
2139 }
2140
2141 /*
2142  * Determine whether the driver can implement a session with the requested
2143  * parameters.
2144  */
2145 static int
2146 safexcel_probesession(device_t dev, const struct crypto_session_params *csp)
2147 {
2148         switch (csp->csp_mode) {
2149         case CSP_MODE_CIPHER:
2150                 if (!safexcel_probe_cipher(csp))
2151                         return (EINVAL);
2152                 break;
2153         case CSP_MODE_DIGEST:
2154                 switch (csp->csp_auth_alg) {
2155                 case CRYPTO_AES_NIST_GMAC:
2156                         if (csp->csp_ivlen != AES_GCM_IV_LEN)
2157                                 return (EINVAL);
2158                         break;
2159                 case CRYPTO_SHA1:
2160                 case CRYPTO_SHA1_HMAC:
2161                 case CRYPTO_SHA2_224:
2162                 case CRYPTO_SHA2_224_HMAC:
2163                 case CRYPTO_SHA2_256:
2164                 case CRYPTO_SHA2_256_HMAC:
2165                 case CRYPTO_SHA2_384:
2166                 case CRYPTO_SHA2_384_HMAC:
2167                 case CRYPTO_SHA2_512:
2168                 case CRYPTO_SHA2_512_HMAC:
2169                         break;
2170                 default:
2171                         return (EINVAL);
2172                 }
2173                 break;
2174         case CSP_MODE_AEAD:
2175                 switch (csp->csp_cipher_alg) {
2176                 case CRYPTO_AES_NIST_GCM_16:
2177                         if (csp->csp_ivlen != AES_GCM_IV_LEN)
2178                                 return (EINVAL);
2179                         break;
2180                 case CRYPTO_AES_CCM_16:
2181                         if (csp->csp_ivlen != AES_CCM_IV_LEN)
2182                                 return (EINVAL);
2183                         break;
2184                 default:
2185                         return (EINVAL);
2186                 }
2187                 break;
2188         case CSP_MODE_ETA:
2189                 if (!safexcel_probe_cipher(csp))
2190                         return (EINVAL);
2191                 switch (csp->csp_cipher_alg) {
2192                 case CRYPTO_AES_CBC:
2193                 case CRYPTO_AES_ICM:
2194                         /*
2195                          * The EIP-97 does not support combining AES-XTS with
2196                          * hash operations.
2197                          */
2198                         if (csp->csp_auth_alg != CRYPTO_SHA1_HMAC &&
2199                             csp->csp_auth_alg != CRYPTO_SHA2_224_HMAC &&
2200                             csp->csp_auth_alg != CRYPTO_SHA2_256_HMAC &&
2201                             csp->csp_auth_alg != CRYPTO_SHA2_384_HMAC &&
2202                             csp->csp_auth_alg != CRYPTO_SHA2_512_HMAC)
2203                                 return (EINVAL);
2204                         break;
2205                 default:
2206                         return (EINVAL);
2207                 }
2208                 break;
2209         default:
2210                 return (EINVAL);
2211         }
2212
2213         return (CRYPTODEV_PROBE_HARDWARE);
2214 }
2215
2216 /*
2217  * Pre-compute the hash key used in GHASH, which is a block of zeroes encrypted
2218  * using the cipher key.
2219  */
2220 static void
2221 safexcel_setkey_ghash(struct safexcel_session *sess, const uint8_t *key,
2222     int klen)
2223 {
2224         uint32_t ks[4 * (RIJNDAEL_MAXNR + 1)];
2225         uint8_t zeros[AES_BLOCK_LEN];
2226         int i, rounds;
2227
2228         memset(zeros, 0, sizeof(zeros));
2229
2230         rounds = rijndaelKeySetupEnc(ks, key, klen * NBBY);
2231         rijndaelEncrypt(ks, rounds, zeros, (uint8_t *)sess->ghash_key);
2232         for (i = 0; i < GMAC_BLOCK_LEN / sizeof(uint32_t); i++)
2233                 sess->ghash_key[i] = htobe32(sess->ghash_key[i]);
2234
2235         explicit_bzero(ks, sizeof(ks));
2236 }
2237
2238 /*
2239  * Pre-compute the combined CBC-MAC key, which consists of three keys K1, K2, K3
2240  * in the hardware implementation.  K1 is the cipher key and comes last in the
2241  * buffer since K2 and K3 have a fixed size of AES_BLOCK_LEN.  For now XCBC-MAC
2242  * is not implemented so K2 and K3 are fixed.
2243  */
2244 static void
2245 safexcel_setkey_xcbcmac(struct safexcel_session *sess, const uint8_t *key,
2246     int klen)
2247 {
2248         int i, off;
2249
2250         memset(sess->xcbc_key, 0, sizeof(sess->xcbc_key));
2251         off = 2 * AES_BLOCK_LEN / sizeof(uint32_t);
2252         for (i = 0; i < klen / sizeof(uint32_t); i++, key += 4)
2253                 sess->xcbc_key[i + off] = htobe32(le32dec(key));
2254 }
2255
2256 static void
2257 safexcel_setkey_hmac_digest(struct auth_hash *ahash, union authctx *ctx,
2258     char *buf)
2259 {
2260         int hashwords, i;
2261
2262         switch (ahash->type) {
2263         case CRYPTO_SHA1_HMAC:
2264                 hashwords = ahash->hashsize / sizeof(uint32_t);
2265                 for (i = 0; i < hashwords; i++)
2266                         ((uint32_t *)buf)[i] = htobe32(ctx->sha1ctx.h.b32[i]);
2267                 break;
2268         case CRYPTO_SHA2_224_HMAC:
2269                 hashwords = auth_hash_hmac_sha2_256.hashsize / sizeof(uint32_t);
2270                 for (i = 0; i < hashwords; i++)
2271                         ((uint32_t *)buf)[i] = htobe32(ctx->sha224ctx.state[i]);
2272                 break;
2273         case CRYPTO_SHA2_256_HMAC:
2274                 hashwords = ahash->hashsize / sizeof(uint32_t);
2275                 for (i = 0; i < hashwords; i++)
2276                         ((uint32_t *)buf)[i] = htobe32(ctx->sha256ctx.state[i]);
2277                 break;
2278         case CRYPTO_SHA2_384_HMAC:
2279                 hashwords = auth_hash_hmac_sha2_512.hashsize / sizeof(uint64_t);
2280                 for (i = 0; i < hashwords; i++)
2281                         ((uint64_t *)buf)[i] = htobe64(ctx->sha384ctx.state[i]);
2282                 break;
2283         case CRYPTO_SHA2_512_HMAC:
2284                 hashwords = ahash->hashsize / sizeof(uint64_t);
2285                 for (i = 0; i < hashwords; i++)
2286                         ((uint64_t *)buf)[i] = htobe64(ctx->sha512ctx.state[i]);
2287                 break;
2288         }
2289 }
2290
2291 /*
2292  * Pre-compute the inner and outer digests used in the HMAC algorithm.
2293  */
2294 static void
2295 safexcel_setkey_hmac(const struct crypto_session_params *csp,
2296     struct safexcel_session *sess, const uint8_t *key, int klen)
2297 {
2298         union authctx ctx;
2299         struct auth_hash *ahash;
2300
2301         ahash = crypto_auth_hash(csp);
2302         hmac_init_ipad(ahash, key, klen, &ctx);
2303         safexcel_setkey_hmac_digest(ahash, &ctx, sess->hmac_ipad);
2304         hmac_init_opad(ahash, key, klen, &ctx);
2305         safexcel_setkey_hmac_digest(ahash, &ctx, sess->hmac_opad);
2306         explicit_bzero(&ctx, ahash->ctxsize);
2307 }
2308
2309 static void
2310 safexcel_setkey_xts(struct safexcel_session *sess, const uint8_t *key, int klen)
2311 {
2312         memcpy(sess->tweak_key, key + klen / 2, klen / 2);
2313 }
2314
2315 static void
2316 safexcel_setkey(struct safexcel_session *sess,
2317     const struct crypto_session_params *csp, struct cryptop *crp)
2318 {
2319         const uint8_t *akey, *ckey;
2320         int aklen, cklen;
2321
2322         aklen = csp->csp_auth_klen;
2323         cklen = csp->csp_cipher_klen;
2324         akey = ckey = NULL;
2325         if (crp != NULL) {
2326                 akey = crp->crp_auth_key;
2327                 ckey = crp->crp_cipher_key;
2328         }
2329         if (akey == NULL)
2330                 akey = csp->csp_auth_key;
2331         if (ckey == NULL)
2332                 ckey = csp->csp_cipher_key;
2333
2334         sess->klen = cklen;
2335         switch (csp->csp_cipher_alg) {
2336         case CRYPTO_AES_NIST_GCM_16:
2337                 safexcel_setkey_ghash(sess, ckey, cklen);
2338                 break;
2339         case CRYPTO_AES_CCM_16:
2340                 safexcel_setkey_xcbcmac(sess, ckey, cklen);
2341                 break;
2342         case CRYPTO_AES_XTS:
2343                 safexcel_setkey_xts(sess, ckey, cklen);
2344                 sess->klen /= 2;
2345                 break;
2346         }
2347
2348         switch (csp->csp_auth_alg) {
2349         case CRYPTO_SHA1_HMAC:
2350         case CRYPTO_SHA2_224_HMAC:
2351         case CRYPTO_SHA2_256_HMAC:
2352         case CRYPTO_SHA2_384_HMAC:
2353         case CRYPTO_SHA2_512_HMAC:
2354                 safexcel_setkey_hmac(csp, sess, akey, aklen);
2355                 break;
2356         case CRYPTO_AES_NIST_GMAC:
2357                 sess->klen = aklen;
2358                 safexcel_setkey_ghash(sess, akey, aklen);
2359                 break;
2360         }
2361 }
2362
2363 static uint32_t
2364 safexcel_aes_algid(int keylen)
2365 {
2366         switch (keylen) {
2367         case 16:
2368                 return (SAFEXCEL_CONTROL0_CRYPTO_ALG_AES128);
2369         case 24:
2370                 return (SAFEXCEL_CONTROL0_CRYPTO_ALG_AES192);
2371         case 32:
2372                 return (SAFEXCEL_CONTROL0_CRYPTO_ALG_AES256);
2373         default:
2374                 panic("invalid AES key length %d", keylen);
2375         }
2376 }
2377
2378 static uint32_t
2379 safexcel_aes_ccm_hashid(int keylen)
2380 {
2381         switch (keylen) {
2382         case 16:
2383                 return (SAFEXCEL_CONTROL0_HASH_ALG_XCBC128);
2384         case 24:
2385                 return (SAFEXCEL_CONTROL0_HASH_ALG_XCBC192);
2386         case 32:
2387                 return (SAFEXCEL_CONTROL0_HASH_ALG_XCBC256);
2388         default:
2389                 panic("invalid AES key length %d", keylen);
2390         }
2391 }
2392
2393 static uint32_t
2394 safexcel_sha_hashid(int alg)
2395 {
2396         switch (alg) {
2397         case CRYPTO_SHA1:
2398         case CRYPTO_SHA1_HMAC:
2399                 return (SAFEXCEL_CONTROL0_HASH_ALG_SHA1);
2400         case CRYPTO_SHA2_224:
2401         case CRYPTO_SHA2_224_HMAC:
2402                 return (SAFEXCEL_CONTROL0_HASH_ALG_SHA224);
2403         case CRYPTO_SHA2_256:
2404         case CRYPTO_SHA2_256_HMAC:
2405                 return (SAFEXCEL_CONTROL0_HASH_ALG_SHA256);
2406         case CRYPTO_SHA2_384:
2407         case CRYPTO_SHA2_384_HMAC:
2408                 return (SAFEXCEL_CONTROL0_HASH_ALG_SHA384);
2409         case CRYPTO_SHA2_512:
2410         case CRYPTO_SHA2_512_HMAC:
2411                 return (SAFEXCEL_CONTROL0_HASH_ALG_SHA512);
2412         default:
2413                 __assert_unreachable();
2414         }
2415 }
2416
2417 static int
2418 safexcel_sha_hashlen(int alg)
2419 {
2420         switch (alg) {
2421         case CRYPTO_SHA1:
2422         case CRYPTO_SHA1_HMAC:
2423                 return (SHA1_HASH_LEN);
2424         case CRYPTO_SHA2_224:
2425         case CRYPTO_SHA2_224_HMAC:
2426                 return (SHA2_224_HASH_LEN);
2427         case CRYPTO_SHA2_256:
2428         case CRYPTO_SHA2_256_HMAC:
2429                 return (SHA2_256_HASH_LEN);
2430         case CRYPTO_SHA2_384:
2431         case CRYPTO_SHA2_384_HMAC:
2432                 return (SHA2_384_HASH_LEN);
2433         case CRYPTO_SHA2_512:
2434         case CRYPTO_SHA2_512_HMAC:
2435                 return (SHA2_512_HASH_LEN);
2436         default:
2437                 __assert_unreachable();
2438         }
2439 }
2440
2441 static int
2442 safexcel_sha_statelen(int alg)
2443 {
2444         switch (alg) {
2445         case CRYPTO_SHA1:
2446         case CRYPTO_SHA1_HMAC:
2447                 return (SHA1_HASH_LEN);
2448         case CRYPTO_SHA2_224:
2449         case CRYPTO_SHA2_224_HMAC:
2450         case CRYPTO_SHA2_256:
2451         case CRYPTO_SHA2_256_HMAC:
2452                 return (SHA2_256_HASH_LEN);
2453         case CRYPTO_SHA2_384:
2454         case CRYPTO_SHA2_384_HMAC:
2455         case CRYPTO_SHA2_512:
2456         case CRYPTO_SHA2_512_HMAC:
2457                 return (SHA2_512_HASH_LEN);
2458         default:
2459                 __assert_unreachable();
2460         }
2461 }
2462
2463 static int
2464 safexcel_newsession(device_t dev, crypto_session_t cses,
2465     const struct crypto_session_params *csp)
2466 {
2467         struct safexcel_session *sess;
2468         struct safexcel_softc *sc;
2469
2470         sc = device_get_softc(dev);
2471         sess = crypto_get_driver_session(cses);
2472
2473         switch (csp->csp_auth_alg) {
2474         case CRYPTO_SHA1:
2475         case CRYPTO_SHA2_224:
2476         case CRYPTO_SHA2_256:
2477         case CRYPTO_SHA2_384:
2478         case CRYPTO_SHA2_512:
2479                 sess->digest = SAFEXCEL_CONTROL0_DIGEST_PRECOMPUTED;
2480                 sess->hash = safexcel_sha_hashid(csp->csp_auth_alg);
2481                 sess->digestlen = safexcel_sha_hashlen(csp->csp_auth_alg);
2482                 sess->statelen = safexcel_sha_statelen(csp->csp_auth_alg);
2483                 break;
2484         case CRYPTO_SHA1_HMAC:
2485         case CRYPTO_SHA2_224_HMAC:
2486         case CRYPTO_SHA2_256_HMAC:
2487         case CRYPTO_SHA2_384_HMAC:
2488         case CRYPTO_SHA2_512_HMAC:
2489                 sess->digest = SAFEXCEL_CONTROL0_DIGEST_HMAC;
2490                 sess->hash = safexcel_sha_hashid(csp->csp_auth_alg);
2491                 sess->digestlen = safexcel_sha_hashlen(csp->csp_auth_alg);
2492                 sess->statelen = safexcel_sha_statelen(csp->csp_auth_alg);
2493                 break;
2494         case CRYPTO_AES_NIST_GMAC:
2495                 sess->digest = SAFEXCEL_CONTROL0_DIGEST_GMAC;
2496                 sess->digestlen = GMAC_DIGEST_LEN;
2497                 sess->hash = SAFEXCEL_CONTROL0_HASH_ALG_GHASH;
2498                 sess->alg = safexcel_aes_algid(csp->csp_auth_klen);
2499                 sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_GCM;
2500                 break;
2501         }
2502
2503         switch (csp->csp_cipher_alg) {
2504         case CRYPTO_AES_NIST_GCM_16:
2505                 sess->digest = SAFEXCEL_CONTROL0_DIGEST_GMAC;
2506                 sess->digestlen = GMAC_DIGEST_LEN;
2507                 sess->hash = SAFEXCEL_CONTROL0_HASH_ALG_GHASH;
2508                 sess->alg = safexcel_aes_algid(csp->csp_cipher_klen);
2509                 sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_GCM;
2510                 break;
2511         case CRYPTO_AES_CCM_16:
2512                 sess->hash = safexcel_aes_ccm_hashid(csp->csp_cipher_klen);
2513                 sess->digest = SAFEXCEL_CONTROL0_DIGEST_CCM;
2514                 sess->digestlen = CCM_CBC_MAX_DIGEST_LEN;
2515                 sess->alg = safexcel_aes_algid(csp->csp_cipher_klen);
2516                 sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_CCM;
2517                 break;
2518         case CRYPTO_AES_CBC:
2519                 sess->alg = safexcel_aes_algid(csp->csp_cipher_klen);
2520                 sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_CBC;
2521                 break;
2522         case CRYPTO_AES_ICM:
2523                 sess->alg = safexcel_aes_algid(csp->csp_cipher_klen);
2524                 sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_CTR;
2525                 break;
2526         case CRYPTO_AES_XTS:
2527                 sess->alg = safexcel_aes_algid(csp->csp_cipher_klen / 2);
2528                 sess->mode = SAFEXCEL_CONTROL1_CRYPTO_MODE_XTS;
2529                 break;
2530         }
2531
2532         if (csp->csp_auth_mlen != 0)
2533                 sess->digestlen = csp->csp_auth_mlen;
2534
2535         safexcel_setkey(sess, csp, NULL);
2536
2537         /* Bind each session to a fixed ring to minimize lock contention. */
2538         sess->ringidx = atomic_fetchadd_int(&sc->sc_ringidx, 1);
2539         sess->ringidx %= sc->sc_config.rings;
2540
2541         return (0);
2542 }
2543
2544 static int
2545 safexcel_process(device_t dev, struct cryptop *crp, int hint)
2546 {
2547         const struct crypto_session_params *csp;
2548         struct safexcel_request *req;
2549         struct safexcel_ring *ring;
2550         struct safexcel_session *sess;
2551         struct safexcel_softc *sc;
2552         int error;
2553
2554         sc = device_get_softc(dev);
2555         sess = crypto_get_driver_session(crp->crp_session);
2556         csp = crypto_get_params(crp->crp_session);
2557
2558         if (__predict_false(crypto_buffer_len(&crp->crp_buf) >
2559             SAFEXCEL_MAX_REQUEST_SIZE)) {
2560                 crp->crp_etype = E2BIG;
2561                 crypto_done(crp);
2562                 return (0);
2563         }
2564
2565         if (crp->crp_cipher_key != NULL || crp->crp_auth_key != NULL)
2566                 safexcel_setkey(sess, csp, crp);
2567
2568         ring = &sc->sc_ring[sess->ringidx];
2569         mtx_lock(&ring->mtx);
2570         req = safexcel_alloc_request(sc, ring);
2571         if (__predict_false(req == NULL)) {
2572                 mtx_lock(&sc->sc_mtx);
2573                 mtx_unlock(&ring->mtx);
2574                 sc->sc_blocked = CRYPTO_SYMQ;
2575                 mtx_unlock(&sc->sc_mtx);
2576                 return (ERESTART);
2577         }
2578
2579         req->crp = crp;
2580         req->sess = sess;
2581
2582         crypto_read_iv(crp, req->iv);
2583
2584         error = safexcel_create_chain(ring, req);
2585         if (__predict_false(error != 0)) {
2586                 safexcel_free_request(ring, req);
2587                 mtx_unlock(&ring->mtx);
2588                 crp->crp_etype = error;
2589                 crypto_done(crp);
2590                 return (0);
2591         }
2592
2593         safexcel_set_token(req);
2594
2595         bus_dmamap_sync(ring->data_dtag, req->dmap,
2596             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2597         bus_dmamap_sync(req->ctx.tag, req->ctx.map,
2598             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2599         bus_dmamap_sync(ring->cdr.dma.tag, ring->cdr.dma.map,
2600             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2601         bus_dmamap_sync(ring->dma_atok.tag, ring->dma_atok.map,
2602             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2603         bus_dmamap_sync(ring->rdr.dma.tag, ring->rdr.dma.map,
2604             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2605
2606         safexcel_enqueue_request(sc, ring, req);
2607
2608         if ((hint & CRYPTO_HINT_MORE) == 0)
2609                 safexcel_execute(sc, ring, req);
2610         mtx_unlock(&ring->mtx);
2611
2612         return (0);
2613 }
2614
2615 static device_method_t safexcel_methods[] = {
2616         /* Device interface */
2617         DEVMETHOD(device_probe,         safexcel_probe),
2618         DEVMETHOD(device_attach,        safexcel_attach),
2619         DEVMETHOD(device_detach,        safexcel_detach),
2620
2621         /* Cryptodev interface */
2622         DEVMETHOD(cryptodev_probesession, safexcel_probesession),
2623         DEVMETHOD(cryptodev_newsession, safexcel_newsession),
2624         DEVMETHOD(cryptodev_process,    safexcel_process),
2625
2626         DEVMETHOD_END
2627 };
2628
2629 static devclass_t safexcel_devclass;
2630
2631 static driver_t safexcel_driver = {
2632         .name           = "safexcel",
2633         .methods        = safexcel_methods,
2634         .size           = sizeof(struct safexcel_softc),
2635 };
2636
2637 DRIVER_MODULE(safexcel, simplebus, safexcel_driver, safexcel_devclass, 0, 0);
2638 MODULE_VERSION(safexcel, 1);
2639 MODULE_DEPEND(safexcel, crypto, 1, 1, 1);