2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (C) 2009-2011 Semihalf.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * CESA SRAM Memory Map:
32 * +------------------------+ <= sc->sc_sram_base_va + CESA_SRAM_SIZE
36 * +------------------------+ <= sc->sc_sram_base_va + CESA_DATA(0)
37 * | struct cesa_sa_data |
38 * +------------------------+
39 * | struct cesa_sa_hdesc |
40 * +------------------------+ <= sc->sc_sram_base_va
43 #include <sys/cdefs.h>
44 __FBSDID("$FreeBSD$");
46 #include <sys/param.h>
47 #include <sys/systm.h>
49 #include <sys/endian.h>
50 #include <sys/kernel.h>
53 #include <sys/module.h>
54 #include <sys/mutex.h>
57 #include <machine/bus.h>
58 #include <machine/intr.h>
59 #include <machine/resource.h>
60 #include <machine/fdt.h>
62 #include <dev/fdt/fdt_common.h>
63 #include <dev/ofw/ofw_bus.h>
64 #include <dev/ofw/ofw_bus_subr.h>
67 #include <crypto/sha1.h>
68 #include <crypto/sha2/sha256.h>
69 #include <crypto/rijndael/rijndael.h>
70 #include <opencrypto/cryptodev.h>
71 #include "cryptodev_if.h"
73 #include <arm/mv/mvreg.h>
74 #include <arm/mv/mvvar.h>
77 static int cesa_probe(device_t);
78 static int cesa_attach(device_t);
79 static int cesa_detach(device_t);
80 static void cesa_intr(void *);
81 static int cesa_newsession(device_t, u_int32_t *, struct cryptoini *);
82 static int cesa_freesession(device_t, u_int64_t);
83 static int cesa_process(device_t, struct cryptop *, int);
85 static struct resource_spec cesa_res_spec[] = {
86 { SYS_RES_MEMORY, 0, RF_ACTIVE },
87 { SYS_RES_MEMORY, 1, RF_ACTIVE },
88 { SYS_RES_IRQ, 0, RF_ACTIVE | RF_SHAREABLE },
92 static device_method_t cesa_methods[] = {
93 /* Device interface */
94 DEVMETHOD(device_probe, cesa_probe),
95 DEVMETHOD(device_attach, cesa_attach),
96 DEVMETHOD(device_detach, cesa_detach),
98 /* Crypto device methods */
99 DEVMETHOD(cryptodev_newsession, cesa_newsession),
100 DEVMETHOD(cryptodev_freesession,cesa_freesession),
101 DEVMETHOD(cryptodev_process, cesa_process),
106 static driver_t cesa_driver = {
109 sizeof (struct cesa_softc)
111 static devclass_t cesa_devclass;
113 DRIVER_MODULE(cesa, simplebus, cesa_driver, cesa_devclass, 0, 0);
114 MODULE_DEPEND(cesa, crypto, 1, 1, 1);
117 cesa_dump_cshd(struct cesa_softc *sc, struct cesa_sa_hdesc *cshd)
123 device_printf(dev, "CESA SA Hardware Descriptor:\n");
124 device_printf(dev, "\t\tconfig: 0x%08X\n", cshd->cshd_config);
125 device_printf(dev, "\t\te_src: 0x%08X\n", cshd->cshd_enc_src);
126 device_printf(dev, "\t\te_dst: 0x%08X\n", cshd->cshd_enc_dst);
127 device_printf(dev, "\t\te_dlen: 0x%08X\n", cshd->cshd_enc_dlen);
128 device_printf(dev, "\t\te_key: 0x%08X\n", cshd->cshd_enc_key);
129 device_printf(dev, "\t\te_iv_1: 0x%08X\n", cshd->cshd_enc_iv);
130 device_printf(dev, "\t\te_iv_2: 0x%08X\n", cshd->cshd_enc_iv_buf);
131 device_printf(dev, "\t\tm_src: 0x%08X\n", cshd->cshd_mac_src);
132 device_printf(dev, "\t\tm_dst: 0x%08X\n", cshd->cshd_mac_dst);
133 device_printf(dev, "\t\tm_dlen: 0x%08X\n", cshd->cshd_mac_dlen);
134 device_printf(dev, "\t\tm_tlen: 0x%08X\n", cshd->cshd_mac_total_dlen);
135 device_printf(dev, "\t\tm_iv_i: 0x%08X\n", cshd->cshd_mac_iv_in);
136 device_printf(dev, "\t\tm_iv_o: 0x%08X\n", cshd->cshd_mac_iv_out);
141 cesa_alloc_dma_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
143 struct cesa_dma_mem *cdm;
148 KASSERT(nseg == 1, ("Got wrong number of DMA segments, should be 1."));
150 cdm->cdm_paddr = segs->ds_addr;
154 cesa_alloc_dma_mem(struct cesa_softc *sc, struct cesa_dma_mem *cdm,
159 KASSERT(cdm->cdm_vaddr == NULL,
160 ("%s(): DMA memory descriptor in use.", __func__));
162 error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), /* parent */
163 PAGE_SIZE, 0, /* alignment, boundary */
164 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
165 BUS_SPACE_MAXADDR, /* highaddr */
166 NULL, NULL, /* filtfunc, filtfuncarg */
167 size, 1, /* maxsize, nsegments */
168 size, 0, /* maxsegsz, flags */
169 NULL, NULL, /* lockfunc, lockfuncarg */
170 &cdm->cdm_tag); /* dmat */
172 device_printf(sc->sc_dev, "failed to allocate busdma tag, error"
178 error = bus_dmamem_alloc(cdm->cdm_tag, &cdm->cdm_vaddr,
179 BUS_DMA_NOWAIT | BUS_DMA_ZERO, &cdm->cdm_map);
181 device_printf(sc->sc_dev, "failed to allocate DMA safe"
182 " memory, error %i!\n", error);
187 error = bus_dmamap_load(cdm->cdm_tag, cdm->cdm_map, cdm->cdm_vaddr,
188 size, cesa_alloc_dma_mem_cb, cdm, BUS_DMA_NOWAIT);
190 device_printf(sc->sc_dev, "cannot get address of the DMA"
191 " memory, error %i\n", error);
198 bus_dmamem_free(cdm->cdm_tag, cdm->cdm_vaddr, cdm->cdm_map);
200 bus_dma_tag_destroy(cdm->cdm_tag);
202 cdm->cdm_vaddr = NULL;
207 cesa_free_dma_mem(struct cesa_dma_mem *cdm)
210 bus_dmamap_unload(cdm->cdm_tag, cdm->cdm_map);
211 bus_dmamem_free(cdm->cdm_tag, cdm->cdm_vaddr, cdm->cdm_map);
212 bus_dma_tag_destroy(cdm->cdm_tag);
213 cdm->cdm_vaddr = NULL;
217 cesa_sync_dma_mem(struct cesa_dma_mem *cdm, bus_dmasync_op_t op)
220 /* Sync only if dma memory is valid */
221 if (cdm->cdm_vaddr != NULL)
222 bus_dmamap_sync(cdm->cdm_tag, cdm->cdm_map, op);
226 cesa_sync_desc(struct cesa_softc *sc, bus_dmasync_op_t op)
229 cesa_sync_dma_mem(&sc->sc_tdesc_cdm, op);
230 cesa_sync_dma_mem(&sc->sc_sdesc_cdm, op);
231 cesa_sync_dma_mem(&sc->sc_requests_cdm, op);
234 static struct cesa_session *
235 cesa_alloc_session(struct cesa_softc *sc)
237 struct cesa_session *cs;
239 CESA_GENERIC_ALLOC_LOCKED(sc, cs, sessions);
244 static struct cesa_session *
245 cesa_get_session(struct cesa_softc *sc, uint32_t sid)
248 if (sid >= CESA_SESSIONS)
251 return (&sc->sc_sessions[sid]);
255 cesa_free_session(struct cesa_softc *sc, struct cesa_session *cs)
258 CESA_GENERIC_FREE_LOCKED(sc, cs, sessions);
261 static struct cesa_request *
262 cesa_alloc_request(struct cesa_softc *sc)
264 struct cesa_request *cr;
266 CESA_GENERIC_ALLOC_LOCKED(sc, cr, requests);
270 STAILQ_INIT(&cr->cr_tdesc);
271 STAILQ_INIT(&cr->cr_sdesc);
277 cesa_free_request(struct cesa_softc *sc, struct cesa_request *cr)
280 /* Free TDMA descriptors assigned to this request */
281 CESA_LOCK(sc, tdesc);
282 STAILQ_CONCAT(&sc->sc_free_tdesc, &cr->cr_tdesc);
283 CESA_UNLOCK(sc, tdesc);
285 /* Free SA descriptors assigned to this request */
286 CESA_LOCK(sc, sdesc);
287 STAILQ_CONCAT(&sc->sc_free_sdesc, &cr->cr_sdesc);
288 CESA_UNLOCK(sc, sdesc);
290 /* Unload DMA memory associated with request */
291 if (cr->cr_dmap_loaded) {
292 bus_dmamap_unload(sc->sc_data_dtag, cr->cr_dmap);
293 cr->cr_dmap_loaded = 0;
296 CESA_GENERIC_FREE_LOCKED(sc, cr, requests);
300 cesa_enqueue_request(struct cesa_softc *sc, struct cesa_request *cr)
303 CESA_LOCK(sc, requests);
304 STAILQ_INSERT_TAIL(&sc->sc_ready_requests, cr, cr_stq);
305 CESA_UNLOCK(sc, requests);
308 static struct cesa_tdma_desc *
309 cesa_alloc_tdesc(struct cesa_softc *sc)
311 struct cesa_tdma_desc *ctd;
313 CESA_GENERIC_ALLOC_LOCKED(sc, ctd, tdesc);
316 device_printf(sc->sc_dev, "TDMA descriptors pool exhaused. "
317 "Consider increasing CESA_TDMA_DESCRIPTORS.\n");
322 static struct cesa_sa_desc *
323 cesa_alloc_sdesc(struct cesa_softc *sc, struct cesa_request *cr)
325 struct cesa_sa_desc *csd;
327 CESA_GENERIC_ALLOC_LOCKED(sc, csd, sdesc);
329 device_printf(sc->sc_dev, "SA descriptors pool exhaused. "
330 "Consider increasing CESA_SA_DESCRIPTORS.\n");
334 STAILQ_INSERT_TAIL(&cr->cr_sdesc, csd, csd_stq);
336 /* Fill-in SA descriptor with default values */
337 csd->csd_cshd->cshd_enc_key = CESA_SA_DATA(csd_key);
338 csd->csd_cshd->cshd_enc_iv = CESA_SA_DATA(csd_iv);
339 csd->csd_cshd->cshd_enc_iv_buf = CESA_SA_DATA(csd_iv);
340 csd->csd_cshd->cshd_enc_src = 0;
341 csd->csd_cshd->cshd_enc_dst = 0;
342 csd->csd_cshd->cshd_enc_dlen = 0;
343 csd->csd_cshd->cshd_mac_dst = CESA_SA_DATA(csd_hash);
344 csd->csd_cshd->cshd_mac_iv_in = CESA_SA_DATA(csd_hiv_in);
345 csd->csd_cshd->cshd_mac_iv_out = CESA_SA_DATA(csd_hiv_out);
346 csd->csd_cshd->cshd_mac_src = 0;
347 csd->csd_cshd->cshd_mac_dlen = 0;
352 static struct cesa_tdma_desc *
353 cesa_tdma_copy(struct cesa_softc *sc, bus_addr_t dst, bus_addr_t src,
356 struct cesa_tdma_desc *ctd;
358 ctd = cesa_alloc_tdesc(sc);
362 ctd->ctd_cthd->cthd_dst = dst;
363 ctd->ctd_cthd->cthd_src = src;
364 ctd->ctd_cthd->cthd_byte_count = size;
366 /* Handle special control packet */
368 ctd->ctd_cthd->cthd_flags = CESA_CTHD_OWNED;
370 ctd->ctd_cthd->cthd_flags = 0;
375 static struct cesa_tdma_desc *
376 cesa_tdma_copyin_sa_data(struct cesa_softc *sc, struct cesa_request *cr)
379 return (cesa_tdma_copy(sc, sc->sc_sram_base_pa +
380 sizeof(struct cesa_sa_hdesc), cr->cr_csd_paddr,
381 sizeof(struct cesa_sa_data)));
384 static struct cesa_tdma_desc *
385 cesa_tdma_copyout_sa_data(struct cesa_softc *sc, struct cesa_request *cr)
388 return (cesa_tdma_copy(sc, cr->cr_csd_paddr, sc->sc_sram_base_pa +
389 sizeof(struct cesa_sa_hdesc), sizeof(struct cesa_sa_data)));
392 static struct cesa_tdma_desc *
393 cesa_tdma_copy_sdesc(struct cesa_softc *sc, struct cesa_sa_desc *csd)
396 return (cesa_tdma_copy(sc, sc->sc_sram_base_pa, csd->csd_cshd_paddr,
397 sizeof(struct cesa_sa_hdesc)));
401 cesa_append_tdesc(struct cesa_request *cr, struct cesa_tdma_desc *ctd)
403 struct cesa_tdma_desc *ctd_prev;
405 if (!STAILQ_EMPTY(&cr->cr_tdesc)) {
406 ctd_prev = STAILQ_LAST(&cr->cr_tdesc, cesa_tdma_desc, ctd_stq);
407 ctd_prev->ctd_cthd->cthd_next = ctd->ctd_cthd_paddr;
410 ctd->ctd_cthd->cthd_next = 0;
411 STAILQ_INSERT_TAIL(&cr->cr_tdesc, ctd, ctd_stq);
415 cesa_append_packet(struct cesa_softc *sc, struct cesa_request *cr,
416 struct cesa_packet *cp, struct cesa_sa_desc *csd)
418 struct cesa_tdma_desc *ctd, *tmp;
420 /* Copy SA descriptor for this packet */
421 ctd = cesa_tdma_copy_sdesc(sc, csd);
425 cesa_append_tdesc(cr, ctd);
427 /* Copy data to be processed */
428 STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyin, ctd_stq, tmp)
429 cesa_append_tdesc(cr, ctd);
430 STAILQ_INIT(&cp->cp_copyin);
432 /* Insert control descriptor */
433 ctd = cesa_tdma_copy(sc, 0, 0, 0);
437 cesa_append_tdesc(cr, ctd);
439 /* Copy back results */
440 STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyout, ctd_stq, tmp)
441 cesa_append_tdesc(cr, ctd);
442 STAILQ_INIT(&cp->cp_copyout);
448 cesa_set_mkey(struct cesa_session *cs, int alg, const uint8_t *mkey, int mklen)
450 uint8_t ipad[CESA_MAX_HMAC_BLOCK_LEN];
451 uint8_t opad[CESA_MAX_HMAC_BLOCK_LEN];
453 SHA256_CTX sha256ctx;
459 memset(ipad, HMAC_IPAD_VAL, CESA_MAX_HMAC_BLOCK_LEN);
460 memset(opad, HMAC_OPAD_VAL, CESA_MAX_HMAC_BLOCK_LEN);
461 for (i = 0; i < mklen; i++) {
466 hin = (uint32_t *)cs->cs_hiv_in;
467 hout = (uint32_t *)cs->cs_hiv_out;
470 case CRYPTO_MD5_HMAC:
472 MD5Update(&md5ctx, ipad, MD5_HMAC_BLOCK_LEN);
473 memcpy(hin, md5ctx.state, sizeof(md5ctx.state));
475 MD5Update(&md5ctx, opad, MD5_HMAC_BLOCK_LEN);
476 memcpy(hout, md5ctx.state, sizeof(md5ctx.state));
478 case CRYPTO_SHA1_HMAC:
480 SHA1Update(&sha1ctx, ipad, SHA1_HMAC_BLOCK_LEN);
481 memcpy(hin, sha1ctx.h.b32, sizeof(sha1ctx.h.b32));
483 SHA1Update(&sha1ctx, opad, SHA1_HMAC_BLOCK_LEN);
484 memcpy(hout, sha1ctx.h.b32, sizeof(sha1ctx.h.b32));
486 case CRYPTO_SHA2_256_HMAC:
487 SHA256_Init(&sha256ctx);
488 SHA256_Update(&sha256ctx, ipad, SHA2_256_HMAC_BLOCK_LEN);
489 memcpy(hin, sha256ctx.state, sizeof(sha256ctx.state));
490 SHA256_Init(&sha256ctx);
491 SHA256_Update(&sha256ctx, opad, SHA2_256_HMAC_BLOCK_LEN);
492 memcpy(hout, sha256ctx.state, sizeof(sha256ctx.state));
498 for (i = 0; i < CESA_MAX_HASH_LEN / sizeof(uint32_t); i++) {
499 hin[i] = htobe32(hin[i]);
500 hout[i] = htobe32(hout[i]);
507 cesa_prep_aes_key(struct cesa_session *cs)
509 uint32_t ek[4 * (RIJNDAEL_MAXNR + 1)];
513 rijndaelKeySetupEnc(ek, cs->cs_key, cs->cs_klen * 8);
515 cs->cs_config &= ~CESA_CSH_AES_KLEN_MASK;
516 dkey = (uint32_t *)cs->cs_aes_dkey;
518 switch (cs->cs_klen) {
520 cs->cs_config |= CESA_CSH_AES_KLEN_128;
521 for (i = 0; i < 4; i++)
522 *dkey++ = htobe32(ek[4 * 10 + i]);
525 cs->cs_config |= CESA_CSH_AES_KLEN_192;
526 for (i = 0; i < 4; i++)
527 *dkey++ = htobe32(ek[4 * 12 + i]);
528 for (i = 0; i < 2; i++)
529 *dkey++ = htobe32(ek[4 * 11 + 2 + i]);
532 cs->cs_config |= CESA_CSH_AES_KLEN_256;
533 for (i = 0; i < 4; i++)
534 *dkey++ = htobe32(ek[4 * 14 + i]);
535 for (i = 0; i < 4; i++)
536 *dkey++ = htobe32(ek[4 * 13 + i]);
546 cesa_is_hash(int alg)
551 case CRYPTO_MD5_HMAC:
553 case CRYPTO_SHA1_HMAC:
554 case CRYPTO_SHA2_256_HMAC:
562 cesa_start_packet(struct cesa_packet *cp, unsigned int size)
567 STAILQ_INIT(&cp->cp_copyin);
568 STAILQ_INIT(&cp->cp_copyout);
572 cesa_fill_packet(struct cesa_softc *sc, struct cesa_packet *cp,
573 bus_dma_segment_t *seg)
575 struct cesa_tdma_desc *ctd;
578 /* Calculate size of block copy */
579 bsize = MIN(seg->ds_len, cp->cp_size - cp->cp_offset);
582 ctd = cesa_tdma_copy(sc, sc->sc_sram_base_pa +
583 CESA_DATA(cp->cp_offset), seg->ds_addr, bsize);
587 STAILQ_INSERT_TAIL(&cp->cp_copyin, ctd, ctd_stq);
589 ctd = cesa_tdma_copy(sc, seg->ds_addr, sc->sc_sram_base_pa +
590 CESA_DATA(cp->cp_offset), bsize);
594 STAILQ_INSERT_TAIL(&cp->cp_copyout, ctd, ctd_stq);
596 seg->ds_len -= bsize;
597 seg->ds_addr += bsize;
598 cp->cp_offset += bsize;
605 cesa_create_chain_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
607 unsigned int mpsize, fragmented;
608 unsigned int mlen, mskip, tmlen;
609 struct cesa_chain_info *cci;
610 unsigned int elen, eskip;
611 unsigned int skip, len;
612 struct cesa_sa_desc *csd;
613 struct cesa_request *cr;
614 struct cesa_softc *sc;
615 struct cesa_packet cp;
616 bus_dma_segment_t seg;
625 cci->cci_error = error;
629 elen = cci->cci_enc ? cci->cci_enc->crd_len : 0;
630 eskip = cci->cci_enc ? cci->cci_enc->crd_skip : 0;
631 mlen = cci->cci_mac ? cci->cci_mac->crd_len : 0;
632 mskip = cci->cci_mac ? cci->cci_mac->crd_skip : 0;
635 ((eskip > mskip && ((eskip - mskip) & (cr->cr_cs->cs_ivlen - 1))) ||
636 (mskip > eskip && ((mskip - eskip) & (cr->cr_cs->cs_mblen - 1))) ||
637 (eskip > (mskip + mlen)) || (mskip > (eskip + elen)))) {
639 * Data alignment in the request does not meet CESA requiremnts
640 * for combined encryption/decryption and hashing. We have to
641 * split the request to separate operations and process them
644 config = cci->cci_config;
645 if ((config & CESA_CSHD_OP_MASK) == CESA_CSHD_MAC_AND_ENC) {
646 config &= ~CESA_CSHD_OP_MASK;
648 cci->cci_config = config | CESA_CSHD_MAC;
650 cci->cci_mac = cr->cr_mac;
651 cesa_create_chain_cb(cci, segs, nseg, cci->cci_error);
653 cci->cci_config = config | CESA_CSHD_ENC;
654 cci->cci_enc = cr->cr_enc;
656 cesa_create_chain_cb(cci, segs, nseg, cci->cci_error);
658 config &= ~CESA_CSHD_OP_MASK;
660 cci->cci_config = config | CESA_CSHD_ENC;
661 cci->cci_enc = cr->cr_enc;
663 cesa_create_chain_cb(cci, segs, nseg, cci->cci_error);
665 cci->cci_config = config | CESA_CSHD_MAC;
667 cci->cci_mac = cr->cr_mac;
668 cesa_create_chain_cb(cci, segs, nseg, cci->cci_error);
676 mpsize = CESA_MAX_PACKET_SIZE;
677 mpsize &= ~((cr->cr_cs->cs_ivlen - 1) | (cr->cr_cs->cs_mblen - 1));
680 skip = MIN(eskip, mskip);
681 len = MAX(elen + eskip, mlen + mskip) - skip;
690 /* Start first packet in chain */
691 cesa_start_packet(&cp, MIN(mpsize, len));
693 while (nseg-- && len > 0) {
697 * Skip data in buffer on which neither ENC nor MAC operation
701 size = MIN(skip, seg.ds_len);
719 * Fill in current packet with data. Break if there is
720 * no more data in current DMA segment or an error
723 size = cesa_fill_packet(sc, &cp, &seg);
731 /* If packet is full, append it to the chain */
732 if (cp.cp_size == cp.cp_offset) {
733 csd = cesa_alloc_sdesc(sc, cr);
739 /* Create SA descriptor for this packet */
740 csd->csd_cshd->cshd_config = cci->cci_config;
741 csd->csd_cshd->cshd_mac_total_dlen = tmlen;
744 * Enable fragmentation if request will not fit
750 csd->csd_cshd->cshd_config |=
751 CESA_CSHD_FRAG_FIRST;
753 csd->csd_cshd->cshd_config |=
754 CESA_CSHD_FRAG_MIDDLE;
755 } else if (fragmented)
756 csd->csd_cshd->cshd_config |=
759 if (eskip < cp.cp_size && elen > 0) {
760 csd->csd_cshd->cshd_enc_src =
762 csd->csd_cshd->cshd_enc_dst =
764 csd->csd_cshd->cshd_enc_dlen =
765 MIN(elen, cp.cp_size - eskip);
768 if (mskip < cp.cp_size && mlen > 0) {
769 csd->csd_cshd->cshd_mac_src =
771 csd->csd_cshd->cshd_mac_dlen =
772 MIN(mlen, cp.cp_size - mskip);
775 elen -= csd->csd_cshd->cshd_enc_dlen;
776 eskip -= MIN(eskip, cp.cp_size);
777 mlen -= csd->csd_cshd->cshd_mac_dlen;
778 mskip -= MIN(mskip, cp.cp_size);
780 cesa_dump_cshd(sc, csd->csd_cshd);
782 /* Append packet to the request */
783 error = cesa_append_packet(sc, cr, &cp, csd);
787 /* Start a new packet, as current is full */
788 cesa_start_packet(&cp, MIN(mpsize, len));
798 * Move all allocated resources to the request. They will be
801 STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyin);
802 STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyout);
803 cci->cci_error = error;
808 cesa_create_chain_cb2(void *arg, bus_dma_segment_t *segs, int nseg,
809 bus_size_t size, int error)
812 cesa_create_chain_cb(arg, segs, nseg, error);
816 cesa_create_chain(struct cesa_softc *sc, struct cesa_request *cr)
818 struct cesa_chain_info cci;
819 struct cesa_tdma_desc *ctd;
824 CESA_LOCK_ASSERT(sc, sessions);
826 /* Create request metadata */
828 if (cr->cr_enc->crd_alg == CRYPTO_AES_CBC &&
829 (cr->cr_enc->crd_flags & CRD_F_ENCRYPT) == 0)
830 memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_aes_dkey,
833 memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_key,
838 memcpy(cr->cr_csd->csd_hiv_in, cr->cr_cs->cs_hiv_in,
840 memcpy(cr->cr_csd->csd_hiv_out, cr->cr_cs->cs_hiv_out,
844 ctd = cesa_tdma_copyin_sa_data(sc, cr);
848 cesa_append_tdesc(cr, ctd);
850 /* Prepare SA configuration */
851 config = cr->cr_cs->cs_config;
853 if (cr->cr_enc && (cr->cr_enc->crd_flags & CRD_F_ENCRYPT) == 0)
854 config |= CESA_CSHD_DECRYPT;
855 if (cr->cr_enc && !cr->cr_mac)
856 config |= CESA_CSHD_ENC;
857 if (!cr->cr_enc && cr->cr_mac)
858 config |= CESA_CSHD_MAC;
859 if (cr->cr_enc && cr->cr_mac)
860 config |= (config & CESA_CSHD_DECRYPT) ? CESA_CSHD_MAC_AND_ENC :
861 CESA_CSHD_ENC_AND_MAC;
863 /* Create data packets */
866 cci.cci_enc = cr->cr_enc;
867 cci.cci_mac = cr->cr_mac;
868 cci.cci_config = config;
871 if (cr->cr_crp->crp_flags & CRYPTO_F_IOV)
872 error = bus_dmamap_load_uio(sc->sc_data_dtag,
873 cr->cr_dmap, (struct uio *)cr->cr_crp->crp_buf,
874 cesa_create_chain_cb2, &cci, BUS_DMA_NOWAIT);
875 else if (cr->cr_crp->crp_flags & CRYPTO_F_IMBUF)
876 error = bus_dmamap_load_mbuf(sc->sc_data_dtag,
877 cr->cr_dmap, (struct mbuf *)cr->cr_crp->crp_buf,
878 cesa_create_chain_cb2, &cci, BUS_DMA_NOWAIT);
880 error = bus_dmamap_load(sc->sc_data_dtag,
881 cr->cr_dmap, cr->cr_crp->crp_buf,
882 cr->cr_crp->crp_ilen, cesa_create_chain_cb, &cci,
886 cr->cr_dmap_loaded = 1;
889 error = cci.cci_error;
894 /* Read back request metadata */
895 ctd = cesa_tdma_copyout_sa_data(sc, cr);
899 cesa_append_tdesc(cr, ctd);
905 cesa_execute(struct cesa_softc *sc)
907 struct cesa_tdma_desc *prev_ctd, *ctd;
908 struct cesa_request *prev_cr, *cr;
910 CESA_LOCK(sc, requests);
913 * If ready list is empty, there is nothing to execute. If queued list
914 * is not empty, the hardware is busy and we cannot start another
917 if (STAILQ_EMPTY(&sc->sc_ready_requests) ||
918 !STAILQ_EMPTY(&sc->sc_queued_requests)) {
919 CESA_UNLOCK(sc, requests);
923 /* Move all ready requests to queued list */
924 STAILQ_CONCAT(&sc->sc_queued_requests, &sc->sc_ready_requests);
925 STAILQ_INIT(&sc->sc_ready_requests);
927 /* Create one execution chain from all requests on the list */
928 if (STAILQ_FIRST(&sc->sc_queued_requests) !=
929 STAILQ_LAST(&sc->sc_queued_requests, cesa_request, cr_stq)) {
931 cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_POSTREAD |
932 BUS_DMASYNC_POSTWRITE);
934 STAILQ_FOREACH(cr, &sc->sc_queued_requests, cr_stq) {
936 ctd = STAILQ_FIRST(&cr->cr_tdesc);
937 prev_ctd = STAILQ_LAST(&prev_cr->cr_tdesc,
938 cesa_tdma_desc, ctd_stq);
940 prev_ctd->ctd_cthd->cthd_next =
947 cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_PREREAD |
948 BUS_DMASYNC_PREWRITE);
951 /* Start chain execution in hardware */
952 cr = STAILQ_FIRST(&sc->sc_queued_requests);
953 ctd = STAILQ_FIRST(&cr->cr_tdesc);
955 CESA_TDMA_WRITE(sc, CESA_TDMA_ND, ctd->ctd_cthd_paddr);
957 if (sc->sc_soc_id == MV_DEV_88F6828 ||
958 sc->sc_soc_id == MV_DEV_88F6820 ||
959 sc->sc_soc_id == MV_DEV_88F6810)
960 CESA_REG_WRITE(sc, CESA_SA_CMD, CESA_SA_CMD_ACTVATE | CESA_SA_CMD_SHA2);
962 CESA_REG_WRITE(sc, CESA_SA_CMD, CESA_SA_CMD_ACTVATE);
964 CESA_UNLOCK(sc, requests);
968 cesa_setup_sram(struct cesa_softc *sc)
971 ihandle_t sram_ihandle;
972 pcell_t sram_handle, sram_reg[2];
976 rv = OF_getencprop(ofw_bus_get_node(sc->sc_dev), "sram-handle",
977 (void *)&sram_handle, sizeof(sram_handle));
981 sram_ihandle = (ihandle_t)sram_handle;
982 sram_node = OF_instance_to_package(sram_ihandle);
984 rv = OF_getencprop(sram_node, "reg", (void *)sram_reg, sizeof(sram_reg));
988 sc->sc_sram_base_pa = sram_reg[0];
989 /* Store SRAM size to be able to unmap in detach() */
990 sc->sc_sram_size = sram_reg[1];
992 if (sc->sc_soc_id != MV_DEV_88F6828 &&
993 sc->sc_soc_id != MV_DEV_88F6820 &&
994 sc->sc_soc_id != MV_DEV_88F6810)
997 /* SRAM memory was not mapped in platform_sram_devmap(), map it now */
998 sram_va = pmap_mapdev(sc->sc_sram_base_pa, sc->sc_sram_size);
1001 sc->sc_sram_base_va = (vm_offset_t)sram_va;
1007 cesa_probe(device_t dev)
1010 if (!ofw_bus_status_okay(dev))
1013 if (!ofw_bus_is_compatible(dev, "mrvl,cesa"))
1016 device_set_desc(dev, "Marvell Cryptographic Engine and Security "
1019 return (BUS_PROBE_DEFAULT);
1023 cesa_attach(device_t dev)
1025 struct cesa_softc *sc;
1030 sc = device_get_softc(dev);
1038 case MV_DEV_88F6281:
1039 case MV_DEV_88F6282:
1040 /* Check if CESA peripheral device has power turned on */
1041 if (soc_power_ctrl_get(CPU_PM_CTRL_CRYPTO) ==
1042 CPU_PM_CTRL_CRYPTO) {
1043 device_printf(dev, "not powered on\n");
1048 case MV_DEV_88F6828:
1049 case MV_DEV_88F6820:
1050 case MV_DEV_88F6810:
1053 case MV_DEV_MV78100:
1054 case MV_DEV_MV78100_Z0:
1055 /* Check if CESA peripheral device has power turned on */
1056 if (soc_power_ctrl_get(CPU_PM_CTRL_CRYPTO) !=
1057 CPU_PM_CTRL_CRYPTO) {
1058 device_printf(dev, "not powered on\n");
1061 sc->sc_tperr = CESA_ICR_TPERR;
1069 /* Initialize mutexes */
1070 mtx_init(&sc->sc_sc_lock, device_get_nameunit(dev),
1071 "CESA Shared Data", MTX_DEF);
1072 mtx_init(&sc->sc_tdesc_lock, device_get_nameunit(dev),
1073 "CESA TDMA Descriptors Pool", MTX_DEF);
1074 mtx_init(&sc->sc_sdesc_lock, device_get_nameunit(dev),
1075 "CESA SA Descriptors Pool", MTX_DEF);
1076 mtx_init(&sc->sc_requests_lock, device_get_nameunit(dev),
1077 "CESA Requests Pool", MTX_DEF);
1078 mtx_init(&sc->sc_sessions_lock, device_get_nameunit(dev),
1079 "CESA Sessions Pool", MTX_DEF);
1081 /* Allocate I/O and IRQ resources */
1082 error = bus_alloc_resources(dev, cesa_res_spec, sc->sc_res);
1084 device_printf(dev, "could not allocate resources\n");
1088 /* Acquire SRAM base address */
1089 error = cesa_setup_sram(sc);
1091 device_printf(dev, "could not setup SRAM\n");
1095 /* Setup interrupt handler */
1096 error = bus_setup_intr(dev, sc->sc_res[RES_CESA_IRQ], INTR_TYPE_NET |
1097 INTR_MPSAFE, NULL, cesa_intr, sc, &(sc->sc_icookie));
1099 device_printf(dev, "could not setup engine completion irq\n");
1103 /* Create DMA tag for processed data */
1104 error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
1105 1, 0, /* alignment, boundary */
1106 BUS_SPACE_MAXADDR_32BIT, /* lowaddr */
1107 BUS_SPACE_MAXADDR, /* highaddr */
1108 NULL, NULL, /* filtfunc, filtfuncarg */
1109 CESA_MAX_REQUEST_SIZE, /* maxsize */
1110 CESA_MAX_FRAGMENTS, /* nsegments */
1111 CESA_MAX_REQUEST_SIZE, 0, /* maxsegsz, flags */
1112 NULL, NULL, /* lockfunc, lockfuncarg */
1113 &sc->sc_data_dtag); /* dmat */
1117 /* Initialize data structures: TDMA Descriptors Pool */
1118 error = cesa_alloc_dma_mem(sc, &sc->sc_tdesc_cdm,
1119 CESA_TDMA_DESCRIPTORS * sizeof(struct cesa_tdma_hdesc));
1123 STAILQ_INIT(&sc->sc_free_tdesc);
1124 for (i = 0; i < CESA_TDMA_DESCRIPTORS; i++) {
1125 sc->sc_tdesc[i].ctd_cthd =
1126 (struct cesa_tdma_hdesc *)(sc->sc_tdesc_cdm.cdm_vaddr) + i;
1127 sc->sc_tdesc[i].ctd_cthd_paddr = sc->sc_tdesc_cdm.cdm_paddr +
1128 (i * sizeof(struct cesa_tdma_hdesc));
1129 STAILQ_INSERT_TAIL(&sc->sc_free_tdesc, &sc->sc_tdesc[i],
1133 /* Initialize data structures: SA Descriptors Pool */
1134 error = cesa_alloc_dma_mem(sc, &sc->sc_sdesc_cdm,
1135 CESA_SA_DESCRIPTORS * sizeof(struct cesa_sa_hdesc));
1139 STAILQ_INIT(&sc->sc_free_sdesc);
1140 for (i = 0; i < CESA_SA_DESCRIPTORS; i++) {
1141 sc->sc_sdesc[i].csd_cshd =
1142 (struct cesa_sa_hdesc *)(sc->sc_sdesc_cdm.cdm_vaddr) + i;
1143 sc->sc_sdesc[i].csd_cshd_paddr = sc->sc_sdesc_cdm.cdm_paddr +
1144 (i * sizeof(struct cesa_sa_hdesc));
1145 STAILQ_INSERT_TAIL(&sc->sc_free_sdesc, &sc->sc_sdesc[i],
1149 /* Initialize data structures: Requests Pool */
1150 error = cesa_alloc_dma_mem(sc, &sc->sc_requests_cdm,
1151 CESA_REQUESTS * sizeof(struct cesa_sa_data));
1155 STAILQ_INIT(&sc->sc_free_requests);
1156 STAILQ_INIT(&sc->sc_ready_requests);
1157 STAILQ_INIT(&sc->sc_queued_requests);
1158 for (i = 0; i < CESA_REQUESTS; i++) {
1159 sc->sc_requests[i].cr_csd =
1160 (struct cesa_sa_data *)(sc->sc_requests_cdm.cdm_vaddr) + i;
1161 sc->sc_requests[i].cr_csd_paddr =
1162 sc->sc_requests_cdm.cdm_paddr +
1163 (i * sizeof(struct cesa_sa_data));
1165 /* Preallocate DMA maps */
1166 error = bus_dmamap_create(sc->sc_data_dtag, 0,
1167 &sc->sc_requests[i].cr_dmap);
1168 if (error && i > 0) {
1171 bus_dmamap_destroy(sc->sc_data_dtag,
1172 sc->sc_requests[i].cr_dmap);
1178 STAILQ_INSERT_TAIL(&sc->sc_free_requests, &sc->sc_requests[i],
1182 /* Initialize data structures: Sessions Pool */
1183 STAILQ_INIT(&sc->sc_free_sessions);
1184 for (i = 0; i < CESA_SESSIONS; i++) {
1185 sc->sc_sessions[i].cs_sid = i;
1186 STAILQ_INSERT_TAIL(&sc->sc_free_sessions, &sc->sc_sessions[i],
1192 * - Burst limit: 128 bytes,
1193 * - Outstanding reads enabled,
1196 val = CESA_TDMA_CR_DBL128 | CESA_TDMA_CR_SBL128 |
1197 CESA_TDMA_CR_ORDEN | CESA_TDMA_CR_NBS | CESA_TDMA_CR_ENABLE;
1199 if (sc->sc_soc_id == MV_DEV_88F6828 ||
1200 sc->sc_soc_id == MV_DEV_88F6820 ||
1201 sc->sc_soc_id == MV_DEV_88F6810)
1202 val |= CESA_TDMA_NUM_OUTSTAND;
1204 CESA_TDMA_WRITE(sc, CESA_TDMA_CR, val);
1208 * - SA descriptor is present at beginning of CESA SRAM,
1209 * - Multi-packet chain mode,
1210 * - Cooperation with TDMA enabled.
1212 CESA_REG_WRITE(sc, CESA_SA_DPR, 0);
1213 CESA_REG_WRITE(sc, CESA_SA_CR, CESA_SA_CR_ACTIVATE_TDMA |
1214 CESA_SA_CR_WAIT_FOR_TDMA | CESA_SA_CR_MULTI_MODE);
1216 /* Unmask interrupts */
1217 CESA_REG_WRITE(sc, CESA_ICR, 0);
1218 CESA_REG_WRITE(sc, CESA_ICM, CESA_ICM_ACCTDMA | sc->sc_tperr);
1219 CESA_TDMA_WRITE(sc, CESA_TDMA_ECR, 0);
1220 CESA_TDMA_WRITE(sc, CESA_TDMA_EMR, CESA_TDMA_EMR_MISS |
1221 CESA_TDMA_EMR_DOUBLE_HIT | CESA_TDMA_EMR_BOTH_HIT |
1222 CESA_TDMA_EMR_DATA_ERROR);
1224 /* Register in OCF */
1225 sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE);
1226 if (sc->sc_cid < 0) {
1227 device_printf(dev, "could not get crypto driver id\n");
1231 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
1232 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
1233 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
1234 crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0);
1235 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
1236 crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0);
1237 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
1238 if (sc->sc_soc_id == MV_DEV_88F6828 ||
1239 sc->sc_soc_id == MV_DEV_88F6820 ||
1240 sc->sc_soc_id == MV_DEV_88F6810)
1241 crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0);
1245 for (i = 0; i < CESA_REQUESTS; i++)
1246 bus_dmamap_destroy(sc->sc_data_dtag,
1247 sc->sc_requests[i].cr_dmap);
1249 cesa_free_dma_mem(&sc->sc_requests_cdm);
1251 cesa_free_dma_mem(&sc->sc_sdesc_cdm);
1253 cesa_free_dma_mem(&sc->sc_tdesc_cdm);
1255 bus_dma_tag_destroy(sc->sc_data_dtag);
1257 bus_teardown_intr(dev, sc->sc_res[RES_CESA_IRQ], sc->sc_icookie);
1259 if (sc->sc_soc_id == MV_DEV_88F6828 ||
1260 sc->sc_soc_id == MV_DEV_88F6820 ||
1261 sc->sc_soc_id == MV_DEV_88F6810)
1262 pmap_unmapdev(sc->sc_sram_base_va, sc->sc_sram_size);
1264 bus_release_resources(dev, cesa_res_spec, sc->sc_res);
1266 mtx_destroy(&sc->sc_sessions_lock);
1267 mtx_destroy(&sc->sc_requests_lock);
1268 mtx_destroy(&sc->sc_sdesc_lock);
1269 mtx_destroy(&sc->sc_tdesc_lock);
1270 mtx_destroy(&sc->sc_sc_lock);
1275 cesa_detach(device_t dev)
1277 struct cesa_softc *sc;
1280 sc = device_get_softc(dev);
1282 /* TODO: Wait for queued requests completion before shutdown. */
1284 /* Mask interrupts */
1285 CESA_REG_WRITE(sc, CESA_ICM, 0);
1286 CESA_TDMA_WRITE(sc, CESA_TDMA_EMR, 0);
1288 /* Unregister from OCF */
1289 crypto_unregister_all(sc->sc_cid);
1292 for (i = 0; i < CESA_REQUESTS; i++)
1293 bus_dmamap_destroy(sc->sc_data_dtag,
1294 sc->sc_requests[i].cr_dmap);
1296 /* Free DMA Memory */
1297 cesa_free_dma_mem(&sc->sc_requests_cdm);
1298 cesa_free_dma_mem(&sc->sc_sdesc_cdm);
1299 cesa_free_dma_mem(&sc->sc_tdesc_cdm);
1302 bus_dma_tag_destroy(sc->sc_data_dtag);
1304 /* Stop interrupt */
1305 bus_teardown_intr(dev, sc->sc_res[RES_CESA_IRQ], sc->sc_icookie);
1307 /* Relase I/O and IRQ resources */
1308 bus_release_resources(dev, cesa_res_spec, sc->sc_res);
1310 /* Unmap SRAM memory */
1311 if (sc->sc_soc_id == MV_DEV_88F6828 ||
1312 sc->sc_soc_id == MV_DEV_88F6820 ||
1313 sc->sc_soc_id == MV_DEV_88F6810)
1314 pmap_unmapdev(sc->sc_sram_base_va, sc->sc_sram_size);
1316 /* Destroy mutexes */
1317 mtx_destroy(&sc->sc_sessions_lock);
1318 mtx_destroy(&sc->sc_requests_lock);
1319 mtx_destroy(&sc->sc_sdesc_lock);
1320 mtx_destroy(&sc->sc_tdesc_lock);
1321 mtx_destroy(&sc->sc_sc_lock);
1327 cesa_intr(void *arg)
1329 STAILQ_HEAD(, cesa_request) requests;
1330 struct cesa_request *cr, *tmp;
1331 struct cesa_softc *sc;
1338 ecr = CESA_TDMA_READ(sc, CESA_TDMA_ECR);
1339 CESA_TDMA_WRITE(sc, CESA_TDMA_ECR, 0);
1340 icr = CESA_REG_READ(sc, CESA_ICR);
1341 CESA_REG_WRITE(sc, CESA_ICR, 0);
1343 /* Check for TDMA errors */
1344 if (ecr & CESA_TDMA_ECR_MISS) {
1345 device_printf(sc->sc_dev, "TDMA Miss error detected!\n");
1349 if (ecr & CESA_TDMA_ECR_DOUBLE_HIT) {
1350 device_printf(sc->sc_dev, "TDMA Double Hit error detected!\n");
1354 if (ecr & CESA_TDMA_ECR_BOTH_HIT) {
1355 device_printf(sc->sc_dev, "TDMA Both Hit error detected!\n");
1359 if (ecr & CESA_TDMA_ECR_DATA_ERROR) {
1360 device_printf(sc->sc_dev, "TDMA Data error detected!\n");
1364 /* Check for CESA errors */
1365 if (icr & sc->sc_tperr) {
1366 device_printf(sc->sc_dev, "CESA SRAM Parity error detected!\n");
1370 /* If there is nothing more to do, return */
1371 if ((icr & CESA_ICR_ACCTDMA) == 0)
1374 /* Get all finished requests */
1375 CESA_LOCK(sc, requests);
1376 STAILQ_INIT(&requests);
1377 STAILQ_CONCAT(&requests, &sc->sc_queued_requests);
1378 STAILQ_INIT(&sc->sc_queued_requests);
1379 CESA_UNLOCK(sc, requests);
1381 /* Execute all ready requests */
1384 /* Process completed requests */
1385 cesa_sync_dma_mem(&sc->sc_requests_cdm, BUS_DMASYNC_POSTREAD |
1386 BUS_DMASYNC_POSTWRITE);
1388 STAILQ_FOREACH_SAFE(cr, &requests, cr_stq, tmp) {
1389 bus_dmamap_sync(sc->sc_data_dtag, cr->cr_dmap,
1390 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1392 cr->cr_crp->crp_etype = sc->sc_error;
1394 crypto_copyback(cr->cr_crp->crp_flags,
1395 cr->cr_crp->crp_buf, cr->cr_mac->crd_inject,
1396 cr->cr_cs->cs_hlen, cr->cr_csd->csd_hash);
1398 crypto_done(cr->cr_crp);
1399 cesa_free_request(sc, cr);
1402 cesa_sync_dma_mem(&sc->sc_requests_cdm, BUS_DMASYNC_PREREAD |
1403 BUS_DMASYNC_PREWRITE);
1407 /* Unblock driver if it ran out of resources */
1409 blocked = sc->sc_blocked;
1411 CESA_UNLOCK(sc, sc);
1414 crypto_unblock(sc->sc_cid, blocked);
1418 cesa_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri)
1420 struct cesa_session *cs;
1421 struct cesa_softc *sc;
1422 struct cryptoini *enc;
1423 struct cryptoini *mac;
1426 sc = device_get_softc(dev);
1431 /* Check and parse input */
1432 if (cesa_is_hash(cri->cri_alg))
1437 cri = cri->cri_next;
1440 if (!enc && !cesa_is_hash(cri->cri_alg))
1443 if (!mac && cesa_is_hash(cri->cri_alg))
1446 if (cri->cri_next || !(enc && mac))
1450 if ((enc && (enc->cri_klen / 8) > CESA_MAX_KEY_LEN) ||
1451 (mac && (mac->cri_klen / 8) > CESA_MAX_MKEY_LEN))
1454 /* Allocate session */
1455 cs = cesa_alloc_session(sc);
1459 /* Prepare CESA configuration */
1465 switch (enc->cri_alg) {
1466 case CRYPTO_AES_CBC:
1467 cs->cs_config |= CESA_CSHD_AES | CESA_CSHD_CBC;
1468 cs->cs_ivlen = AES_BLOCK_LEN;
1470 case CRYPTO_DES_CBC:
1471 cs->cs_config |= CESA_CSHD_DES | CESA_CSHD_CBC;
1472 cs->cs_ivlen = DES_BLOCK_LEN;
1474 case CRYPTO_3DES_CBC:
1475 cs->cs_config |= CESA_CSHD_3DES | CESA_CSHD_3DES_EDE |
1477 cs->cs_ivlen = DES3_BLOCK_LEN;
1485 if (!error && mac) {
1486 switch (mac->cri_alg) {
1489 cs->cs_hlen = (mac->cri_mlen == 0) ? MD5_HASH_LEN :
1491 cs->cs_config |= CESA_CSHD_MD5;
1493 case CRYPTO_MD5_HMAC:
1494 cs->cs_mblen = MD5_HMAC_BLOCK_LEN;
1495 cs->cs_hlen = (mac->cri_mlen == 0) ? MD5_HASH_LEN :
1497 cs->cs_config |= CESA_CSHD_MD5_HMAC;
1498 if (cs->cs_hlen == CESA_HMAC_TRUNC_LEN)
1499 cs->cs_config |= CESA_CSHD_96_BIT_HMAC;
1503 cs->cs_hlen = (mac->cri_mlen == 0) ? SHA1_HASH_LEN :
1505 cs->cs_config |= CESA_CSHD_SHA1;
1507 case CRYPTO_SHA1_HMAC:
1508 cs->cs_mblen = SHA1_HMAC_BLOCK_LEN;
1509 cs->cs_hlen = (mac->cri_mlen == 0) ? SHA1_HASH_LEN :
1511 cs->cs_config |= CESA_CSHD_SHA1_HMAC;
1512 if (cs->cs_hlen == CESA_HMAC_TRUNC_LEN)
1513 cs->cs_config |= CESA_CSHD_96_BIT_HMAC;
1515 case CRYPTO_SHA2_256_HMAC:
1516 cs->cs_mblen = SHA2_256_HMAC_BLOCK_LEN;
1517 cs->cs_hlen = (mac->cri_mlen == 0) ? SHA2_256_HASH_LEN :
1519 cs->cs_config |= CESA_CSHD_SHA2_256_HMAC;
1527 /* Save cipher key */
1528 if (!error && enc && enc->cri_key) {
1529 cs->cs_klen = enc->cri_klen / 8;
1530 memcpy(cs->cs_key, enc->cri_key, cs->cs_klen);
1531 if (enc->cri_alg == CRYPTO_AES_CBC)
1532 error = cesa_prep_aes_key(cs);
1535 /* Save digest key */
1536 if (!error && mac && mac->cri_key)
1537 error = cesa_set_mkey(cs, mac->cri_alg, mac->cri_key,
1541 cesa_free_session(sc, cs);
1551 cesa_freesession(device_t dev, uint64_t tid)
1553 struct cesa_session *cs;
1554 struct cesa_softc *sc;
1556 sc = device_get_softc(dev);
1557 cs = cesa_get_session(sc, CRYPTO_SESID2LID(tid));
1562 cesa_free_session(sc, cs);
1568 cesa_process(device_t dev, struct cryptop *crp, int hint)
1570 struct cesa_request *cr;
1571 struct cesa_session *cs;
1572 struct cryptodesc *crd;
1573 struct cryptodesc *enc;
1574 struct cryptodesc *mac;
1575 struct cesa_softc *sc;
1578 sc = device_get_softc(dev);
1579 crd = crp->crp_desc;
1584 /* Check session ID */
1585 cs = cesa_get_session(sc, CRYPTO_SESID2LID(crp->crp_sid));
1587 crp->crp_etype = EINVAL;
1592 /* Check and parse input */
1593 if (crp->crp_ilen > CESA_MAX_REQUEST_SIZE) {
1594 crp->crp_etype = E2BIG;
1599 if (cesa_is_hash(crd->crd_alg))
1604 crd = crd->crd_next;
1607 if (!enc && !cesa_is_hash(crd->crd_alg))
1610 if (!mac && cesa_is_hash(crd->crd_alg))
1613 if (crd->crd_next || !(enc && mac)) {
1614 crp->crp_etype = EINVAL;
1621 * Get request descriptor. Block driver if there is no free
1622 * descriptors in pool.
1624 cr = cesa_alloc_request(sc);
1627 sc->sc_blocked = CRYPTO_SYMQ;
1628 CESA_UNLOCK(sc, sc);
1632 /* Prepare request */
1638 CESA_LOCK(sc, sessions);
1639 cesa_sync_desc(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1641 if (enc && enc->crd_flags & CRD_F_ENCRYPT) {
1642 if (enc->crd_flags & CRD_F_IV_EXPLICIT)
1643 memcpy(cr->cr_csd->csd_iv, enc->crd_iv, cs->cs_ivlen);
1645 arc4rand(cr->cr_csd->csd_iv, cs->cs_ivlen, 0);
1647 if ((enc->crd_flags & CRD_F_IV_PRESENT) == 0)
1648 crypto_copyback(crp->crp_flags, crp->crp_buf,
1649 enc->crd_inject, cs->cs_ivlen, cr->cr_csd->csd_iv);
1651 if (enc->crd_flags & CRD_F_IV_EXPLICIT)
1652 memcpy(cr->cr_csd->csd_iv, enc->crd_iv, cs->cs_ivlen);
1654 crypto_copydata(crp->crp_flags, crp->crp_buf,
1655 enc->crd_inject, cs->cs_ivlen, cr->cr_csd->csd_iv);
1658 if (enc && enc->crd_flags & CRD_F_KEY_EXPLICIT) {
1659 if ((enc->crd_klen / 8) <= CESA_MAX_KEY_LEN) {
1660 cs->cs_klen = enc->crd_klen / 8;
1661 memcpy(cs->cs_key, enc->crd_key, cs->cs_klen);
1662 if (enc->crd_alg == CRYPTO_AES_CBC)
1663 error = cesa_prep_aes_key(cs);
1668 if (!error && mac && mac->crd_flags & CRD_F_KEY_EXPLICIT) {
1669 if ((mac->crd_klen / 8) <= CESA_MAX_MKEY_LEN)
1670 error = cesa_set_mkey(cs, mac->crd_alg, mac->crd_key,
1676 /* Convert request to chain of TDMA and SA descriptors */
1678 error = cesa_create_chain(sc, cr);
1680 cesa_sync_desc(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1681 CESA_UNLOCK(sc, sessions);
1684 cesa_free_request(sc, cr);
1685 crp->crp_etype = error;
1690 bus_dmamap_sync(sc->sc_data_dtag, cr->cr_dmap, BUS_DMASYNC_PREREAD |
1691 BUS_DMASYNC_PREWRITE);
1693 /* Enqueue request to execution */
1694 cesa_enqueue_request(sc, cr);
1696 /* Start execution, if we have no more requests in queue */
1697 if ((hint & CRYPTO_HINT_MORE) == 0)