1 /* $OpenBSD: criov.c,v 1.9 2002/01/29 15:48:29 jason Exp $ */
4 * Copyright (c) 1999 Theo de Raadt
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. The name of the author may not be used to endorse or promote products
16 * derived from this software without specific prior written permission.
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
33 #include <sys/param.h>
34 #include <sys/systm.h>
36 #include <sys/errno.h>
37 #include <sys/malloc.h>
38 #include <sys/kernel.h>
41 #include <sys/limits.h>
45 #include <machine/vmparam.h>
48 #include <vm/vm_page.h>
51 #include <opencrypto/cryptodev.h>
53 SDT_PROVIDER_DECLARE(opencrypto);
56 * These macros are only for avoiding code duplication, as we need to skip
57 * given number of bytes in the same way in several functions below.
59 #define CUIO_SKIP() do { \
60 KASSERT(off >= 0, ("%s: off %d < 0", __func__, off)); \
61 KASSERT(len >= 0, ("%s: len %d < 0", __func__, len)); \
63 KASSERT(iol >= 0, ("%s: empty in skip", __func__)); \
64 if (off < iov->iov_len) \
66 off -= iov->iov_len; \
72 #define CVM_PAGE_SKIP() do { \
73 KASSERT(off >= 0, ("%s: off %d < 0", __func__, off)); \
74 KASSERT(len >= 0, ("%s: len %d < 0", __func__, len)); \
76 if (off < PAGE_SIZE) \
78 processed += PAGE_SIZE - off; \
79 off -= PAGE_SIZE - off; \
85 cuio_copydata(struct uio* uio, int off, int len, caddr_t cp)
87 struct iovec *iov = uio->uio_iov;
88 int iol = uio->uio_iovcnt;
93 KASSERT(iol >= 0, ("%s: empty", __func__));
94 count = min(iov->iov_len - off, len);
95 bcopy(((caddr_t)iov->iov_base) + off, cp, count);
105 cuio_copyback(struct uio* uio, int off, int len, c_caddr_t cp)
107 struct iovec *iov = uio->uio_iov;
108 int iol = uio->uio_iovcnt;
113 KASSERT(iol >= 0, ("%s: empty", __func__));
114 count = min(iov->iov_len - off, len);
115 bcopy(cp, ((caddr_t)iov->iov_base) + off, count);
125 * Return the index and offset of location in iovec list.
128 cuio_getptr(struct uio *uio, int loc, int *off)
133 while (loc >= 0 && ind < uio->uio_iovcnt) {
134 len = uio->uio_iov[ind].iov_len;
143 if (ind > 0 && loc == 0) {
145 *off = uio->uio_iov[ind].iov_len;
152 #if CRYPTO_MAY_HAVE_VMPAGE
154 * Apply function f to the data in a vm_page_t list starting "off" bytes from
155 * the beginning, continuing for "len" bytes.
158 cvm_page_apply(vm_page_t *pages, int off, int len,
159 int (*f)(void *, const void *, u_int), void *arg)
167 char *kaddr = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages));
168 count = min(PAGE_SIZE - off, len);
169 rval = (*f)(arg, kaddr + off, count);
181 cvm_page_contiguous_segment(vm_page_t *pages, size_t skip, int len)
183 if ((skip + len - 1) / PAGE_SIZE > skip / PAGE_SIZE)
186 pages += (skip / PAGE_SIZE);
187 skip -= rounddown(skip, PAGE_SIZE);
188 return (((char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages))) + skip);
192 * Copy len bytes of data from the vm_page_t array, skipping the first off
193 * bytes, into the pointer cp. Return the number of bytes skipped and copied.
194 * Does not verify the length of the array.
197 cvm_page_copyback(vm_page_t *pages, int off, int len, c_caddr_t cp)
204 count = min(PAGE_SIZE - off, len);
205 bcopy(cp, (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages)) + off,
217 * Copy len bytes of data from the pointer cp into the vm_page_t array,
218 * skipping the first off bytes, Return the number of bytes skipped and copied.
219 * Does not verify the length of the array.
222 cvm_page_copydata(vm_page_t *pages, int off, int len, caddr_t cp)
229 count = min(PAGE_SIZE - off, len);
230 bcopy(((char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(*pages)) + off), cp,
240 #endif /* CRYPTO_MAY_HAVE_VMPAGE */
243 crypto_cursor_init(struct crypto_buffer_cursor *cc,
244 const struct crypto_buffer *cb)
246 memset(cc, 0, sizeof(*cc));
247 cc->cc_type = cb->cb_type;
248 switch (cc->cc_type) {
249 case CRYPTO_BUF_CONTIG:
250 cc->cc_buf = cb->cb_buf;
251 cc->cc_buf_len = cb->cb_buf_len;
253 case CRYPTO_BUF_MBUF:
254 cc->cc_mbuf = cb->cb_mbuf;
256 case CRYPTO_BUF_VMPAGE:
257 cc->cc_vmpage = cb->cb_vm_page;
258 cc->cc_buf_len = cb->cb_vm_page_len;
259 cc->cc_offset = cb->cb_vm_page_offset;
262 cc->cc_iov = cb->cb_uio->uio_iov;
266 panic("%s: invalid buffer type %d", __func__, cb->cb_type);
272 SDT_PROBE_DEFINE2(opencrypto, criov, cursor_advance, vmpage, "struct crypto_buffer_cursor*", "size_t");
275 crypto_cursor_advance(struct crypto_buffer_cursor *cc, size_t amount)
279 switch (cc->cc_type) {
280 case CRYPTO_BUF_CONTIG:
281 MPASS(cc->cc_buf_len >= amount);
282 cc->cc_buf += amount;
283 cc->cc_buf_len -= amount;
285 case CRYPTO_BUF_MBUF:
287 remain = cc->cc_mbuf->m_len - cc->cc_offset;
288 if (amount < remain) {
289 cc->cc_offset += amount;
293 cc->cc_mbuf = cc->cc_mbuf->m_next;
299 case CRYPTO_BUF_VMPAGE:
301 SDT_PROBE2(opencrypto, criov, cursor_advance, vmpage,
303 remain = MIN(PAGE_SIZE - cc->cc_offset, cc->cc_buf_len);
304 if (amount < remain) {
305 cc->cc_buf_len -= amount;
306 cc->cc_offset += amount;
309 cc->cc_buf_len -= remain;
313 if (amount == 0 || cc->cc_buf_len == 0)
319 remain = cc->cc_iov->iov_len - cc->cc_offset;
320 if (amount < remain) {
321 cc->cc_offset += amount;
333 panic("%s: invalid buffer type %d", __func__, cc->cc_type);
340 crypto_cursor_segbase(struct crypto_buffer_cursor *cc)
342 switch (cc->cc_type) {
343 case CRYPTO_BUF_CONTIG:
345 case CRYPTO_BUF_MBUF:
346 if (cc->cc_mbuf == NULL)
348 KASSERT((cc->cc_mbuf->m_flags & M_EXTPG) == 0,
349 ("%s: not supported for unmapped mbufs", __func__));
350 return (mtod(cc->cc_mbuf, char *) + cc->cc_offset);
351 case CRYPTO_BUF_VMPAGE:
352 return ((char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(
353 *cc->cc_vmpage)) + cc->cc_offset);
355 return ((char *)cc->cc_iov->iov_base + cc->cc_offset);
358 panic("%s: invalid buffer type %d", __func__, cc->cc_type);
365 crypto_cursor_seglen(struct crypto_buffer_cursor *cc)
367 switch (cc->cc_type) {
368 case CRYPTO_BUF_CONTIG:
369 return (cc->cc_buf_len);
370 case CRYPTO_BUF_VMPAGE:
371 return (PAGE_SIZE - cc->cc_offset);
372 case CRYPTO_BUF_MBUF:
373 if (cc->cc_mbuf == NULL)
375 return (cc->cc_mbuf->m_len - cc->cc_offset);
377 return (cc->cc_iov->iov_len - cc->cc_offset);
380 panic("%s: invalid buffer type %d", __func__, cc->cc_type);
387 crypto_cursor_copyback(struct crypto_buffer_cursor *cc, int size,
395 switch (cc->cc_type) {
396 case CRYPTO_BUF_CONTIG:
397 MPASS(cc->cc_buf_len >= size);
398 memcpy(cc->cc_buf, src, size);
400 cc->cc_buf_len -= size;
402 case CRYPTO_BUF_MBUF:
404 KASSERT((cc->cc_mbuf->m_flags & M_EXTPG) == 0,
405 ("%s: not supported for unmapped mbufs", __func__));
406 dst = mtod(cc->cc_mbuf, char *) + cc->cc_offset;
407 remain = cc->cc_mbuf->m_len - cc->cc_offset;
408 todo = MIN(remain, size);
409 memcpy(dst, src, todo);
412 cc->cc_offset += todo;
416 cc->cc_mbuf = cc->cc_mbuf->m_next;
422 case CRYPTO_BUF_VMPAGE:
424 dst = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(
425 *cc->cc_vmpage)) + cc->cc_offset;
426 remain = MIN(PAGE_SIZE - cc->cc_offset, cc->cc_buf_len);
427 todo = MIN(remain, size);
428 memcpy(dst, src, todo);
430 cc->cc_buf_len -= todo;
432 cc->cc_offset += todo;
444 dst = (char *)cc->cc_iov->iov_base + cc->cc_offset;
445 remain = cc->cc_iov->iov_len - cc->cc_offset;
446 todo = MIN(remain, size);
447 memcpy(dst, src, todo);
450 cc->cc_offset += todo;
462 panic("%s: invalid buffer type %d", __func__, cc->cc_type);
469 crypto_cursor_copydata(struct crypto_buffer_cursor *cc, int size, void *vdst)
476 switch (cc->cc_type) {
477 case CRYPTO_BUF_CONTIG:
478 MPASS(cc->cc_buf_len >= size);
479 memcpy(dst, cc->cc_buf, size);
481 cc->cc_buf_len -= size;
483 case CRYPTO_BUF_MBUF:
485 KASSERT((cc->cc_mbuf->m_flags & M_EXTPG) == 0,
486 ("%s: not supported for unmapped mbufs", __func__));
487 src = mtod(cc->cc_mbuf, const char *) + cc->cc_offset;
488 remain = cc->cc_mbuf->m_len - cc->cc_offset;
489 todo = MIN(remain, size);
490 memcpy(dst, src, todo);
493 cc->cc_offset += todo;
497 cc->cc_mbuf = cc->cc_mbuf->m_next;
503 case CRYPTO_BUF_VMPAGE:
505 src = (char *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(
506 *cc->cc_vmpage)) + cc->cc_offset;
507 remain = MIN(PAGE_SIZE - cc->cc_offset, cc->cc_buf_len);
508 todo = MIN(remain, size);
509 memcpy(dst, src, todo);
511 cc->cc_buf_len -= todo;
513 cc->cc_offset += todo;
525 src = (const char *)cc->cc_iov->iov_base +
527 remain = cc->cc_iov->iov_len - cc->cc_offset;
528 todo = MIN(remain, size);
529 memcpy(dst, src, todo);
532 cc->cc_offset += todo;
544 panic("%s: invalid buffer type %d", __func__, cc->cc_type);
551 * To avoid advancing 'cursor', make a local copy that gets advanced
555 crypto_cursor_copydata_noadv(struct crypto_buffer_cursor *cc, int size,
558 struct crypto_buffer_cursor copy;
561 crypto_cursor_copydata(©, size, vdst);
565 * Apply function f to the data in an iovec list starting "off" bytes from
566 * the beginning, continuing for "len" bytes.
569 cuio_apply(struct uio *uio, int off, int len,
570 int (*f)(void *, const void *, u_int), void *arg)
572 struct iovec *iov = uio->uio_iov;
573 int iol = uio->uio_iovcnt;
579 KASSERT(iol >= 0, ("%s: empty", __func__));
580 count = min(iov->iov_len - off, len);
581 rval = (*f)(arg, ((caddr_t)iov->iov_base) + off, count);
593 crypto_copyback(struct cryptop *crp, int off, int size, const void *src)
595 struct crypto_buffer *cb;
597 if (crp->crp_obuf.cb_type != CRYPTO_BUF_NONE)
601 switch (cb->cb_type) {
602 case CRYPTO_BUF_MBUF:
603 m_copyback(cb->cb_mbuf, off, size, src);
605 #if CRYPTO_MAY_HAVE_VMPAGE
606 case CRYPTO_BUF_VMPAGE:
607 MPASS(size <= cb->cb_vm_page_len);
609 cb->cb_vm_page_len + cb->cb_vm_page_offset);
610 cvm_page_copyback(cb->cb_vm_page,
611 off + cb->cb_vm_page_offset, size, src);
613 #endif /* CRYPTO_MAY_HAVE_VMPAGE */
615 cuio_copyback(cb->cb_uio, off, size, src);
617 case CRYPTO_BUF_CONTIG:
618 MPASS(off + size <= cb->cb_buf_len);
619 bcopy(src, cb->cb_buf + off, size);
623 panic("invalid crp buf type %d", cb->cb_type);
630 crypto_copydata(struct cryptop *crp, int off, int size, void *dst)
633 switch (crp->crp_buf.cb_type) {
634 case CRYPTO_BUF_MBUF:
635 m_copydata(crp->crp_buf.cb_mbuf, off, size, dst);
637 #if CRYPTO_MAY_HAVE_VMPAGE
638 case CRYPTO_BUF_VMPAGE:
639 MPASS(size <= crp->crp_buf.cb_vm_page_len);
640 MPASS(size + off <= crp->crp_buf.cb_vm_page_len +
641 crp->crp_buf.cb_vm_page_offset);
642 cvm_page_copydata(crp->crp_buf.cb_vm_page,
643 off + crp->crp_buf.cb_vm_page_offset, size, dst);
645 #endif /* CRYPTO_MAY_HAVE_VMPAGE */
647 cuio_copydata(crp->crp_buf.cb_uio, off, size, dst);
649 case CRYPTO_BUF_CONTIG:
650 MPASS(off + size <= crp->crp_buf.cb_buf_len);
651 bcopy(crp->crp_buf.cb_buf + off, dst, size);
655 panic("invalid crp buf type %d", crp->crp_buf.cb_type);
662 crypto_apply_buf(struct crypto_buffer *cb, int off, int len,
663 int (*f)(void *, const void *, u_int), void *arg)
667 switch (cb->cb_type) {
668 case CRYPTO_BUF_MBUF:
669 error = m_apply(cb->cb_mbuf, off, len,
670 (int (*)(void *, void *, u_int))f, arg);
673 error = cuio_apply(cb->cb_uio, off, len, f, arg);
675 #if CRYPTO_MAY_HAVE_VMPAGE
676 case CRYPTO_BUF_VMPAGE:
677 error = cvm_page_apply(cb->cb_vm_page,
678 off + cb->cb_vm_page_offset, len, f, arg);
680 #endif /* CRYPTO_MAY_HAVE_VMPAGE */
681 case CRYPTO_BUF_CONTIG:
682 MPASS(off + len <= cb->cb_buf_len);
683 error = (*f)(arg, cb->cb_buf + off, len);
687 panic("invalid crypto buf type %d", cb->cb_type);
696 crypto_apply(struct cryptop *crp, int off, int len,
697 int (*f)(void *, const void *, u_int), void *arg)
699 return (crypto_apply_buf(&crp->crp_buf, off, len, f, arg));
703 m_contiguous_subsegment(struct mbuf *m, size_t skip, size_t len)
707 MPASS(skip <= INT_MAX);
709 m = m_getptr(m, (int)skip, &rel_off);
715 if (skip + len > m->m_len)
718 return (mtod(m, char*) + skip);
722 cuio_contiguous_segment(struct uio *uio, size_t skip, size_t len)
726 MPASS(skip <= INT_MAX);
727 idx = cuio_getptr(uio, (int)skip, &rel_off);
733 if (skip + len > uio->uio_iov[idx].iov_len)
735 return ((char *)uio->uio_iov[idx].iov_base + skip);
739 crypto_buffer_contiguous_subsegment(struct crypto_buffer *cb, size_t skip,
743 switch (cb->cb_type) {
744 case CRYPTO_BUF_MBUF:
745 return (m_contiguous_subsegment(cb->cb_mbuf, skip, len));
747 return (cuio_contiguous_segment(cb->cb_uio, skip, len));
748 #if CRYPTO_MAY_HAVE_VMPAGE
749 case CRYPTO_BUF_VMPAGE:
750 MPASS(skip + len <= cb->cb_vm_page_len);
751 return (cvm_page_contiguous_segment(cb->cb_vm_page,
752 skip + cb->cb_vm_page_offset, len));
753 #endif /* CRYPTO_MAY_HAVE_VMPAGE */
754 case CRYPTO_BUF_CONTIG:
755 MPASS(skip + len <= cb->cb_buf_len);
756 return (cb->cb_buf + skip);
759 panic("invalid crp buf type %d", cb->cb_type);
766 crypto_contiguous_subsegment(struct cryptop *crp, size_t skip, size_t len)
768 return (crypto_buffer_contiguous_subsegment(&crp->crp_buf, skip, len));