3 * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 #ifdef USB_GLOBAL_INCLUDE_FILE
28 #include USB_GLOBAL_INCLUDE_FILE
30 #include <sys/stdint.h>
31 #include <sys/stddef.h>
32 #include <sys/param.h>
33 #include <sys/queue.h>
34 #include <sys/types.h>
35 #include <sys/systm.h>
36 #include <sys/kernel.h>
38 #include <sys/module.h>
40 #include <sys/mutex.h>
41 #include <sys/condvar.h>
42 #include <sys/sysctl.h>
44 #include <sys/unistd.h>
45 #include <sys/callout.h>
46 #include <sys/malloc.h>
49 #include <dev/usb/usb.h>
50 #include <dev/usb/usbdi.h>
51 #include <dev/usb/usbdi_util.h>
53 #define USB_DEBUG_VAR usb_debug
55 #include <dev/usb/usb_core.h>
56 #include <dev/usb/usb_busdma.h>
57 #include <dev/usb/usb_process.h>
58 #include <dev/usb/usb_transfer.h>
59 #include <dev/usb/usb_device.h>
60 #include <dev/usb/usb_util.h>
61 #include <dev/usb/usb_debug.h>
63 #include <dev/usb/usb_controller.h>
64 #include <dev/usb/usb_bus.h>
65 #endif /* USB_GLOBAL_INCLUDE_FILE */
68 static void usb_dma_tag_create(struct usb_dma_tag *, usb_size_t, usb_size_t);
69 static void usb_dma_tag_destroy(struct usb_dma_tag *);
70 static void usb_dma_lock_cb(void *, bus_dma_lock_op_t);
71 static void usb_pc_alloc_mem_cb(void *, bus_dma_segment_t *, int, int);
72 static void usb_pc_load_mem_cb(void *, bus_dma_segment_t *, int, int);
73 static void usb_pc_common_mem_cb(void *, bus_dma_segment_t *, int, int,
77 /*------------------------------------------------------------------------*
78 * usbd_get_page - lookup DMA-able memory for the given offset
80 * NOTE: Only call this function when the "page_cache" structure has
81 * been properly initialized !
82 *------------------------------------------------------------------------*/
84 usbd_get_page(struct usb_page_cache *pc, usb_frlength_t offset,
85 struct usb_page_search *res)
88 struct usb_page *page;
92 /* Case 1 - something has been loaded into DMA */
96 /* Case 1a - Kernel Virtual Address */
98 res->buffer = USB_ADD_BYTES(pc->buffer, offset);
100 offset += pc->page_offset_buf;
102 /* compute destination page */
104 page = pc->page_start;
106 if (pc->ismultiseg) {
108 page += (offset / USB_PAGE_SIZE);
110 offset %= USB_PAGE_SIZE;
112 res->length = USB_PAGE_SIZE - offset;
113 res->physaddr = page->physaddr + offset;
115 res->length = (usb_size_t)-1;
116 res->physaddr = page->physaddr + offset;
120 /* Case 1b - Non Kernel Virtual Address */
122 res->buffer = USB_ADD_BYTES(page->buffer, offset);
127 /* Case 2 - Plain PIO */
129 res->buffer = USB_ADD_BYTES(pc->buffer, offset);
130 res->length = (usb_size_t)-1;
136 /*------------------------------------------------------------------------*
137 * usb_pc_buffer_is_aligned - verify alignment
139 * This function is used to check if a page cache buffer is properly
140 * aligned to reduce the use of bounce buffers in PIO mode.
141 *------------------------------------------------------------------------*/
143 usb_pc_buffer_is_aligned(struct usb_page_cache *pc, usb_frlength_t offset,
144 usb_frlength_t len, usb_frlength_t mask)
146 struct usb_page_search buf_res;
150 usbd_get_page(pc, offset, &buf_res);
152 if (buf_res.length > len)
153 buf_res.length = len;
154 if (USB_P2U(buf_res.buffer) & mask)
156 if (buf_res.length & mask)
159 offset += buf_res.length;
160 len -= buf_res.length;
165 /*------------------------------------------------------------------------*
166 * usbd_copy_in - copy directly to DMA-able memory
167 *------------------------------------------------------------------------*/
169 usbd_copy_in(struct usb_page_cache *cache, usb_frlength_t offset,
170 const void *ptr, usb_frlength_t len)
172 struct usb_page_search buf_res;
176 usbd_get_page(cache, offset, &buf_res);
178 if (buf_res.length > len) {
179 buf_res.length = len;
181 memcpy(buf_res.buffer, ptr, buf_res.length);
183 offset += buf_res.length;
184 len -= buf_res.length;
185 ptr = USB_ADD_BYTES(ptr, buf_res.length);
189 /*------------------------------------------------------------------------*
190 * usbd_copy_in_user - copy directly to DMA-able memory from userland
195 *------------------------------------------------------------------------*/
198 usbd_copy_in_user(struct usb_page_cache *cache, usb_frlength_t offset,
199 const void *ptr, usb_frlength_t len)
201 struct usb_page_search buf_res;
206 usbd_get_page(cache, offset, &buf_res);
208 if (buf_res.length > len) {
209 buf_res.length = len;
211 error = copyin(ptr, buf_res.buffer, buf_res.length);
215 offset += buf_res.length;
216 len -= buf_res.length;
217 ptr = USB_ADD_BYTES(ptr, buf_res.length);
219 return (0); /* success */
223 /*------------------------------------------------------------------------*
224 * usbd_m_copy_in - copy a mbuf chain directly into DMA-able memory
225 *------------------------------------------------------------------------*/
227 struct usb_m_copy_in_arg {
228 struct usb_page_cache *cache;
229 usb_frlength_t dst_offset;
233 usbd_m_copy_in_cb(void *arg, void *src, uint32_t count)
235 register struct usb_m_copy_in_arg *ua = arg;
237 usbd_copy_in(ua->cache, ua->dst_offset, src, count);
238 ua->dst_offset += count;
243 usbd_m_copy_in(struct usb_page_cache *cache, usb_frlength_t dst_offset,
244 struct mbuf *m, usb_size_t src_offset, usb_frlength_t src_len)
246 struct usb_m_copy_in_arg arg = {cache, dst_offset};
247 (void) m_apply(m, src_offset, src_len, &usbd_m_copy_in_cb, &arg);
251 /*------------------------------------------------------------------------*
252 * usb_uiomove - factored out code
253 *------------------------------------------------------------------------*/
256 usb_uiomove(struct usb_page_cache *pc, struct uio *uio,
257 usb_frlength_t pc_offset, usb_frlength_t len)
259 struct usb_page_search res;
264 usbd_get_page(pc, pc_offset, &res);
266 if (res.length > len) {
270 * "uiomove()" can sleep so one needs to make a wrapper,
271 * exiting the mutex and checking things
273 error = uiomove(res.buffer, res.length, uio);
278 pc_offset += res.length;
285 /*------------------------------------------------------------------------*
286 * usbd_copy_out - copy directly from DMA-able memory
287 *------------------------------------------------------------------------*/
289 usbd_copy_out(struct usb_page_cache *cache, usb_frlength_t offset,
290 void *ptr, usb_frlength_t len)
292 struct usb_page_search res;
296 usbd_get_page(cache, offset, &res);
298 if (res.length > len) {
301 memcpy(ptr, res.buffer, res.length);
303 offset += res.length;
305 ptr = USB_ADD_BYTES(ptr, res.length);
309 /*------------------------------------------------------------------------*
310 * usbd_copy_out_user - copy directly from DMA-able memory to userland
315 *------------------------------------------------------------------------*/
318 usbd_copy_out_user(struct usb_page_cache *cache, usb_frlength_t offset,
319 void *ptr, usb_frlength_t len)
321 struct usb_page_search res;
326 usbd_get_page(cache, offset, &res);
328 if (res.length > len) {
331 error = copyout(res.buffer, ptr, res.length);
335 offset += res.length;
337 ptr = USB_ADD_BYTES(ptr, res.length);
339 return (0); /* success */
343 /*------------------------------------------------------------------------*
344 * usbd_frame_zero - zero DMA-able memory
345 *------------------------------------------------------------------------*/
347 usbd_frame_zero(struct usb_page_cache *cache, usb_frlength_t offset,
350 struct usb_page_search res;
354 usbd_get_page(cache, offset, &res);
356 if (res.length > len) {
359 memset(res.buffer, 0, res.length);
361 offset += res.length;
368 /*------------------------------------------------------------------------*
369 * usb_dma_lock_cb - dummy callback
370 *------------------------------------------------------------------------*/
372 usb_dma_lock_cb(void *arg, bus_dma_lock_op_t op)
374 /* we use "mtx_owned()" instead of this function */
377 /*------------------------------------------------------------------------*
378 * usb_dma_tag_create - allocate a DMA tag
380 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will
381 * allow multi-segment mappings. Else all mappings are single-segment.
382 *------------------------------------------------------------------------*/
384 usb_dma_tag_create(struct usb_dma_tag *udt,
385 usb_size_t size, usb_size_t align)
389 if (bus_dma_tag_create
390 ( /* parent */ udt->tag_parent->tag,
391 /* alignment */ align,
393 /* lowaddr */ (2ULL << (udt->tag_parent->dma_bits - 1)) - 1,
394 /* highaddr */ BUS_SPACE_MAXADDR,
396 /* filterarg */ NULL,
398 /* nsegments */ (align == 1 && size > 1) ?
399 (2 + (size / USB_PAGE_SIZE)) : 1,
400 /* maxsegsz */ (align == 1 && size > USB_PAGE_SIZE) ?
401 USB_PAGE_SIZE : size,
402 /* flags */ BUS_DMA_KEEP_PG_OFFSET,
403 /* lockfn */ &usb_dma_lock_cb,
411 /*------------------------------------------------------------------------*
412 * usb_dma_tag_free - free a DMA tag
413 *------------------------------------------------------------------------*/
415 usb_dma_tag_destroy(struct usb_dma_tag *udt)
417 bus_dma_tag_destroy(udt->tag);
420 /*------------------------------------------------------------------------*
421 * usb_pc_alloc_mem_cb - BUS-DMA callback function
422 *------------------------------------------------------------------------*/
424 usb_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs,
427 usb_pc_common_mem_cb(arg, segs, nseg, error, 0);
430 /*------------------------------------------------------------------------*
431 * usb_pc_load_mem_cb - BUS-DMA callback function
432 *------------------------------------------------------------------------*/
434 usb_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs,
437 usb_pc_common_mem_cb(arg, segs, nseg, error, 1);
440 /*------------------------------------------------------------------------*
441 * usb_pc_common_mem_cb - BUS-DMA callback function
442 *------------------------------------------------------------------------*/
444 usb_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs,
445 int nseg, int error, uint8_t isload)
447 struct usb_dma_parent_tag *uptag;
448 struct usb_page_cache *pc;
455 uptag = pc->tag_parent;
458 * XXX There is sometimes recursive locking here.
459 * XXX We should try to find a better solution.
460 * XXX Until further the "owned" variable does
470 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1);
471 rem = segs->ds_addr & (USB_PAGE_SIZE - 1);
472 pc->page_offset_buf = rem;
473 pc->page_offset_end += rem;
476 ((segs->ds_addr + segs->ds_len) & (USB_PAGE_SIZE - 1)) !=
477 ((segs + 1)->ds_addr & (USB_PAGE_SIZE - 1))) {
479 * This check verifies there is no page offset hole
480 * between the first and second segment. See the
481 * BUS_DMA_KEEP_PG_OFFSET flag.
483 DPRINTFN(0, "Page offset was not preserved\n");
488 while (pc->ismultiseg) {
489 off += USB_PAGE_SIZE;
490 if (off >= (segs->ds_len + rem)) {
500 pg->physaddr = (segs->ds_addr + off) & ~(USB_PAGE_SIZE - 1);
504 owned = mtx_owned(uptag->mtx);
506 mtx_lock(uptag->mtx);
508 uptag->dma_error = (error ? 1 : 0);
510 (uptag->func) (uptag);
512 cv_broadcast(uptag->cv);
515 mtx_unlock(uptag->mtx);
518 /*------------------------------------------------------------------------*
519 * usb_pc_alloc_mem - allocate DMA'able memory
524 *------------------------------------------------------------------------*/
526 usb_pc_alloc_mem(struct usb_page_cache *pc, struct usb_page *pg,
527 usb_size_t size, usb_size_t align)
529 struct usb_dma_parent_tag *uptag;
530 struct usb_dma_tag *utag;
535 uptag = pc->tag_parent;
539 * The alignment must be greater or equal to the
540 * "size" else the object can be split between two
541 * memory pages and we get a problem!
543 while (align < size) {
551 * XXX BUS-DMA workaround - FIXME later:
553 * We assume that that the aligment at this point of
554 * the code is greater than or equal to the size and
555 * less than two times the size, so that if we double
556 * the size, the size will be greater than the
559 * The bus-dma system has a check for "alignment"
560 * being less than "size". If that check fails we end
561 * up using contigmalloc which is page based even for
562 * small allocations. Try to avoid that to save
563 * memory, hence we sometimes to a large number of
566 if (size <= (USB_PAGE_SIZE / 2)) {
571 /* get the correct DMA tag */
572 utag = usb_dma_tag_find(uptag, size, align);
576 /* allocate memory */
577 if (bus_dmamem_alloc(
578 utag->tag, &ptr, (BUS_DMA_WAITOK | BUS_DMA_COHERENT), &map)) {
581 /* setup page cache */
584 pc->page_offset_buf = 0;
585 pc->page_offset_end = size;
588 pc->ismultiseg = (align == 1);
590 mtx_lock(uptag->mtx);
592 /* load memory into DMA */
593 err = bus_dmamap_load(
594 utag->tag, map, ptr, size, &usb_pc_alloc_mem_cb,
595 pc, (BUS_DMA_WAITOK | BUS_DMA_COHERENT));
597 if (err == EINPROGRESS) {
598 cv_wait(uptag->cv, uptag->mtx);
601 mtx_unlock(uptag->mtx);
603 if (err || uptag->dma_error) {
604 bus_dmamem_free(utag->tag, ptr, map);
607 memset(ptr, 0, size);
609 usb_pc_cpu_flush(pc);
614 /* reset most of the page cache */
616 pc->page_start = NULL;
617 pc->page_offset_buf = 0;
618 pc->page_offset_end = 0;
624 /*------------------------------------------------------------------------*
625 * usb_pc_free_mem - free DMA memory
627 * This function is NULL safe.
628 *------------------------------------------------------------------------*/
630 usb_pc_free_mem(struct usb_page_cache *pc)
632 if (pc && pc->buffer) {
634 bus_dmamap_unload(pc->tag, pc->map);
636 bus_dmamem_free(pc->tag, pc->buffer, pc->map);
642 /*------------------------------------------------------------------------*
643 * usb_pc_load_mem - load virtual memory into DMA
648 *------------------------------------------------------------------------*/
650 usb_pc_load_mem(struct usb_page_cache *pc, usb_size_t size, uint8_t sync)
652 /* setup page cache */
653 pc->page_offset_buf = 0;
654 pc->page_offset_end = size;
657 mtx_assert(pc->tag_parent->mtx, MA_OWNED);
661 struct usb_dma_parent_tag *uptag;
664 uptag = pc->tag_parent;
667 * We have to unload the previous loaded DMA
668 * pages before trying to load a new one!
670 bus_dmamap_unload(pc->tag, pc->map);
673 * Try to load memory into DMA.
675 err = bus_dmamap_load(
676 pc->tag, pc->map, pc->buffer, size,
677 &usb_pc_alloc_mem_cb, pc, BUS_DMA_WAITOK);
678 if (err == EINPROGRESS) {
679 cv_wait(uptag->cv, uptag->mtx);
682 if (err || uptag->dma_error) {
688 * We have to unload the previous loaded DMA
689 * pages before trying to load a new one!
691 bus_dmamap_unload(pc->tag, pc->map);
694 * Try to load memory into DMA. The callback
695 * will be called in all cases:
698 pc->tag, pc->map, pc->buffer, size,
699 &usb_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) {
705 * Call callback so that refcount is decremented
708 pc->tag_parent->dma_error = 0;
709 (pc->tag_parent->func) (pc->tag_parent);
715 /*------------------------------------------------------------------------*
716 * usb_pc_cpu_invalidate - invalidate CPU cache
717 *------------------------------------------------------------------------*/
719 usb_pc_cpu_invalidate(struct usb_page_cache *pc)
721 if (pc->page_offset_end == pc->page_offset_buf) {
722 /* nothing has been loaded into this page cache! */
727 * TODO: We currently do XXX_POSTREAD and XXX_PREREAD at the
728 * same time, but in the future we should try to isolate the
729 * different cases to optimise the code. --HPS
731 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_POSTREAD);
732 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREREAD);
735 /*------------------------------------------------------------------------*
736 * usb_pc_cpu_flush - flush CPU cache
737 *------------------------------------------------------------------------*/
739 usb_pc_cpu_flush(struct usb_page_cache *pc)
741 if (pc->page_offset_end == pc->page_offset_buf) {
742 /* nothing has been loaded into this page cache! */
745 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREWRITE);
748 /*------------------------------------------------------------------------*
749 * usb_pc_dmamap_create - create a DMA map
754 *------------------------------------------------------------------------*/
756 usb_pc_dmamap_create(struct usb_page_cache *pc, usb_size_t size)
758 struct usb_xfer_root *info;
759 struct usb_dma_tag *utag;
762 info = USB_DMATAG_TO_XROOT(pc->tag_parent);
768 utag = usb_dma_tag_find(pc->tag_parent, size, 1);
773 if (bus_dmamap_create(utag->tag, 0, &pc->map)) {
777 return 0; /* success */
782 return 1; /* failure */
785 /*------------------------------------------------------------------------*
786 * usb_pc_dmamap_destroy
788 * This function is NULL safe.
789 *------------------------------------------------------------------------*/
791 usb_pc_dmamap_destroy(struct usb_page_cache *pc)
794 bus_dmamap_destroy(pc->tag, pc->map);
800 /*------------------------------------------------------------------------*
801 * usb_dma_tag_find - factored out code
802 *------------------------------------------------------------------------*/
804 usb_dma_tag_find(struct usb_dma_parent_tag *udpt,
805 usb_size_t size, usb_size_t align)
807 struct usb_dma_tag *udt;
810 USB_ASSERT(align > 0, ("Invalid parameter align = 0\n"));
811 USB_ASSERT(size > 0, ("Invalid parameter size = 0\n"));
813 udt = udpt->utag_first;
814 nudt = udpt->utag_max;
818 if (udt->align == 0) {
819 usb_dma_tag_create(udt, size, align);
820 if (udt->tag == NULL) {
827 if ((udt->align == align) && (udt->size == size)) {
835 /*------------------------------------------------------------------------*
836 * usb_dma_tag_setup - initialise USB DMA tags
837 *------------------------------------------------------------------------*/
839 usb_dma_tag_setup(struct usb_dma_parent_tag *udpt,
840 struct usb_dma_tag *udt, bus_dma_tag_t dmat,
841 struct mtx *mtx, usb_dma_callback_t *func,
842 uint8_t ndmabits, uint8_t nudt)
844 memset(udpt, 0, sizeof(*udpt));
846 /* sanity checking */
850 /* something is corrupt */
853 /* initialise condition variable */
854 cv_init(udpt->cv, "USB DMA CV");
856 /* store some information */
860 udpt->utag_first = udt;
861 udpt->utag_max = nudt;
862 udpt->dma_bits = ndmabits;
865 memset(udt, 0, sizeof(*udt));
866 udt->tag_parent = udpt;
871 /*------------------------------------------------------------------------*
872 * usb_bus_tag_unsetup - factored out code
873 *------------------------------------------------------------------------*/
875 usb_dma_tag_unsetup(struct usb_dma_parent_tag *udpt)
877 struct usb_dma_tag *udt;
880 udt = udpt->utag_first;
881 nudt = udpt->utag_max;
886 /* destroy the USB DMA tag */
887 usb_dma_tag_destroy(udt);
893 if (udpt->utag_max) {
894 /* destroy the condition variable */
895 cv_destroy(udpt->cv);
899 /*------------------------------------------------------------------------*
902 * This function handles loading of virtual buffers into DMA and is
903 * only called when "dma_refcount" is zero.
904 *------------------------------------------------------------------------*/
906 usb_bdma_work_loop(struct usb_xfer_queue *pq)
908 struct usb_xfer_root *info;
909 struct usb_xfer *xfer;
910 usb_frcount_t nframes;
915 mtx_assert(info->xfer_mtx, MA_OWNED);
918 /* some error happened */
919 USB_BUS_LOCK(info->bus);
920 usbd_transfer_done(xfer, 0);
921 USB_BUS_UNLOCK(info->bus);
924 if (!xfer->flags_int.bdma_setup) {
926 usb_frlength_t frlength_0;
929 xfer->flags_int.bdma_setup = 1;
931 /* reset BUS-DMA load state */
935 if (xfer->flags_int.isochronous_xfr) {
936 /* only one frame buffer */
938 frlength_0 = xfer->sumlen;
940 /* can be multiple frame buffers */
941 nframes = xfer->nframes;
942 frlength_0 = xfer->frlengths[0];
946 * Set DMA direction first. This is needed to
947 * select the correct cache invalidate and cache
950 isread = USB_GET_DATA_ISREAD(xfer);
951 pg = xfer->dma_page_ptr;
953 if (xfer->flags_int.control_xfr &&
954 xfer->flags_int.control_hdr) {
956 if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
957 /* The device controller writes to memory */
958 xfer->frbuffers[0].isread = 1;
960 /* The host controller reads from memory */
961 xfer->frbuffers[0].isread = 0;
965 xfer->frbuffers[0].isread = isread;
969 * Setup the "page_start" pointer which points to an array of
970 * USB pages where information about the physical address of a
971 * page will be stored. Also initialise the "isread" field of
972 * the USB page caches.
974 xfer->frbuffers[0].page_start = pg;
976 info->dma_nframes = nframes;
977 info->dma_currframe = 0;
978 info->dma_frlength_0 = frlength_0;
980 pg += (frlength_0 / USB_PAGE_SIZE);
983 while (--nframes > 0) {
984 xfer->frbuffers[nframes].isread = isread;
985 xfer->frbuffers[nframes].page_start = pg;
987 pg += (xfer->frlengths[nframes] / USB_PAGE_SIZE);
992 if (info->dma_error) {
993 USB_BUS_LOCK(info->bus);
994 usbd_transfer_done(xfer, USB_ERR_DMA_LOAD_FAILED);
995 USB_BUS_UNLOCK(info->bus);
998 if (info->dma_currframe != info->dma_nframes) {
1000 if (info->dma_currframe == 0) {
1002 usb_pc_load_mem(xfer->frbuffers,
1003 info->dma_frlength_0, 0);
1006 nframes = info->dma_currframe;
1007 usb_pc_load_mem(xfer->frbuffers + nframes,
1008 xfer->frlengths[nframes], 0);
1011 /* advance frame index */
1012 info->dma_currframe++;
1017 usb_bdma_pre_sync(xfer);
1019 /* start loading next USB transfer, if any */
1020 usb_command_wrapper(pq, NULL);
1022 /* finally start the hardware */
1023 usbd_pipe_enter(xfer);
1026 /*------------------------------------------------------------------------*
1027 * usb_bdma_done_event
1029 * This function is called when the BUS-DMA has loaded virtual memory
1031 *------------------------------------------------------------------------*/
1033 usb_bdma_done_event(struct usb_dma_parent_tag *udpt)
1035 struct usb_xfer_root *info;
1037 info = USB_DMATAG_TO_XROOT(udpt);
1039 mtx_assert(info->xfer_mtx, MA_OWNED);
1042 info->dma_error = udpt->dma_error;
1044 /* enter workloop again */
1045 usb_command_wrapper(&info->dma_q,
1049 /*------------------------------------------------------------------------*
1052 * This function handles DMA synchronisation that must be done before
1053 * an USB transfer is started.
1054 *------------------------------------------------------------------------*/
1056 usb_bdma_pre_sync(struct usb_xfer *xfer)
1058 struct usb_page_cache *pc;
1059 usb_frcount_t nframes;
1061 if (xfer->flags_int.isochronous_xfr) {
1062 /* only one frame buffer */
1065 /* can be multiple frame buffers */
1066 nframes = xfer->nframes;
1069 pc = xfer->frbuffers;
1074 usb_pc_cpu_invalidate(pc);
1076 usb_pc_cpu_flush(pc);
1082 /*------------------------------------------------------------------------*
1083 * usb_bdma_post_sync
1085 * This function handles DMA synchronisation that must be done after
1086 * an USB transfer is complete.
1087 *------------------------------------------------------------------------*/
1089 usb_bdma_post_sync(struct usb_xfer *xfer)
1091 struct usb_page_cache *pc;
1092 usb_frcount_t nframes;
1094 if (xfer->flags_int.isochronous_xfr) {
1095 /* only one frame buffer */
1098 /* can be multiple frame buffers */
1099 nframes = xfer->nframes;
1102 pc = xfer->frbuffers;
1106 usb_pc_cpu_invalidate(pc);