3 * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 #ifdef USB_GLOBAL_INCLUDE_FILE
28 #include USB_GLOBAL_INCLUDE_FILE
30 #include <sys/stdint.h>
31 #include <sys/stddef.h>
32 #include <sys/param.h>
33 #include <sys/queue.h>
34 #include <sys/types.h>
35 #include <sys/systm.h>
36 #include <sys/kernel.h>
38 #include <sys/module.h>
40 #include <sys/mutex.h>
41 #include <sys/condvar.h>
42 #include <sys/sysctl.h>
44 #include <sys/unistd.h>
45 #include <sys/callout.h>
46 #include <sys/malloc.h>
49 #include <dev/usb/usb.h>
50 #include <dev/usb/usbdi.h>
51 #include <dev/usb/usbdi_util.h>
53 #define USB_DEBUG_VAR usb_debug
55 #include <dev/usb/usb_core.h>
56 #include <dev/usb/usb_busdma.h>
57 #include <dev/usb/usb_process.h>
58 #include <dev/usb/usb_transfer.h>
59 #include <dev/usb/usb_device.h>
60 #include <dev/usb/usb_util.h>
61 #include <dev/usb/usb_debug.h>
63 #include <dev/usb/usb_controller.h>
64 #include <dev/usb/usb_bus.h>
65 #endif /* USB_GLOBAL_INCLUDE_FILE */
68 static void usb_dma_tag_create(struct usb_dma_tag *, usb_size_t, usb_size_t);
69 static void usb_dma_tag_destroy(struct usb_dma_tag *);
70 static void usb_dma_lock_cb(void *, bus_dma_lock_op_t);
71 static void usb_pc_alloc_mem_cb(void *, bus_dma_segment_t *, int, int);
72 static void usb_pc_load_mem_cb(void *, bus_dma_segment_t *, int, int);
73 static void usb_pc_common_mem_cb(void *, bus_dma_segment_t *, int, int,
77 /*------------------------------------------------------------------------*
78 * usbd_get_page - lookup DMA-able memory for the given offset
80 * NOTE: Only call this function when the "page_cache" structure has
81 * been properly initialized !
82 *------------------------------------------------------------------------*/
84 usbd_get_page(struct usb_page_cache *pc, usb_frlength_t offset,
85 struct usb_page_search *res)
88 struct usb_page *page;
92 /* Case 1 - something has been loaded into DMA */
96 /* Case 1a - Kernel Virtual Address */
98 res->buffer = USB_ADD_BYTES(pc->buffer, offset);
100 offset += pc->page_offset_buf;
102 /* compute destination page */
104 page = pc->page_start;
106 if (pc->ismultiseg) {
108 page += (offset / USB_PAGE_SIZE);
110 offset %= USB_PAGE_SIZE;
112 res->length = USB_PAGE_SIZE - offset;
113 res->physaddr = page->physaddr + offset;
115 res->length = (usb_size_t)-1;
116 res->physaddr = page->physaddr + offset;
120 /* Case 1b - Non Kernel Virtual Address */
122 res->buffer = USB_ADD_BYTES(page->buffer, offset);
127 /* Case 2 - Plain PIO */
129 res->buffer = USB_ADD_BYTES(pc->buffer, offset);
130 res->length = (usb_size_t)-1;
136 /*------------------------------------------------------------------------*
137 * usbd_copy_in - copy directly to DMA-able memory
138 *------------------------------------------------------------------------*/
140 usbd_copy_in(struct usb_page_cache *cache, usb_frlength_t offset,
141 const void *ptr, usb_frlength_t len)
143 struct usb_page_search buf_res;
147 usbd_get_page(cache, offset, &buf_res);
149 if (buf_res.length > len) {
150 buf_res.length = len;
152 memcpy(buf_res.buffer, ptr, buf_res.length);
154 offset += buf_res.length;
155 len -= buf_res.length;
156 ptr = USB_ADD_BYTES(ptr, buf_res.length);
160 /*------------------------------------------------------------------------*
161 * usbd_copy_in_user - copy directly to DMA-able memory from userland
166 *------------------------------------------------------------------------*/
169 usbd_copy_in_user(struct usb_page_cache *cache, usb_frlength_t offset,
170 const void *ptr, usb_frlength_t len)
172 struct usb_page_search buf_res;
177 usbd_get_page(cache, offset, &buf_res);
179 if (buf_res.length > len) {
180 buf_res.length = len;
182 error = copyin(ptr, buf_res.buffer, buf_res.length);
186 offset += buf_res.length;
187 len -= buf_res.length;
188 ptr = USB_ADD_BYTES(ptr, buf_res.length);
190 return (0); /* success */
194 /*------------------------------------------------------------------------*
195 * usbd_m_copy_in - copy a mbuf chain directly into DMA-able memory
196 *------------------------------------------------------------------------*/
198 struct usb_m_copy_in_arg {
199 struct usb_page_cache *cache;
200 usb_frlength_t dst_offset;
204 usbd_m_copy_in_cb(void *arg, void *src, uint32_t count)
206 register struct usb_m_copy_in_arg *ua = arg;
208 usbd_copy_in(ua->cache, ua->dst_offset, src, count);
209 ua->dst_offset += count;
214 usbd_m_copy_in(struct usb_page_cache *cache, usb_frlength_t dst_offset,
215 struct mbuf *m, usb_size_t src_offset, usb_frlength_t src_len)
217 struct usb_m_copy_in_arg arg = {cache, dst_offset};
220 error = m_apply(m, src_offset, src_len, &usbd_m_copy_in_cb, &arg);
224 /*------------------------------------------------------------------------*
225 * usb_uiomove - factored out code
226 *------------------------------------------------------------------------*/
229 usb_uiomove(struct usb_page_cache *pc, struct uio *uio,
230 usb_frlength_t pc_offset, usb_frlength_t len)
232 struct usb_page_search res;
237 usbd_get_page(pc, pc_offset, &res);
239 if (res.length > len) {
243 * "uiomove()" can sleep so one needs to make a wrapper,
244 * exiting the mutex and checking things
246 error = uiomove(res.buffer, res.length, uio);
251 pc_offset += res.length;
258 /*------------------------------------------------------------------------*
259 * usbd_copy_out - copy directly from DMA-able memory
260 *------------------------------------------------------------------------*/
262 usbd_copy_out(struct usb_page_cache *cache, usb_frlength_t offset,
263 void *ptr, usb_frlength_t len)
265 struct usb_page_search res;
269 usbd_get_page(cache, offset, &res);
271 if (res.length > len) {
274 memcpy(ptr, res.buffer, res.length);
276 offset += res.length;
278 ptr = USB_ADD_BYTES(ptr, res.length);
282 /*------------------------------------------------------------------------*
283 * usbd_copy_out_user - copy directly from DMA-able memory to userland
288 *------------------------------------------------------------------------*/
291 usbd_copy_out_user(struct usb_page_cache *cache, usb_frlength_t offset,
292 void *ptr, usb_frlength_t len)
294 struct usb_page_search res;
299 usbd_get_page(cache, offset, &res);
301 if (res.length > len) {
304 error = copyout(res.buffer, ptr, res.length);
308 offset += res.length;
310 ptr = USB_ADD_BYTES(ptr, res.length);
312 return (0); /* success */
316 /*------------------------------------------------------------------------*
317 * usbd_frame_zero - zero DMA-able memory
318 *------------------------------------------------------------------------*/
320 usbd_frame_zero(struct usb_page_cache *cache, usb_frlength_t offset,
323 struct usb_page_search res;
327 usbd_get_page(cache, offset, &res);
329 if (res.length > len) {
332 memset(res.buffer, 0, res.length);
334 offset += res.length;
341 /*------------------------------------------------------------------------*
342 * usb_dma_lock_cb - dummy callback
343 *------------------------------------------------------------------------*/
345 usb_dma_lock_cb(void *arg, bus_dma_lock_op_t op)
347 /* we use "mtx_owned()" instead of this function */
350 /*------------------------------------------------------------------------*
351 * usb_dma_tag_create - allocate a DMA tag
353 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will
354 * allow multi-segment mappings. Else all mappings are single-segment.
355 *------------------------------------------------------------------------*/
357 usb_dma_tag_create(struct usb_dma_tag *udt,
358 usb_size_t size, usb_size_t align)
362 if (bus_dma_tag_create
363 ( /* parent */ udt->tag_parent->tag,
364 /* alignment */ align,
366 /* lowaddr */ (2ULL << (udt->tag_parent->dma_bits - 1)) - 1,
367 /* highaddr */ BUS_SPACE_MAXADDR,
369 /* filterarg */ NULL,
371 /* nsegments */ (align == 1 && size > 1) ?
372 (2 + (size / USB_PAGE_SIZE)) : 1,
373 /* maxsegsz */ (align == 1 && size > USB_PAGE_SIZE) ?
374 USB_PAGE_SIZE : size,
375 /* flags */ BUS_DMA_KEEP_PG_OFFSET,
376 /* lockfn */ &usb_dma_lock_cb,
384 /*------------------------------------------------------------------------*
385 * usb_dma_tag_free - free a DMA tag
386 *------------------------------------------------------------------------*/
388 usb_dma_tag_destroy(struct usb_dma_tag *udt)
390 bus_dma_tag_destroy(udt->tag);
393 /*------------------------------------------------------------------------*
394 * usb_pc_alloc_mem_cb - BUS-DMA callback function
395 *------------------------------------------------------------------------*/
397 usb_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs,
400 usb_pc_common_mem_cb(arg, segs, nseg, error, 0);
403 /*------------------------------------------------------------------------*
404 * usb_pc_load_mem_cb - BUS-DMA callback function
405 *------------------------------------------------------------------------*/
407 usb_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs,
410 usb_pc_common_mem_cb(arg, segs, nseg, error, 1);
413 /*------------------------------------------------------------------------*
414 * usb_pc_common_mem_cb - BUS-DMA callback function
415 *------------------------------------------------------------------------*/
417 usb_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs,
418 int nseg, int error, uint8_t isload)
420 struct usb_dma_parent_tag *uptag;
421 struct usb_page_cache *pc;
428 uptag = pc->tag_parent;
431 * XXX There is sometimes recursive locking here.
432 * XXX We should try to find a better solution.
433 * XXX Until further the "owned" variable does
443 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1);
444 rem = segs->ds_addr & (USB_PAGE_SIZE - 1);
445 pc->page_offset_buf = rem;
446 pc->page_offset_end += rem;
448 if (rem != (USB_P2U(pc->buffer) & (USB_PAGE_SIZE - 1))) {
450 * This check verifies that the physical address is correct:
452 DPRINTFN(0, "Page offset was not preserved\n");
458 off += USB_PAGE_SIZE;
459 if (off >= (segs->ds_len + rem)) {
469 pg->physaddr = (segs->ds_addr + off) & ~(USB_PAGE_SIZE - 1);
473 owned = mtx_owned(uptag->mtx);
475 mtx_lock(uptag->mtx);
477 uptag->dma_error = (error ? 1 : 0);
479 (uptag->func) (uptag);
481 cv_broadcast(uptag->cv);
484 mtx_unlock(uptag->mtx);
487 /*------------------------------------------------------------------------*
488 * usb_pc_alloc_mem - allocate DMA'able memory
493 *------------------------------------------------------------------------*/
495 usb_pc_alloc_mem(struct usb_page_cache *pc, struct usb_page *pg,
496 usb_size_t size, usb_size_t align)
498 struct usb_dma_parent_tag *uptag;
499 struct usb_dma_tag *utag;
504 uptag = pc->tag_parent;
508 * The alignment must be greater or equal to the
509 * "size" else the object can be split between two
510 * memory pages and we get a problem!
512 while (align < size) {
520 * XXX BUS-DMA workaround - FIXME later:
522 * We assume that that the aligment at this point of
523 * the code is greater than or equal to the size and
524 * less than two times the size, so that if we double
525 * the size, the size will be greater than the
528 * The bus-dma system has a check for "alignment"
529 * being less than "size". If that check fails we end
530 * up using contigmalloc which is page based even for
531 * small allocations. Try to avoid that to save
532 * memory, hence we sometimes to a large number of
535 if (size <= (USB_PAGE_SIZE / 2)) {
540 /* get the correct DMA tag */
541 utag = usb_dma_tag_find(uptag, size, align);
545 /* allocate memory */
546 if (bus_dmamem_alloc(
547 utag->tag, &ptr, (BUS_DMA_WAITOK | BUS_DMA_COHERENT), &map)) {
550 /* setup page cache */
553 pc->page_offset_buf = 0;
554 pc->page_offset_end = size;
557 pc->ismultiseg = (align == 1);
559 mtx_lock(uptag->mtx);
561 /* load memory into DMA */
562 err = bus_dmamap_load(
563 utag->tag, map, ptr, size, &usb_pc_alloc_mem_cb,
564 pc, (BUS_DMA_WAITOK | BUS_DMA_COHERENT));
566 if (err == EINPROGRESS) {
567 cv_wait(uptag->cv, uptag->mtx);
570 mtx_unlock(uptag->mtx);
572 if (err || uptag->dma_error) {
573 bus_dmamem_free(utag->tag, ptr, map);
576 memset(ptr, 0, size);
578 usb_pc_cpu_flush(pc);
583 /* reset most of the page cache */
585 pc->page_start = NULL;
586 pc->page_offset_buf = 0;
587 pc->page_offset_end = 0;
593 /*------------------------------------------------------------------------*
594 * usb_pc_free_mem - free DMA memory
596 * This function is NULL safe.
597 *------------------------------------------------------------------------*/
599 usb_pc_free_mem(struct usb_page_cache *pc)
601 if (pc && pc->buffer) {
603 bus_dmamap_unload(pc->tag, pc->map);
605 bus_dmamem_free(pc->tag, pc->buffer, pc->map);
611 /*------------------------------------------------------------------------*
612 * usb_pc_load_mem - load virtual memory into DMA
617 *------------------------------------------------------------------------*/
619 usb_pc_load_mem(struct usb_page_cache *pc, usb_size_t size, uint8_t sync)
621 /* setup page cache */
622 pc->page_offset_buf = 0;
623 pc->page_offset_end = size;
626 mtx_assert(pc->tag_parent->mtx, MA_OWNED);
630 struct usb_dma_parent_tag *uptag;
633 uptag = pc->tag_parent;
636 * We have to unload the previous loaded DMA
637 * pages before trying to load a new one!
639 bus_dmamap_unload(pc->tag, pc->map);
642 * Try to load memory into DMA.
644 err = bus_dmamap_load(
645 pc->tag, pc->map, pc->buffer, size,
646 &usb_pc_alloc_mem_cb, pc, BUS_DMA_WAITOK);
647 if (err == EINPROGRESS) {
648 cv_wait(uptag->cv, uptag->mtx);
651 if (err || uptag->dma_error) {
657 * We have to unload the previous loaded DMA
658 * pages before trying to load a new one!
660 bus_dmamap_unload(pc->tag, pc->map);
663 * Try to load memory into DMA. The callback
664 * will be called in all cases:
667 pc->tag, pc->map, pc->buffer, size,
668 &usb_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) {
674 * Call callback so that refcount is decremented
677 pc->tag_parent->dma_error = 0;
678 (pc->tag_parent->func) (pc->tag_parent);
684 /*------------------------------------------------------------------------*
685 * usb_pc_cpu_invalidate - invalidate CPU cache
686 *------------------------------------------------------------------------*/
688 usb_pc_cpu_invalidate(struct usb_page_cache *pc)
690 if (pc->page_offset_end == pc->page_offset_buf) {
691 /* nothing has been loaded into this page cache! */
696 * TODO: We currently do XXX_POSTREAD and XXX_PREREAD at the
697 * same time, but in the future we should try to isolate the
698 * different cases to optimise the code. --HPS
700 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_POSTREAD);
701 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREREAD);
704 /*------------------------------------------------------------------------*
705 * usb_pc_cpu_flush - flush CPU cache
706 *------------------------------------------------------------------------*/
708 usb_pc_cpu_flush(struct usb_page_cache *pc)
710 if (pc->page_offset_end == pc->page_offset_buf) {
711 /* nothing has been loaded into this page cache! */
714 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREWRITE);
717 /*------------------------------------------------------------------------*
718 * usb_pc_dmamap_create - create a DMA map
723 *------------------------------------------------------------------------*/
725 usb_pc_dmamap_create(struct usb_page_cache *pc, usb_size_t size)
727 struct usb_xfer_root *info;
728 struct usb_dma_tag *utag;
731 info = USB_DMATAG_TO_XROOT(pc->tag_parent);
737 utag = usb_dma_tag_find(pc->tag_parent, size, 1);
742 if (bus_dmamap_create(utag->tag, 0, &pc->map)) {
746 return 0; /* success */
751 return 1; /* failure */
754 /*------------------------------------------------------------------------*
755 * usb_pc_dmamap_destroy
757 * This function is NULL safe.
758 *------------------------------------------------------------------------*/
760 usb_pc_dmamap_destroy(struct usb_page_cache *pc)
763 bus_dmamap_destroy(pc->tag, pc->map);
769 /*------------------------------------------------------------------------*
770 * usb_dma_tag_find - factored out code
771 *------------------------------------------------------------------------*/
773 usb_dma_tag_find(struct usb_dma_parent_tag *udpt,
774 usb_size_t size, usb_size_t align)
776 struct usb_dma_tag *udt;
779 USB_ASSERT(align > 0, ("Invalid parameter align = 0\n"));
780 USB_ASSERT(size > 0, ("Invalid parameter size = 0\n"));
782 udt = udpt->utag_first;
783 nudt = udpt->utag_max;
787 if (udt->align == 0) {
788 usb_dma_tag_create(udt, size, align);
789 if (udt->tag == NULL) {
796 if ((udt->align == align) && (udt->size == size)) {
804 /*------------------------------------------------------------------------*
805 * usb_dma_tag_setup - initialise USB DMA tags
806 *------------------------------------------------------------------------*/
808 usb_dma_tag_setup(struct usb_dma_parent_tag *udpt,
809 struct usb_dma_tag *udt, bus_dma_tag_t dmat,
810 struct mtx *mtx, usb_dma_callback_t *func,
811 uint8_t ndmabits, uint8_t nudt)
813 memset(udpt, 0, sizeof(*udpt));
815 /* sanity checking */
819 /* something is corrupt */
822 /* initialise condition variable */
823 cv_init(udpt->cv, "USB DMA CV");
825 /* store some information */
829 udpt->utag_first = udt;
830 udpt->utag_max = nudt;
831 udpt->dma_bits = ndmabits;
834 memset(udt, 0, sizeof(*udt));
835 udt->tag_parent = udpt;
840 /*------------------------------------------------------------------------*
841 * usb_bus_tag_unsetup - factored out code
842 *------------------------------------------------------------------------*/
844 usb_dma_tag_unsetup(struct usb_dma_parent_tag *udpt)
846 struct usb_dma_tag *udt;
849 udt = udpt->utag_first;
850 nudt = udpt->utag_max;
855 /* destroy the USB DMA tag */
856 usb_dma_tag_destroy(udt);
862 if (udpt->utag_max) {
863 /* destroy the condition variable */
864 cv_destroy(udpt->cv);
868 /*------------------------------------------------------------------------*
871 * This function handles loading of virtual buffers into DMA and is
872 * only called when "dma_refcount" is zero.
873 *------------------------------------------------------------------------*/
875 usb_bdma_work_loop(struct usb_xfer_queue *pq)
877 struct usb_xfer_root *info;
878 struct usb_xfer *xfer;
879 usb_frcount_t nframes;
884 mtx_assert(info->xfer_mtx, MA_OWNED);
887 /* some error happened */
888 USB_BUS_LOCK(info->bus);
889 usbd_transfer_done(xfer, 0);
890 USB_BUS_UNLOCK(info->bus);
893 if (!xfer->flags_int.bdma_setup) {
895 usb_frlength_t frlength_0;
898 xfer->flags_int.bdma_setup = 1;
900 /* reset BUS-DMA load state */
904 if (xfer->flags_int.isochronous_xfr) {
905 /* only one frame buffer */
907 frlength_0 = xfer->sumlen;
909 /* can be multiple frame buffers */
910 nframes = xfer->nframes;
911 frlength_0 = xfer->frlengths[0];
915 * Set DMA direction first. This is needed to
916 * select the correct cache invalidate and cache
919 isread = USB_GET_DATA_ISREAD(xfer);
920 pg = xfer->dma_page_ptr;
922 if (xfer->flags_int.control_xfr &&
923 xfer->flags_int.control_hdr) {
925 if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
926 /* The device controller writes to memory */
927 xfer->frbuffers[0].isread = 1;
929 /* The host controller reads from memory */
930 xfer->frbuffers[0].isread = 0;
934 xfer->frbuffers[0].isread = isread;
938 * Setup the "page_start" pointer which points to an array of
939 * USB pages where information about the physical address of a
940 * page will be stored. Also initialise the "isread" field of
941 * the USB page caches.
943 xfer->frbuffers[0].page_start = pg;
945 info->dma_nframes = nframes;
946 info->dma_currframe = 0;
947 info->dma_frlength_0 = frlength_0;
949 pg += (frlength_0 / USB_PAGE_SIZE);
952 while (--nframes > 0) {
953 xfer->frbuffers[nframes].isread = isread;
954 xfer->frbuffers[nframes].page_start = pg;
956 pg += (xfer->frlengths[nframes] / USB_PAGE_SIZE);
961 if (info->dma_error) {
962 USB_BUS_LOCK(info->bus);
963 usbd_transfer_done(xfer, USB_ERR_DMA_LOAD_FAILED);
964 USB_BUS_UNLOCK(info->bus);
967 if (info->dma_currframe != info->dma_nframes) {
969 if (info->dma_currframe == 0) {
971 usb_pc_load_mem(xfer->frbuffers,
972 info->dma_frlength_0, 0);
975 nframes = info->dma_currframe;
976 usb_pc_load_mem(xfer->frbuffers + nframes,
977 xfer->frlengths[nframes], 0);
980 /* advance frame index */
981 info->dma_currframe++;
986 usb_bdma_pre_sync(xfer);
988 /* start loading next USB transfer, if any */
989 usb_command_wrapper(pq, NULL);
991 /* finally start the hardware */
992 usbd_pipe_enter(xfer);
995 /*------------------------------------------------------------------------*
996 * usb_bdma_done_event
998 * This function is called when the BUS-DMA has loaded virtual memory
1000 *------------------------------------------------------------------------*/
1002 usb_bdma_done_event(struct usb_dma_parent_tag *udpt)
1004 struct usb_xfer_root *info;
1006 info = USB_DMATAG_TO_XROOT(udpt);
1008 mtx_assert(info->xfer_mtx, MA_OWNED);
1011 info->dma_error = udpt->dma_error;
1013 /* enter workloop again */
1014 usb_command_wrapper(&info->dma_q,
1018 /*------------------------------------------------------------------------*
1021 * This function handles DMA synchronisation that must be done before
1022 * an USB transfer is started.
1023 *------------------------------------------------------------------------*/
1025 usb_bdma_pre_sync(struct usb_xfer *xfer)
1027 struct usb_page_cache *pc;
1028 usb_frcount_t nframes;
1030 if (xfer->flags_int.isochronous_xfr) {
1031 /* only one frame buffer */
1034 /* can be multiple frame buffers */
1035 nframes = xfer->nframes;
1038 pc = xfer->frbuffers;
1043 usb_pc_cpu_invalidate(pc);
1045 usb_pc_cpu_flush(pc);
1051 /*------------------------------------------------------------------------*
1052 * usb_bdma_post_sync
1054 * This function handles DMA synchronisation that must be done after
1055 * an USB transfer is complete.
1056 *------------------------------------------------------------------------*/
1058 usb_bdma_post_sync(struct usb_xfer *xfer)
1060 struct usb_page_cache *pc;
1061 usb_frcount_t nframes;
1063 if (xfer->flags_int.isochronous_xfr) {
1064 /* only one frame buffer */
1067 /* can be multiple frame buffers */
1068 nframes = xfer->nframes;
1071 pc = xfer->frbuffers;
1075 usb_pc_cpu_invalidate(pc);