3 * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 #include <sys/stdint.h>
28 #include <sys/stddef.h>
29 #include <sys/param.h>
30 #include <sys/queue.h>
31 #include <sys/types.h>
32 #include <sys/systm.h>
33 #include <sys/kernel.h>
35 #include <sys/module.h>
37 #include <sys/mutex.h>
38 #include <sys/condvar.h>
39 #include <sys/sysctl.h>
41 #include <sys/unistd.h>
42 #include <sys/callout.h>
43 #include <sys/malloc.h>
46 #include <dev/usb/usb.h>
47 #include <dev/usb/usbdi.h>
48 #include <dev/usb/usbdi_util.h>
50 #define USB_DEBUG_VAR usb_debug
52 #include <dev/usb/usb_core.h>
53 #include <dev/usb/usb_busdma.h>
54 #include <dev/usb/usb_process.h>
55 #include <dev/usb/usb_transfer.h>
56 #include <dev/usb/usb_device.h>
57 #include <dev/usb/usb_util.h>
58 #include <dev/usb/usb_debug.h>
60 #include <dev/usb/usb_controller.h>
61 #include <dev/usb/usb_bus.h>
64 static void usb_dma_tag_create(struct usb_dma_tag *, usb_size_t, usb_size_t);
65 static void usb_dma_tag_destroy(struct usb_dma_tag *);
66 static void usb_dma_lock_cb(void *, bus_dma_lock_op_t);
67 static void usb_pc_alloc_mem_cb(void *, bus_dma_segment_t *, int, int);
68 static void usb_pc_load_mem_cb(void *, bus_dma_segment_t *, int, int);
69 static void usb_pc_common_mem_cb(void *, bus_dma_segment_t *, int, int,
73 /*------------------------------------------------------------------------*
74 * usbd_get_page - lookup DMA-able memory for the given offset
76 * NOTE: Only call this function when the "page_cache" structure has
77 * been properly initialized !
78 *------------------------------------------------------------------------*/
80 usbd_get_page(struct usb_page_cache *pc, usb_frlength_t offset,
81 struct usb_page_search *res)
84 struct usb_page *page;
88 /* Case 1 - something has been loaded into DMA */
92 /* Case 1a - Kernel Virtual Address */
94 res->buffer = USB_ADD_BYTES(pc->buffer, offset);
96 offset += pc->page_offset_buf;
98 /* compute destination page */
100 page = pc->page_start;
102 if (pc->ismultiseg) {
104 page += (offset / USB_PAGE_SIZE);
106 offset %= USB_PAGE_SIZE;
108 res->length = USB_PAGE_SIZE - offset;
109 res->physaddr = page->physaddr + offset;
111 res->length = (usb_size_t)-1;
112 res->physaddr = page->physaddr + offset;
116 /* Case 1b - Non Kernel Virtual Address */
118 res->buffer = USB_ADD_BYTES(page->buffer, offset);
123 /* Case 2 - Plain PIO */
125 res->buffer = USB_ADD_BYTES(pc->buffer, offset);
126 res->length = (usb_size_t)-1;
132 /*------------------------------------------------------------------------*
133 * usbd_copy_in - copy directly to DMA-able memory
134 *------------------------------------------------------------------------*/
136 usbd_copy_in(struct usb_page_cache *cache, usb_frlength_t offset,
137 const void *ptr, usb_frlength_t len)
139 struct usb_page_search buf_res;
143 usbd_get_page(cache, offset, &buf_res);
145 if (buf_res.length > len) {
146 buf_res.length = len;
148 memcpy(buf_res.buffer, ptr, buf_res.length);
150 offset += buf_res.length;
151 len -= buf_res.length;
152 ptr = USB_ADD_BYTES(ptr, buf_res.length);
156 /*------------------------------------------------------------------------*
157 * usbd_copy_in_user - copy directly to DMA-able memory from userland
162 *------------------------------------------------------------------------*/
165 usbd_copy_in_user(struct usb_page_cache *cache, usb_frlength_t offset,
166 const void *ptr, usb_frlength_t len)
168 struct usb_page_search buf_res;
173 usbd_get_page(cache, offset, &buf_res);
175 if (buf_res.length > len) {
176 buf_res.length = len;
178 error = copyin(ptr, buf_res.buffer, buf_res.length);
182 offset += buf_res.length;
183 len -= buf_res.length;
184 ptr = USB_ADD_BYTES(ptr, buf_res.length);
186 return (0); /* success */
190 /*------------------------------------------------------------------------*
191 * usbd_m_copy_in - copy a mbuf chain directly into DMA-able memory
192 *------------------------------------------------------------------------*/
194 struct usb_m_copy_in_arg {
195 struct usb_page_cache *cache;
196 usb_frlength_t dst_offset;
200 usbd_m_copy_in_cb(void *arg, void *src, uint32_t count)
202 register struct usb_m_copy_in_arg *ua = arg;
204 usbd_copy_in(ua->cache, ua->dst_offset, src, count);
205 ua->dst_offset += count;
210 usbd_m_copy_in(struct usb_page_cache *cache, usb_frlength_t dst_offset,
211 struct mbuf *m, usb_size_t src_offset, usb_frlength_t src_len)
213 struct usb_m_copy_in_arg arg = {cache, dst_offset};
214 (void) m_apply(m, src_offset, src_len, &usbd_m_copy_in_cb, &arg);
218 /*------------------------------------------------------------------------*
219 * usb_uiomove - factored out code
220 *------------------------------------------------------------------------*/
223 usb_uiomove(struct usb_page_cache *pc, struct uio *uio,
224 usb_frlength_t pc_offset, usb_frlength_t len)
226 struct usb_page_search res;
231 usbd_get_page(pc, pc_offset, &res);
233 if (res.length > len) {
237 * "uiomove()" can sleep so one needs to make a wrapper,
238 * exiting the mutex and checking things
240 error = uiomove(res.buffer, res.length, uio);
245 pc_offset += res.length;
252 /*------------------------------------------------------------------------*
253 * usbd_copy_out - copy directly from DMA-able memory
254 *------------------------------------------------------------------------*/
256 usbd_copy_out(struct usb_page_cache *cache, usb_frlength_t offset,
257 void *ptr, usb_frlength_t len)
259 struct usb_page_search res;
263 usbd_get_page(cache, offset, &res);
265 if (res.length > len) {
268 memcpy(ptr, res.buffer, res.length);
270 offset += res.length;
272 ptr = USB_ADD_BYTES(ptr, res.length);
276 /*------------------------------------------------------------------------*
277 * usbd_copy_out_user - copy directly from DMA-able memory to userland
282 *------------------------------------------------------------------------*/
285 usbd_copy_out_user(struct usb_page_cache *cache, usb_frlength_t offset,
286 void *ptr, usb_frlength_t len)
288 struct usb_page_search res;
293 usbd_get_page(cache, offset, &res);
295 if (res.length > len) {
298 error = copyout(res.buffer, ptr, res.length);
302 offset += res.length;
304 ptr = USB_ADD_BYTES(ptr, res.length);
306 return (0); /* success */
310 /*------------------------------------------------------------------------*
311 * usbd_frame_zero - zero DMA-able memory
312 *------------------------------------------------------------------------*/
314 usbd_frame_zero(struct usb_page_cache *cache, usb_frlength_t offset,
317 struct usb_page_search res;
321 usbd_get_page(cache, offset, &res);
323 if (res.length > len) {
326 memset(res.buffer, 0, res.length);
328 offset += res.length;
335 /*------------------------------------------------------------------------*
336 * usb_dma_lock_cb - dummy callback
337 *------------------------------------------------------------------------*/
339 usb_dma_lock_cb(void *arg, bus_dma_lock_op_t op)
341 /* we use "mtx_owned()" instead of this function */
344 /*------------------------------------------------------------------------*
345 * usb_dma_tag_create - allocate a DMA tag
347 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will
348 * allow multi-segment mappings. Else all mappings are single-segment.
349 *------------------------------------------------------------------------*/
351 usb_dma_tag_create(struct usb_dma_tag *udt,
352 usb_size_t size, usb_size_t align)
356 if (bus_dma_tag_create
357 ( /* parent */ udt->tag_parent->tag,
358 /* alignment */ align,
360 /* lowaddr */ (2ULL << (udt->tag_parent->dma_bits - 1)) - 1,
361 /* highaddr */ BUS_SPACE_MAXADDR,
363 /* filterarg */ NULL,
365 /* nsegments */ (align == 1 && size > 1) ?
366 (2 + (size / USB_PAGE_SIZE)) : 1,
367 /* maxsegsz */ (align == 1 && size > USB_PAGE_SIZE) ?
368 USB_PAGE_SIZE : size,
369 /* flags */ BUS_DMA_KEEP_PG_OFFSET,
370 /* lockfn */ &usb_dma_lock_cb,
378 /*------------------------------------------------------------------------*
379 * usb_dma_tag_free - free a DMA tag
380 *------------------------------------------------------------------------*/
382 usb_dma_tag_destroy(struct usb_dma_tag *udt)
384 bus_dma_tag_destroy(udt->tag);
387 /*------------------------------------------------------------------------*
388 * usb_pc_alloc_mem_cb - BUS-DMA callback function
389 *------------------------------------------------------------------------*/
391 usb_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs,
394 usb_pc_common_mem_cb(arg, segs, nseg, error, 0);
397 /*------------------------------------------------------------------------*
398 * usb_pc_load_mem_cb - BUS-DMA callback function
399 *------------------------------------------------------------------------*/
401 usb_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs,
404 usb_pc_common_mem_cb(arg, segs, nseg, error, 1);
407 /*------------------------------------------------------------------------*
408 * usb_pc_common_mem_cb - BUS-DMA callback function
409 *------------------------------------------------------------------------*/
411 usb_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs,
412 int nseg, int error, uint8_t isload)
414 struct usb_dma_parent_tag *uptag;
415 struct usb_page_cache *pc;
422 uptag = pc->tag_parent;
425 * XXX There is sometimes recursive locking here.
426 * XXX We should try to find a better solution.
427 * XXX Until further the "owned" variable does
437 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1);
438 rem = segs->ds_addr & (USB_PAGE_SIZE - 1);
439 pc->page_offset_buf = rem;
440 pc->page_offset_end += rem;
442 if (rem != (USB_P2U(pc->buffer) & (USB_PAGE_SIZE - 1))) {
444 * This check verifies that the physical address is correct:
446 DPRINTFN(0, "Page offset was not preserved\n");
451 while (pc->ismultiseg) {
452 off += USB_PAGE_SIZE;
453 if (off >= (segs->ds_len + rem)) {
463 pg->physaddr = (segs->ds_addr + off) & ~(USB_PAGE_SIZE - 1);
467 owned = mtx_owned(uptag->mtx);
469 mtx_lock(uptag->mtx);
471 uptag->dma_error = (error ? 1 : 0);
473 (uptag->func) (uptag);
475 cv_broadcast(uptag->cv);
478 mtx_unlock(uptag->mtx);
481 /*------------------------------------------------------------------------*
482 * usb_pc_alloc_mem - allocate DMA'able memory
487 *------------------------------------------------------------------------*/
489 usb_pc_alloc_mem(struct usb_page_cache *pc, struct usb_page *pg,
490 usb_size_t size, usb_size_t align)
492 struct usb_dma_parent_tag *uptag;
493 struct usb_dma_tag *utag;
498 uptag = pc->tag_parent;
502 * The alignment must be greater or equal to the
503 * "size" else the object can be split between two
504 * memory pages and we get a problem!
506 while (align < size) {
514 * XXX BUS-DMA workaround - FIXME later:
516 * We assume that that the aligment at this point of
517 * the code is greater than or equal to the size and
518 * less than two times the size, so that if we double
519 * the size, the size will be greater than the
522 * The bus-dma system has a check for "alignment"
523 * being less than "size". If that check fails we end
524 * up using contigmalloc which is page based even for
525 * small allocations. Try to avoid that to save
526 * memory, hence we sometimes to a large number of
529 if (size <= (USB_PAGE_SIZE / 2)) {
534 /* get the correct DMA tag */
535 utag = usb_dma_tag_find(uptag, size, align);
539 /* allocate memory */
540 if (bus_dmamem_alloc(
541 utag->tag, &ptr, (BUS_DMA_WAITOK | BUS_DMA_COHERENT), &map)) {
544 /* setup page cache */
547 pc->page_offset_buf = 0;
548 pc->page_offset_end = size;
551 pc->ismultiseg = (align == 1);
553 mtx_lock(uptag->mtx);
555 /* load memory into DMA */
556 err = bus_dmamap_load(
557 utag->tag, map, ptr, size, &usb_pc_alloc_mem_cb,
558 pc, (BUS_DMA_WAITOK | BUS_DMA_COHERENT));
560 if (err == EINPROGRESS) {
561 cv_wait(uptag->cv, uptag->mtx);
564 mtx_unlock(uptag->mtx);
566 if (err || uptag->dma_error) {
567 bus_dmamem_free(utag->tag, ptr, map);
570 memset(ptr, 0, size);
572 usb_pc_cpu_flush(pc);
577 /* reset most of the page cache */
579 pc->page_start = NULL;
580 pc->page_offset_buf = 0;
581 pc->page_offset_end = 0;
587 /*------------------------------------------------------------------------*
588 * usb_pc_free_mem - free DMA memory
590 * This function is NULL safe.
591 *------------------------------------------------------------------------*/
593 usb_pc_free_mem(struct usb_page_cache *pc)
595 if (pc && pc->buffer) {
597 bus_dmamap_unload(pc->tag, pc->map);
599 bus_dmamem_free(pc->tag, pc->buffer, pc->map);
605 /*------------------------------------------------------------------------*
606 * usb_pc_load_mem - load virtual memory into DMA
611 *------------------------------------------------------------------------*/
613 usb_pc_load_mem(struct usb_page_cache *pc, usb_size_t size, uint8_t sync)
615 /* setup page cache */
616 pc->page_offset_buf = 0;
617 pc->page_offset_end = size;
620 mtx_assert(pc->tag_parent->mtx, MA_OWNED);
624 struct usb_dma_parent_tag *uptag;
627 uptag = pc->tag_parent;
630 * We have to unload the previous loaded DMA
631 * pages before trying to load a new one!
633 bus_dmamap_unload(pc->tag, pc->map);
636 * Try to load memory into DMA.
638 err = bus_dmamap_load(
639 pc->tag, pc->map, pc->buffer, size,
640 &usb_pc_alloc_mem_cb, pc, BUS_DMA_WAITOK);
641 if (err == EINPROGRESS) {
642 cv_wait(uptag->cv, uptag->mtx);
645 if (err || uptag->dma_error) {
651 * We have to unload the previous loaded DMA
652 * pages before trying to load a new one!
654 bus_dmamap_unload(pc->tag, pc->map);
657 * Try to load memory into DMA. The callback
658 * will be called in all cases:
661 pc->tag, pc->map, pc->buffer, size,
662 &usb_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) {
668 * Call callback so that refcount is decremented
671 pc->tag_parent->dma_error = 0;
672 (pc->tag_parent->func) (pc->tag_parent);
678 /*------------------------------------------------------------------------*
679 * usb_pc_cpu_invalidate - invalidate CPU cache
680 *------------------------------------------------------------------------*/
682 usb_pc_cpu_invalidate(struct usb_page_cache *pc)
684 if (pc->page_offset_end == pc->page_offset_buf) {
685 /* nothing has been loaded into this page cache! */
690 * TODO: We currently do XXX_POSTREAD and XXX_PREREAD at the
691 * same time, but in the future we should try to isolate the
692 * different cases to optimise the code. --HPS
694 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_POSTREAD);
695 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREREAD);
698 /*------------------------------------------------------------------------*
699 * usb_pc_cpu_flush - flush CPU cache
700 *------------------------------------------------------------------------*/
702 usb_pc_cpu_flush(struct usb_page_cache *pc)
704 if (pc->page_offset_end == pc->page_offset_buf) {
705 /* nothing has been loaded into this page cache! */
708 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREWRITE);
711 /*------------------------------------------------------------------------*
712 * usb_pc_dmamap_create - create a DMA map
717 *------------------------------------------------------------------------*/
719 usb_pc_dmamap_create(struct usb_page_cache *pc, usb_size_t size)
721 struct usb_xfer_root *info;
722 struct usb_dma_tag *utag;
725 info = USB_DMATAG_TO_XROOT(pc->tag_parent);
731 utag = usb_dma_tag_find(pc->tag_parent, size, 1);
736 if (bus_dmamap_create(utag->tag, 0, &pc->map)) {
740 return 0; /* success */
745 return 1; /* failure */
748 /*------------------------------------------------------------------------*
749 * usb_pc_dmamap_destroy
751 * This function is NULL safe.
752 *------------------------------------------------------------------------*/
754 usb_pc_dmamap_destroy(struct usb_page_cache *pc)
757 bus_dmamap_destroy(pc->tag, pc->map);
763 /*------------------------------------------------------------------------*
764 * usb_dma_tag_find - factored out code
765 *------------------------------------------------------------------------*/
767 usb_dma_tag_find(struct usb_dma_parent_tag *udpt,
768 usb_size_t size, usb_size_t align)
770 struct usb_dma_tag *udt;
773 USB_ASSERT(align > 0, ("Invalid parameter align = 0\n"));
774 USB_ASSERT(size > 0, ("Invalid parameter size = 0\n"));
776 udt = udpt->utag_first;
777 nudt = udpt->utag_max;
781 if (udt->align == 0) {
782 usb_dma_tag_create(udt, size, align);
783 if (udt->tag == NULL) {
790 if ((udt->align == align) && (udt->size == size)) {
798 /*------------------------------------------------------------------------*
799 * usb_dma_tag_setup - initialise USB DMA tags
800 *------------------------------------------------------------------------*/
802 usb_dma_tag_setup(struct usb_dma_parent_tag *udpt,
803 struct usb_dma_tag *udt, bus_dma_tag_t dmat,
804 struct mtx *mtx, usb_dma_callback_t *func,
805 uint8_t ndmabits, uint8_t nudt)
807 memset(udpt, 0, sizeof(*udpt));
809 /* sanity checking */
813 /* something is corrupt */
816 /* initialise condition variable */
817 cv_init(udpt->cv, "USB DMA CV");
819 /* store some information */
823 udpt->utag_first = udt;
824 udpt->utag_max = nudt;
825 udpt->dma_bits = ndmabits;
828 memset(udt, 0, sizeof(*udt));
829 udt->tag_parent = udpt;
834 /*------------------------------------------------------------------------*
835 * usb_bus_tag_unsetup - factored out code
836 *------------------------------------------------------------------------*/
838 usb_dma_tag_unsetup(struct usb_dma_parent_tag *udpt)
840 struct usb_dma_tag *udt;
843 udt = udpt->utag_first;
844 nudt = udpt->utag_max;
849 /* destroy the USB DMA tag */
850 usb_dma_tag_destroy(udt);
856 if (udpt->utag_max) {
857 /* destroy the condition variable */
858 cv_destroy(udpt->cv);
862 /*------------------------------------------------------------------------*
865 * This function handles loading of virtual buffers into DMA and is
866 * only called when "dma_refcount" is zero.
867 *------------------------------------------------------------------------*/
869 usb_bdma_work_loop(struct usb_xfer_queue *pq)
871 struct usb_xfer_root *info;
872 struct usb_xfer *xfer;
873 usb_frcount_t nframes;
878 mtx_assert(info->xfer_mtx, MA_OWNED);
881 /* some error happened */
882 USB_BUS_LOCK(info->bus);
883 usbd_transfer_done(xfer, 0);
884 USB_BUS_UNLOCK(info->bus);
887 if (!xfer->flags_int.bdma_setup) {
889 usb_frlength_t frlength_0;
892 xfer->flags_int.bdma_setup = 1;
894 /* reset BUS-DMA load state */
898 if (xfer->flags_int.isochronous_xfr) {
899 /* only one frame buffer */
901 frlength_0 = xfer->sumlen;
903 /* can be multiple frame buffers */
904 nframes = xfer->nframes;
905 frlength_0 = xfer->frlengths[0];
909 * Set DMA direction first. This is needed to
910 * select the correct cache invalidate and cache
913 isread = USB_GET_DATA_ISREAD(xfer);
914 pg = xfer->dma_page_ptr;
916 if (xfer->flags_int.control_xfr &&
917 xfer->flags_int.control_hdr) {
919 if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
920 /* The device controller writes to memory */
921 xfer->frbuffers[0].isread = 1;
923 /* The host controller reads from memory */
924 xfer->frbuffers[0].isread = 0;
928 xfer->frbuffers[0].isread = isread;
932 * Setup the "page_start" pointer which points to an array of
933 * USB pages where information about the physical address of a
934 * page will be stored. Also initialise the "isread" field of
935 * the USB page caches.
937 xfer->frbuffers[0].page_start = pg;
939 info->dma_nframes = nframes;
940 info->dma_currframe = 0;
941 info->dma_frlength_0 = frlength_0;
943 pg += (frlength_0 / USB_PAGE_SIZE);
946 while (--nframes > 0) {
947 xfer->frbuffers[nframes].isread = isread;
948 xfer->frbuffers[nframes].page_start = pg;
950 pg += (xfer->frlengths[nframes] / USB_PAGE_SIZE);
955 if (info->dma_error) {
956 USB_BUS_LOCK(info->bus);
957 usbd_transfer_done(xfer, USB_ERR_DMA_LOAD_FAILED);
958 USB_BUS_UNLOCK(info->bus);
961 if (info->dma_currframe != info->dma_nframes) {
963 if (info->dma_currframe == 0) {
965 usb_pc_load_mem(xfer->frbuffers,
966 info->dma_frlength_0, 0);
969 nframes = info->dma_currframe;
970 usb_pc_load_mem(xfer->frbuffers + nframes,
971 xfer->frlengths[nframes], 0);
974 /* advance frame index */
975 info->dma_currframe++;
980 usb_bdma_pre_sync(xfer);
982 /* start loading next USB transfer, if any */
983 usb_command_wrapper(pq, NULL);
985 /* finally start the hardware */
986 usbd_pipe_enter(xfer);
989 /*------------------------------------------------------------------------*
990 * usb_bdma_done_event
992 * This function is called when the BUS-DMA has loaded virtual memory
994 *------------------------------------------------------------------------*/
996 usb_bdma_done_event(struct usb_dma_parent_tag *udpt)
998 struct usb_xfer_root *info;
1000 info = USB_DMATAG_TO_XROOT(udpt);
1002 mtx_assert(info->xfer_mtx, MA_OWNED);
1005 info->dma_error = udpt->dma_error;
1007 /* enter workloop again */
1008 usb_command_wrapper(&info->dma_q,
1012 /*------------------------------------------------------------------------*
1015 * This function handles DMA synchronisation that must be done before
1016 * an USB transfer is started.
1017 *------------------------------------------------------------------------*/
1019 usb_bdma_pre_sync(struct usb_xfer *xfer)
1021 struct usb_page_cache *pc;
1022 usb_frcount_t nframes;
1024 if (xfer->flags_int.isochronous_xfr) {
1025 /* only one frame buffer */
1028 /* can be multiple frame buffers */
1029 nframes = xfer->nframes;
1032 pc = xfer->frbuffers;
1037 usb_pc_cpu_invalidate(pc);
1039 usb_pc_cpu_flush(pc);
1045 /*------------------------------------------------------------------------*
1046 * usb_bdma_post_sync
1048 * This function handles DMA synchronisation that must be done after
1049 * an USB transfer is complete.
1050 *------------------------------------------------------------------------*/
1052 usb_bdma_post_sync(struct usb_xfer *xfer)
1054 struct usb_page_cache *pc;
1055 usb_frcount_t nframes;
1057 if (xfer->flags_int.isochronous_xfr) {
1058 /* only one frame buffer */
1061 /* can be multiple frame buffers */
1062 nframes = xfer->nframes;
1065 pc = xfer->frbuffers;
1069 usb_pc_cpu_invalidate(pc);