3 * Copyright (c) 2013 Hans Petter Selasky. All rights reserved.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 #include <bsd_global.h>
30 static void usb_pc_common_mem_cb(struct usb_page_cache *pc,
31 void *vaddr, uint32_t length);
34 /*------------------------------------------------------------------------*
35 * usbd_get_page - lookup DMA-able memory for the given offset
37 * NOTE: Only call this function when the "page_cache" structure has
38 * been properly initialized !
39 *------------------------------------------------------------------------*/
41 usbd_get_page(struct usb_page_cache *pc, usb_frlength_t offset,
42 struct usb_page_search *res)
45 struct usb_page *page;
49 /* Case 1 - something has been loaded into DMA */
53 /* Case 1a - Kernel Virtual Address */
55 res->buffer = USB_ADD_BYTES(pc->buffer, offset);
57 offset += pc->page_offset_buf;
59 /* compute destination page */
61 page = pc->page_start;
65 page += (offset / USB_PAGE_SIZE);
67 offset %= USB_PAGE_SIZE;
69 res->length = USB_PAGE_SIZE - offset;
70 res->physaddr = page->physaddr + offset;
72 res->length = (usb_size_t)-1;
73 res->physaddr = page->physaddr + offset;
77 /* Case 1b - Non Kernel Virtual Address */
79 res->buffer = USB_ADD_BYTES(page->buffer, offset);
84 /* Case 2 - Plain PIO */
86 res->buffer = USB_ADD_BYTES(pc->buffer, offset);
87 res->length = (usb_size_t)-1;
93 /*------------------------------------------------------------------------*
94 * usbd_copy_in - copy directly to DMA-able memory
95 *------------------------------------------------------------------------*/
97 usbd_copy_in(struct usb_page_cache *cache, usb_frlength_t offset,
98 const void *ptr, usb_frlength_t len)
100 struct usb_page_search buf_res;
104 usbd_get_page(cache, offset, &buf_res);
106 if (buf_res.length > len) {
107 buf_res.length = len;
109 memcpy(buf_res.buffer, ptr, buf_res.length);
111 offset += buf_res.length;
112 len -= buf_res.length;
113 ptr = USB_ADD_BYTES(ptr, buf_res.length);
117 /*------------------------------------------------------------------------*
118 * usbd_copy_out - copy directly from DMA-able memory
119 *------------------------------------------------------------------------*/
121 usbd_copy_out(struct usb_page_cache *cache, usb_frlength_t offset,
122 void *ptr, usb_frlength_t len)
124 struct usb_page_search res;
128 usbd_get_page(cache, offset, &res);
130 if (res.length > len) {
133 memcpy(ptr, res.buffer, res.length);
135 offset += res.length;
137 ptr = USB_ADD_BYTES(ptr, res.length);
141 /*------------------------------------------------------------------------*
142 * usbd_frame_zero - zero DMA-able memory
143 *------------------------------------------------------------------------*/
145 usbd_frame_zero(struct usb_page_cache *cache, usb_frlength_t offset,
148 struct usb_page_search res;
152 usbd_get_page(cache, offset, &res);
154 if (res.length > len) {
157 memset(res.buffer, 0, res.length);
159 offset += res.length;
166 /*------------------------------------------------------------------------*
167 * usb_pc_common_mem_cb - BUS-DMA callback function
168 *------------------------------------------------------------------------*/
170 usb_pc_common_mem_cb(struct usb_page_cache *pc,
171 void *vaddr, uint32_t length)
176 bus_addr_t phys = (uintptr_t)vaddr; /* XXX */
182 nseg = ((length + USB_PAGE_SIZE - 1) / USB_PAGE_SIZE);
185 pg->physaddr = phys & ~(USB_PAGE_SIZE - 1);
186 rem = phys & (USB_PAGE_SIZE - 1);
187 pc->page_offset_buf = rem;
188 pc->page_offset_end += rem;
191 for (off = USB_PAGE_SIZE; off < length; off += USB_PAGE_SIZE) {
193 pg->physaddr = (phys + off) & ~(USB_PAGE_SIZE - 1);
197 /*------------------------------------------------------------------------*
198 * usb_pc_alloc_mem - allocate DMA'able memory
203 *------------------------------------------------------------------------*/
205 usb_pc_alloc_mem(struct usb_page_cache *pc, struct usb_page *pg,
206 usb_size_t size, usb_size_t align)
211 /* allocate zeroed memory */
214 ptr = malloc(size + align, XXX, XXX);
218 rem = (-((uintptr_t)ptr)) & (align - 1);
220 ptr = malloc(size, XXX, XXX);
226 /* setup page cache */
227 pc->buffer = ((uint8_t *)ptr) + rem;
229 pc->page_offset_buf = 0;
230 pc->page_offset_end = size;
233 pc->ismultiseg = (align == 1);
235 /* compute physical address */
236 usb_pc_common_mem_cb(pc, pc->buffer, size);
238 usb_pc_cpu_flush(pc);
242 /* reset most of the page cache */
244 pc->page_start = NULL;
245 pc->page_offset_buf = 0;
246 pc->page_offset_end = 0;
252 /*------------------------------------------------------------------------*
253 * usb_pc_free_mem - free DMA memory
255 * This function is NULL safe.
256 *------------------------------------------------------------------------*/
258 usb_pc_free_mem(struct usb_page_cache *pc)
260 if (pc != NULL && pc->buffer != NULL) {
266 /*------------------------------------------------------------------------*
267 * usb_pc_load_mem - load virtual memory into DMA
272 *------------------------------------------------------------------------*/
274 usb_pc_load_mem(struct usb_page_cache *pc, usb_size_t size, uint8_t sync)
276 /* setup page cache */
277 pc->page_offset_buf = 0;
278 pc->page_offset_end = size;
281 mtx_assert(pc->tag_parent->mtx, MA_OWNED);
284 /* compute physical address */
285 usb_pc_common_mem_cb(pc, pc->buffer, size);
289 * Call callback so that refcount is decremented
292 pc->tag_parent->dma_error = 0;
293 (pc->tag_parent->func) (pc->tag_parent);
298 /*------------------------------------------------------------------------*
299 * usb_pc_cpu_invalidate - invalidate CPU cache
300 *------------------------------------------------------------------------*/
302 usb_pc_cpu_invalidate(struct usb_page_cache *pc)
304 if (pc->page_offset_end == pc->page_offset_buf) {
305 /* nothing has been loaded into this page cache! */
311 /*------------------------------------------------------------------------*
312 * usb_pc_cpu_flush - flush CPU cache
313 *------------------------------------------------------------------------*/
315 usb_pc_cpu_flush(struct usb_page_cache *pc)
317 if (pc->page_offset_end == pc->page_offset_buf) {
318 /* nothing has been loaded into this page cache! */
324 /*------------------------------------------------------------------------*
325 * usb_pc_dmamap_create - create a DMA map
330 *------------------------------------------------------------------------*/
332 usb_pc_dmamap_create(struct usb_page_cache *pc, usb_size_t size)
334 return (0); /* NOP, success */
337 /*------------------------------------------------------------------------*
338 * usb_pc_dmamap_destroy
340 * This function is NULL safe.
341 *------------------------------------------------------------------------*/
343 usb_pc_dmamap_destroy(struct usb_page_cache *pc)
348 /*------------------------------------------------------------------------*
349 * usb_dma_tag_setup - initialise USB DMA tags
350 *------------------------------------------------------------------------*/
352 usb_dma_tag_setup(struct usb_dma_parent_tag *udpt,
353 struct usb_dma_tag *udt, bus_dma_tag_t dmat,
354 struct mtx *mtx, usb_dma_callback_t *func,
355 uint8_t ndmabits, uint8_t nudt)
357 memset(udpt, 0, sizeof(*udpt));
359 /* sanity checking */
363 /* something is corrupt */
366 /* initialise condition variable */
367 cv_init(udpt->cv, "USB DMA CV");
369 /* store some information */
373 udpt->utag_first = udt;
374 udpt->utag_max = nudt;
375 udpt->dma_bits = ndmabits;
378 memset(udt, 0, sizeof(*udt));
379 udt->tag_parent = udpt;
384 /*------------------------------------------------------------------------*
385 * usb_bus_tag_unsetup - factored out code
386 *------------------------------------------------------------------------*/
388 usb_dma_tag_unsetup(struct usb_dma_parent_tag *udpt)
390 struct usb_dma_tag *udt;
393 udt = udpt->utag_first;
394 nudt = udpt->utag_max;
401 if (udpt->utag_max) {
402 /* destroy the condition variable */
403 cv_destroy(udpt->cv);
407 /*------------------------------------------------------------------------*
410 * This function handles loading of virtual buffers into DMA and is
411 * only called when "dma_refcount" is zero.
412 *------------------------------------------------------------------------*/
414 usb_bdma_work_loop(struct usb_xfer_queue *pq)
416 struct usb_xfer_root *info;
417 struct usb_xfer *xfer;
418 usb_frcount_t nframes;
423 mtx_assert(info->xfer_mtx, MA_OWNED);
426 /* some error happened */
427 USB_BUS_LOCK(info->bus);
428 usbd_transfer_done(xfer, 0);
429 USB_BUS_UNLOCK(info->bus);
432 if (!xfer->flags_int.bdma_setup) {
434 usb_frlength_t frlength_0;
437 xfer->flags_int.bdma_setup = 1;
439 /* reset BUS-DMA load state */
443 if (xfer->flags_int.isochronous_xfr) {
444 /* only one frame buffer */
446 frlength_0 = xfer->sumlen;
448 /* can be multiple frame buffers */
449 nframes = xfer->nframes;
450 frlength_0 = xfer->frlengths[0];
454 * Set DMA direction first. This is needed to
455 * select the correct cache invalidate and cache
458 isread = USB_GET_DATA_ISREAD(xfer);
459 pg = xfer->dma_page_ptr;
461 if (xfer->flags_int.control_xfr &&
462 xfer->flags_int.control_hdr) {
464 if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) {
465 /* The device controller writes to memory */
466 xfer->frbuffers[0].isread = 1;
468 /* The host controller reads from memory */
469 xfer->frbuffers[0].isread = 0;
473 xfer->frbuffers[0].isread = isread;
477 * Setup the "page_start" pointer which points to an array of
478 * USB pages where information about the physical address of a
479 * page will be stored. Also initialise the "isread" field of
480 * the USB page caches.
482 xfer->frbuffers[0].page_start = pg;
484 info->dma_nframes = nframes;
485 info->dma_currframe = 0;
486 info->dma_frlength_0 = frlength_0;
488 pg += (frlength_0 / USB_PAGE_SIZE);
491 while (--nframes > 0) {
492 xfer->frbuffers[nframes].isread = isread;
493 xfer->frbuffers[nframes].page_start = pg;
495 pg += (xfer->frlengths[nframes] / USB_PAGE_SIZE);
500 if (info->dma_error) {
501 USB_BUS_LOCK(info->bus);
502 usbd_transfer_done(xfer, USB_ERR_DMA_LOAD_FAILED);
503 USB_BUS_UNLOCK(info->bus);
506 if (info->dma_currframe != info->dma_nframes) {
508 if (info->dma_currframe == 0) {
510 usb_pc_load_mem(xfer->frbuffers,
511 info->dma_frlength_0, 0);
514 nframes = info->dma_currframe;
515 usb_pc_load_mem(xfer->frbuffers + nframes,
516 xfer->frlengths[nframes], 0);
519 /* advance frame index */
520 info->dma_currframe++;
525 usb_bdma_pre_sync(xfer);
527 /* start loading next USB transfer, if any */
528 usb_command_wrapper(pq, NULL);
530 /* finally start the hardware */
531 usbd_pipe_enter(xfer);
534 /*------------------------------------------------------------------------*
535 * usb_bdma_done_event
537 * This function is called when the BUS-DMA has loaded virtual memory
539 *------------------------------------------------------------------------*/
541 usb_bdma_done_event(struct usb_dma_parent_tag *udpt)
543 struct usb_xfer_root *info;
545 info = USB_DMATAG_TO_XROOT(udpt);
547 mtx_assert(info->xfer_mtx, MA_OWNED);
550 info->dma_error = udpt->dma_error;
552 /* enter workloop again */
553 usb_command_wrapper(&info->dma_q,
557 /*------------------------------------------------------------------------*
560 * This function handles DMA synchronisation that must be done before
561 * an USB transfer is started.
562 *------------------------------------------------------------------------*/
564 usb_bdma_pre_sync(struct usb_xfer *xfer)
566 struct usb_page_cache *pc;
567 usb_frcount_t nframes;
569 if (xfer->flags_int.isochronous_xfr) {
570 /* only one frame buffer */
573 /* can be multiple frame buffers */
574 nframes = xfer->nframes;
577 pc = xfer->frbuffers;
582 usb_pc_cpu_invalidate(pc);
584 usb_pc_cpu_flush(pc);
590 /*------------------------------------------------------------------------*
593 * This function handles DMA synchronisation that must be done after
594 * an USB transfer is complete.
595 *------------------------------------------------------------------------*/
597 usb_bdma_post_sync(struct usb_xfer *xfer)
599 struct usb_page_cache *pc;
600 usb_frcount_t nframes;
602 if (xfer->flags_int.isochronous_xfr) {
603 /* only one frame buffer */
606 /* can be multiple frame buffers */
607 nframes = xfer->nframes;
610 pc = xfer->frbuffers;
614 usb_pc_cpu_invalidate(pc);