2 * Copyright (c) 2005 Olivier Houchard. All rights reserved.
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
14 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
15 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
16 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
17 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
18 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
19 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
20 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
21 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
22 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 #include <sys/cdefs.h>
26 __FBSDID("$FreeBSD$");
27 #include <sys/param.h>
28 #include <sys/systm.h>
30 #include <sys/kernel.h>
31 #include <sys/module.h>
32 #include <sys/malloc.h>
35 #include <sys/mutex.h>
40 #include <vm/vm_map.h>
41 #include <machine/bus.h>
42 #include <machine/cpu.h>
43 #include <machine/md_var.h>
45 #include <arm/xscale/i80321/i80321reg.h>
46 #include <arm/xscale/i80321/i80321var.h>
47 #include <arm/xscale/i80321/iq80321reg.h>
48 #include <arm/xscale/i80321/iq80321var.h>
49 #include <arm/xscale/i80321/i80321_intr.h>
51 typedef struct i80321_dmadesc_s {
53 vm_paddr_t low_pciaddr;
54 vm_paddr_t high_pciaddr;
55 vm_paddr_t local_addr;
59 } __packed i80321_dmadesc_t;
61 typedef struct i80321_dmaring_s {
62 i80321_dmadesc_t *desc;
67 #define DMA_RING_SIZE 64
69 struct i80321_dma_softc {
70 bus_space_tag_t sc_st;
71 bus_space_handle_t sc_dma_sh;
73 i80321_dmaring_t dmaring[DMA_RING_SIZE];
81 i80321_dma_probe(device_t dev)
83 device_set_desc(dev, "I80321 DMA Unit");
87 static struct i80321_dma_softc *softcs[2]; /* XXX */
90 i80321_mapphys(void *arg, bus_dma_segment_t *segs, int nseg, int error)
92 vm_paddr_t *addr = (vm_paddr_t *)arg;
94 *addr = segs->ds_addr;
97 #define DMA_REG_WRITE(softc, reg, val) \
98 bus_space_write_4((softc)->sc_st, (softc)->sc_dma_sh, \
100 #define DMA_REG_READ(softc, reg) \
101 bus_space_read_4((softc)->sc_st, (softc)->sc_dma_sh, \
104 #define DMA_CLEAN_MASK (0x2|0x4|0x8|0x20|0x100|0x200)
105 static int dma_memcpy(void *, void *, int, int);
108 i80321_dma_attach(device_t dev)
110 struct i80321_dma_softc *softc = device_get_softc(dev);
111 struct i80321_softc *sc = device_get_softc(device_get_parent(dev));
112 int unit = device_get_unit(dev);
113 i80321_dmadesc_t *dmadescs;
115 mtx_init(&softc->mtx, "DMA engine mtx", NULL, MTX_SPIN);
116 softc->sc_st = sc->sc_st;
117 if (bus_space_subregion(softc->sc_st, sc->sc_sh, unit == 0 ?
118 VERDE_DMA_BASE0 : VERDE_DMA_BASE1, VERDE_DMA_SIZE,
119 &softc->sc_dma_sh) != 0)
120 panic("%s: unable to subregion DMA registers",
121 device_get_name(dev));
122 if (bus_dma_tag_create(NULL, sizeof(i80321_dmadesc_t),
123 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
124 DMA_RING_SIZE * sizeof(i80321_dmadesc_t), 1,
125 sizeof(i80321_dmadesc_t), BUS_DMA_ALLOCNOW, busdma_lock_mutex,
126 &Giant, &softc->dmatag))
127 panic("Couldn't create a dma tag");
128 DMA_REG_WRITE(softc, 0, 0);
129 if (bus_dmamem_alloc(softc->dmatag, (void **)&dmadescs,
130 BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &softc->dmaring[0].map))
131 panic("Couldn't alloc dma memory");
132 for (int i = 0; i < DMA_RING_SIZE; i++) {
134 if (bus_dmamap_create(softc->dmatag, 0,
135 &softc->dmaring[i].map))
136 panic("Couldn't alloc dmamap");
137 softc->dmaring[i].desc = &dmadescs[i];
138 bus_dmamap_load(softc->dmatag, softc->dmaring[i].map,
139 softc->dmaring[i].desc, sizeof(i80321_dmadesc_t),
140 i80321_mapphys, &softc->dmaring[i].phys_addr, 0);
143 softcs[unit] = softc;
144 _arm_memcpy = dma_memcpy;
145 _min_memcpy_size = 1024;
150 virt_addr_is_valid(void *addr, int len, int write, int is_kernel)
159 else if (subyte(addr, 0) != 0) {
164 badaddr_read(addr, 1, &tmp);
165 else if (fubyte(addr) == -1) {
169 to_nextpage = ((vm_offset_t)addr & ~PAGE_MASK) +
170 PAGE_SIZE - (vm_offset_t)addr;
171 if (to_nextpage >= len)
174 addr = (void *)((vm_offset_t)addr + to_nextpage);
180 dma_memcpy(void *dst, void *src, int len, int flags)
182 struct i80321_dma_softc *sc;
183 i80321_dmadesc_t *desc;
188 int to_nextpagesrc, to_nextpagedst;
190 vm_paddr_t pa, pa2, tmppa;
191 pmap_t pmap = vmspace_pmap(curthread->td_proc->p_vmspace);
193 if (!softcs[0] || !softcs[1])
195 mtx_lock_spin(&softcs[0]->mtx);
196 if (softcs[0]->flags & BUSY) {
197 mtx_unlock_spin(&softcs[0]->mtx);
198 mtx_lock_spin(&softcs[1]->mtx);
199 if (softcs[1]->flags & BUSY) {
200 mtx_unlock(&softcs[1]->mtx);
207 mtx_unlock_spin(&sc->mtx);
208 desc = sc->dmaring[0].desc;
209 if (flags & IS_PHYSICAL) {
211 desc->low_pciaddr = (vm_paddr_t)src;
212 desc->high_pciaddr = 0;
213 desc->local_addr = (vm_paddr_t)dst;
215 desc->descr_ctrl = 1 << 6; /* Local memory to local memory. */
216 bus_dmamap_sync(sc->dmatag,
218 BUS_DMASYNC_PREWRITE);
220 if (!virt_addr_is_valid(dst, len, 1, !(flags & DST_IS_USER)) ||
221 !virt_addr_is_valid(src, len, 0, !(flags & SRC_IS_USER))) {
222 mtx_lock_spin(&sc->mtx);
224 mtx_unlock_spin(&sc->mtx);
227 cpu_dcache_wb_range((vm_offset_t)src, len);
228 if ((vm_offset_t)dst & (31))
229 cpu_dcache_wb_range((vm_offset_t)dst & ~31, 32);
230 if (((vm_offset_t)dst + len) & 31)
231 cpu_dcache_wb_range(((vm_offset_t)dst + len) & ~31,
233 cpu_dcache_inv_range((vm_offset_t)dst, len);
235 pa = (flags & SRC_IS_USER) ?
236 pmap_extract(pmap, (vm_offset_t)src) :
238 pa2 = (flags & DST_IS_USER) ?
239 pmap_extract(pmap, (vm_offset_t)dst) :
241 to_nextpagesrc = ((vm_offset_t)src & ~PAGE_MASK) +
242 PAGE_SIZE - (vm_offset_t)src;
243 to_nextpagedst = ((vm_offset_t)dst & ~PAGE_MASK) +
244 PAGE_SIZE - (vm_offset_t)dst;
245 while (to_nextpagesrc < tmplen) {
246 tmppa = (flags & SRC_IS_USER) ?
247 pmap_extract(pmap, (vm_offset_t)src +
249 vtophys((vm_offset_t)src +
251 if (tmppa != pa + to_nextpagesrc)
253 to_nextpagesrc += PAGE_SIZE;
255 while (to_nextpagedst < tmplen) {
256 tmppa = (flags & DST_IS_USER) ?
257 pmap_extract(pmap, (vm_offset_t)dst +
259 vtophys((vm_offset_t)dst +
261 if (tmppa != pa2 + to_nextpagedst)
263 to_nextpagedst += PAGE_SIZE;
265 min_hop = to_nextpagedst > to_nextpagesrc ?
266 to_nextpagesrc : to_nextpagedst;
269 memcpy(dst, src, min_hop);
270 cpu_dcache_wbinv_range((vm_offset_t)dst,
273 src = (void *)((vm_offset_t)src + min_hop);
274 dst = (void *)((vm_offset_t)dst + min_hop);
275 if (tmplen <= 0 && descnb > 0) {
276 sc->dmaring[descnb - 1].desc->next_desc
278 bus_dmamap_sync(sc->dmatag,
279 sc->dmaring[descnb - 1].map,
280 BUS_DMASYNC_PREWRITE);
284 desc->low_pciaddr = pa;
285 desc->high_pciaddr = 0;
286 desc->local_addr = pa2;
287 desc->count = tmplen > min_hop ? min_hop : tmplen;
288 desc->descr_ctrl = 1 << 6;
289 if (min_hop < tmplen) {
291 src = (void *)((vm_offset_t)src + min_hop);
292 dst = (void *)((vm_offset_t)dst + min_hop);
295 if (descnb + 1 >= DMA_RING_SIZE) {
296 mtx_lock_spin(&sc->mtx);
298 mtx_unlock_spin(&sc->mtx);
302 desc->next_desc = sc->dmaring[descnb + 1].
304 bus_dmamap_sync(sc->dmatag,
305 sc->dmaring[descnb].map,
306 BUS_DMASYNC_PREWRITE);
307 desc = sc->dmaring[descnb + 1].desc;
311 bus_dmamap_sync(sc->dmatag,
312 sc->dmaring[descnb].map,
313 BUS_DMASYNC_PREWRITE);
319 DMA_REG_WRITE(sc, 4 /* Status register */,
320 DMA_REG_READ(sc, 4) | DMA_CLEAN_MASK);
321 DMA_REG_WRITE(sc, 0x10 /* Descriptor addr */,
322 sc->dmaring[0].phys_addr);
323 DMA_REG_WRITE(sc, 0 /* Control register */, 1 | 2/* Start transfer */);
324 while ((csr = DMA_REG_READ(sc, 0x4)) & (1 << 10));
325 /* Wait until it's done. */
326 if (csr & 0x2e) /* error */
330 DMA_REG_WRITE(sc, 0, 0);
331 mtx_lock_spin(&sc->mtx);
333 mtx_unlock_spin(&sc->mtx);
337 static device_method_t i80321_dma_methods[] = {
338 DEVMETHOD(device_probe, i80321_dma_probe),
339 DEVMETHOD(device_attach, i80321_dma_attach),
343 static driver_t i80321_dma_driver = {
346 sizeof(struct i80321_dma_softc),
349 static devclass_t i80321_dma_devclass;
351 DRIVER_MODULE(i80321_dma, iq, i80321_dma_driver, i80321_dma_devclass, 0, 0);