sys/arm/xscale/i80321/i80321_dma.c

   1 /*-
   2  * Copyright (c) 2005 Olivier Houchard.  All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  14  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  15  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  16  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  17  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  18  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  19  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  20  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  21  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  22  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  23  */
  24
  25 #include <sys/cdefs.h>
  26 __FBSDID("$FreeBSD$");
  27 #include <sys/param.h>
  28 #include <sys/systm.h>
  29 #include <sys/bus.h>
  30 #include <sys/kernel.h>
  31 #include <sys/module.h>
  32 #include <sys/malloc.h>
  33 #include <sys/rman.h>
  34 #include <sys/lock.h>
  35 #include <sys/mutex.h>
  36 #include <sys/proc.h>
  37
  38 #include <vm/vm.h>
  39 #include <vm/pmap.h>
  40 #include <vm/vm_map.h>
  41 #include <machine/bus.h>
  42 #include <machine/cpu.h>
  43 #include <machine/md_var.h>
  44
  45 #include <arm/xscale/i80321/i80321reg.h>
  46 #include <arm/xscale/i80321/i80321var.h>
  47 #include <arm/xscale/i80321/iq80321reg.h>
  48 #include <arm/xscale/i80321/iq80321var.h>
  49 #include <arm/xscale/i80321/i80321_intr.h>
  50
  51 typedef struct i80321_dmadesc_s {
  52         vm_paddr_t next_desc;
  53         vm_paddr_t low_pciaddr;
  54         vm_paddr_t high_pciaddr;
  55         vm_paddr_t local_addr;
  56         vm_size_t count;
  57         uint32_t descr_ctrl;
  58         uint64_t unused;
  59 } __packed      i80321_dmadesc_t;
  60
  61 typedef struct i80321_dmaring_s {
  62         i80321_dmadesc_t *desc;
  63         vm_paddr_t phys_addr;
  64         bus_dmamap_t map;
  65 } i80321_dmaring_t;
  66
  67 #define DMA_RING_SIZE 64
  68
  69 struct i80321_dma_softc {
  70         bus_space_tag_t sc_st;
  71         bus_space_handle_t sc_dma_sh;
  72         bus_dma_tag_t dmatag;
  73         i80321_dmaring_t dmaring[DMA_RING_SIZE];
  74         int flags;
  75 #define BUSY    0x1
  76         int unit;
  77         struct mtx mtx;
  78 };
  79
  80 static int
  81 i80321_dma_probe(device_t dev)
  82 {
  83         device_set_desc(dev, "I80321 DMA Unit");
  84         return (0);
  85 }
  86
  87 static struct i80321_dma_softc *softcs[2]; /* XXX */
  88
  89 static void
  90 i80321_mapphys(void *arg, bus_dma_segment_t *segs, int nseg, int error)
  91 {
  92         vm_paddr_t *addr = (vm_paddr_t *)arg;
  93
  94         *addr = segs->ds_addr;
  95 }
  96
  97 #define DMA_REG_WRITE(softc, reg, val) \
  98     bus_space_write_4((softc)->sc_st, (softc)->sc_dma_sh, \
  99         (reg), (val))
 100 #define DMA_REG_READ(softc, reg) \
 101     bus_space_read_4((softc)->sc_st, (softc)->sc_dma_sh, \
 102         (reg))
 103
 104 #define DMA_CLEAN_MASK (0x2|0x4|0x8|0x20|0x100|0x200)
 105 static int dma_memcpy(void *, void *, int, int);
 106
 107 static int
 108 i80321_dma_attach(device_t dev)
 109 {
 110         struct i80321_dma_softc *softc = device_get_softc(dev);
 111         struct i80321_softc *sc = device_get_softc(device_get_parent(dev));
 112         int unit = device_get_unit(dev);
 113         i80321_dmadesc_t *dmadescs;
 114
 115         mtx_init(&softc->mtx, "DMA engine mtx", NULL, MTX_SPIN);
 116         softc->sc_st = sc->sc_st;
 117         if (bus_space_subregion(softc->sc_st, sc->sc_sh, unit == 0 ?
 118             VERDE_DMA_BASE0 : VERDE_DMA_BASE1, VERDE_DMA_SIZE,
 119             &softc->sc_dma_sh) != 0)
 120                 panic("%s: unable to subregion DMA registers",
 121                     device_get_name(dev));
 122         if (bus_dma_tag_create(NULL, sizeof(i80321_dmadesc_t),
 123             0, BUS_SPACE_MAXADDR,  BUS_SPACE_MAXADDR, NULL, NULL,
 124             DMA_RING_SIZE * sizeof(i80321_dmadesc_t), 1,
 125             sizeof(i80321_dmadesc_t), BUS_DMA_ALLOCNOW, busdma_lock_mutex,
 126             &Giant, &softc->dmatag))
 127                 panic("Couldn't create a dma tag");
 128         DMA_REG_WRITE(softc, 0, 0);
 129         if (bus_dmamem_alloc(softc->dmatag, (void **)&dmadescs,
 130             BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &softc->dmaring[0].map))
 131                 panic("Couldn't alloc dma memory");
 132         for (int i = 0; i < DMA_RING_SIZE; i++) {
 133                 if (i > 0)
 134                         if (bus_dmamap_create(softc->dmatag, 0,
 135                             &softc->dmaring[i].map))
 136                                 panic("Couldn't alloc dmamap");
 137                 softc->dmaring[i].desc = &dmadescs[i];
 138                 bus_dmamap_load(softc->dmatag, softc->dmaring[i].map,
 139                     softc->dmaring[i].desc, sizeof(i80321_dmadesc_t),
 140                     i80321_mapphys, &softc->dmaring[i].phys_addr, 0);
 141         }
 142         softc->unit = unit;
 143         softcs[unit] = softc;
 144         _arm_memcpy = dma_memcpy;
 145         _min_memcpy_size = 1024;
 146         return (0);
 147 }
 148
 149 static __inline int
 150 virt_addr_is_valid(void *addr, int len, int write, int is_kernel)
 151 {
 152         int to_nextpage;
 153         char tmp = 0;
 154
 155         while (len > 0) {
 156                 if (write) {
 157                         if (is_kernel)
 158                                 *(char *)addr = 0;
 159                         else if (subyte(addr, 0) != 0) {
 160                                 return (0);
 161                         }
 162                 } else {
 163                         if (is_kernel)
 164                                 badaddr_read(addr, 1, &tmp);
 165                         else if (fubyte(addr) == -1) {
 166                                 return (0);
 167                         }
 168                 }
 169                 to_nextpage = ((vm_offset_t)addr & ~PAGE_MASK) +
 170                     PAGE_SIZE - (vm_offset_t)addr;
 171                 if (to_nextpage >= len)
 172                         break;
 173                 len -= to_nextpage;
 174                 addr = (void *)((vm_offset_t)addr + to_nextpage);
 175         }
 176         return (1);
 177 }
 178
 179 static int
 180 dma_memcpy(void *dst, void *src, int len, int flags)
 181 {
 182         struct i80321_dma_softc *sc;
 183         i80321_dmadesc_t *desc;
 184         int ret;
 185         int csr;
 186         int descnb = 0;
 187         int tmplen = len;
 188         int to_nextpagesrc, to_nextpagedst;
 189         int min_hop;
 190         vm_paddr_t pa, pa2, tmppa;
 191         pmap_t pmap = vmspace_pmap(curthread->td_proc->p_vmspace);
 192
 193         if (!softcs[0] || !softcs[1])
 194                 return (-1);
 195         mtx_lock_spin(&softcs[0]->mtx);
 196         if (softcs[0]->flags & BUSY) {
 197                 mtx_unlock_spin(&softcs[0]->mtx);
 198                 mtx_lock_spin(&softcs[1]->mtx);
 199                 if (softcs[1]->flags & BUSY) {
 200                         mtx_unlock(&softcs[1]->mtx);
 201                         return (-1);
 202                 }
 203                 sc = softcs[1];
 204         } else
 205                 sc = softcs[0];
 206         sc->flags |= BUSY;
 207         mtx_unlock_spin(&sc->mtx);
 208         desc = sc->dmaring[0].desc;
 209         if (flags & IS_PHYSICAL) {
 210                 desc->next_desc = 0;
 211                 desc->low_pciaddr = (vm_paddr_t)src;
 212                 desc->high_pciaddr = 0;
 213                 desc->local_addr = (vm_paddr_t)dst;
 214                 desc->count = len;
 215                 desc->descr_ctrl = 1 << 6; /* Local memory to local memory. */
 216                 bus_dmamap_sync(sc->dmatag,
 217                     sc->dmaring[0].map,
 218                     BUS_DMASYNC_PREWRITE);
 219         } else {
 220                 if (!virt_addr_is_valid(dst, len, 1, !(flags & DST_IS_USER)) ||
 221                     !virt_addr_is_valid(src, len, 0, !(flags & SRC_IS_USER))) {
 222                         mtx_lock_spin(&sc->mtx);
 223                         sc->flags &= ~BUSY;
 224                         mtx_unlock_spin(&sc->mtx);
 225                         return (-1);
 226                 }
 227                 cpu_dcache_wb_range((vm_offset_t)src, len);
 228                 if ((vm_offset_t)dst & (31))
 229                         cpu_dcache_wb_range((vm_offset_t)dst & ~31, 32);
 230                 if (((vm_offset_t)dst + len) & 31)
 231                         cpu_dcache_wb_range(((vm_offset_t)dst + len) & ~31,
 232                             32);
 233                 cpu_dcache_inv_range((vm_offset_t)dst, len);
 234                 while (tmplen > 0) {
 235                         pa = (flags & SRC_IS_USER) ?
 236                             pmap_extract(pmap, (vm_offset_t)src) :
 237                                     vtophys(src);
 238                         pa2 = (flags & DST_IS_USER) ?
 239                             pmap_extract(pmap, (vm_offset_t)dst) :
 240                                     vtophys(dst);
 241                         to_nextpagesrc = ((vm_offset_t)src & ~PAGE_MASK) +
 242                             PAGE_SIZE - (vm_offset_t)src;
 243                         to_nextpagedst = ((vm_offset_t)dst & ~PAGE_MASK) +
 244                             PAGE_SIZE - (vm_offset_t)dst;
 245                         while (to_nextpagesrc < tmplen) {
 246                                 tmppa = (flags & SRC_IS_USER) ?
 247                                     pmap_extract(pmap, (vm_offset_t)src +
 248                                     to_nextpagesrc) :
 249                                             vtophys((vm_offset_t)src +
 250                                                 to_nextpagesrc);
 251                                 if (tmppa != pa + to_nextpagesrc)
 252                                         break;
 253                                 to_nextpagesrc += PAGE_SIZE;
 254                         }
 255                         while (to_nextpagedst < tmplen) {
 256                                 tmppa = (flags & DST_IS_USER) ?
 257                                     pmap_extract(pmap, (vm_offset_t)dst +
 258                                     to_nextpagedst) :
 259                                             vtophys((vm_offset_t)dst +
 260                                                 to_nextpagedst);
 261                                 if (tmppa != pa2 + to_nextpagedst)
 262                                         break;
 263                                 to_nextpagedst += PAGE_SIZE;
 264                         }
 265                         min_hop = to_nextpagedst > to_nextpagesrc ?
 266                             to_nextpagesrc : to_nextpagedst;
 267                         if (min_hop < 64) {
 268                                 tmplen -= min_hop;
 269                                 memcpy(dst, src, min_hop);
 270                                 cpu_dcache_wbinv_range((vm_offset_t)dst,
 271                                     min_hop);
 272
 273                                 src = (void *)((vm_offset_t)src + min_hop);
 274                                 dst = (void *)((vm_offset_t)dst + min_hop);
 275                                 if (tmplen <= 0 && descnb > 0) {
 276                                         sc->dmaring[descnb - 1].desc->next_desc
 277                                             = 0;
 278                                         bus_dmamap_sync(sc->dmatag,
 279                                             sc->dmaring[descnb - 1].map,
 280                                             BUS_DMASYNC_PREWRITE);
 281                                 }
 282                                 continue;
 283                         }
 284                         desc->low_pciaddr = pa;
 285                         desc->high_pciaddr = 0;
 286                         desc->local_addr = pa2;
 287                         desc->count = tmplen > min_hop ? min_hop : tmplen;
 288                         desc->descr_ctrl = 1 << 6;
 289                         if (min_hop < tmplen) {
 290                                 tmplen -= min_hop;
 291                                 src = (void *)((vm_offset_t)src + min_hop);
 292                                 dst = (void *)((vm_offset_t)dst + min_hop);
 293                         } else
 294                                 tmplen = 0;
 295                         if (descnb + 1 >= DMA_RING_SIZE) {
 296                                 mtx_lock_spin(&sc->mtx);
 297                                 sc->flags &= ~BUSY;
 298                                 mtx_unlock_spin(&sc->mtx);
 299                                 return (-1);
 300                         }
 301                         if (tmplen > 0) {
 302                                 desc->next_desc = sc->dmaring[descnb + 1].
 303                                     phys_addr;
 304                                 bus_dmamap_sync(sc->dmatag,
 305                                     sc->dmaring[descnb].map,
 306                                     BUS_DMASYNC_PREWRITE);
 307                                 desc = sc->dmaring[descnb + 1].desc;
 308                                 descnb++;
 309                         } else {
 310                                 desc->next_desc = 0;
 311                                 bus_dmamap_sync(sc->dmatag,
 312                                     sc->dmaring[descnb].map,
 313                                     BUS_DMASYNC_PREWRITE);
 314                         }
 315
 316                 }
 317
 318         }
 319         DMA_REG_WRITE(sc, 4 /* Status register */,
 320             DMA_REG_READ(sc, 4) | DMA_CLEAN_MASK);
 321         DMA_REG_WRITE(sc, 0x10 /* Descriptor addr */,
 322             sc->dmaring[0].phys_addr);
 323         DMA_REG_WRITE(sc, 0 /* Control register */, 1 | 2/* Start transfer */);
 324         while ((csr = DMA_REG_READ(sc, 0x4)) & (1 << 10));
 325         /* Wait until it's done. */
 326         if (csr & 0x2e) /* error */
 327                 ret = -1;
 328         else
 329                 ret = 0;
 330         DMA_REG_WRITE(sc, 0, 0);
 331         mtx_lock_spin(&sc->mtx);
 332         sc->flags &= ~BUSY;
 333         mtx_unlock_spin(&sc->mtx);
 334         return (ret);
 335 }
 336
 337 static device_method_t i80321_dma_methods[] = {
 338         DEVMETHOD(device_probe, i80321_dma_probe),
 339         DEVMETHOD(device_attach, i80321_dma_attach),
 340         {0, 0},
 341 };
 342
 343 static driver_t i80321_dma_driver = {
 344         "i80321_dma",
 345         i80321_dma_methods,
 346         sizeof(struct i80321_dma_softc),
 347 };
 348
 349 static devclass_t i80321_dma_devclass;
 350
 351 DRIVER_MODULE(i80321_dma, iq, i80321_dma_driver, i80321_dma_devclass, 0, 0);