2 * SPDX-License-Identifier: BSD-2-Clause-NetBSD AND BSD-3-Clause
4 * Copyright (c) 1999, 2000 Matthew R. Green
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * from: NetBSD: iommu.c,v 1.82 2008/05/30 02:29:37 mrg Exp
31 * Copyright (c) 1999-2002 Eduardo Horvath
32 * Copyright (c) 2001-2003 Thomas Moestl
33 * Copyright (c) 2007, 2009 Marius Strobl <marius@FreeBSD.org>
34 * All rights reserved.
36 * Redistribution and use in source and binary forms, with or without
37 * modification, are permitted provided that the following conditions
39 * 1. Redistributions of source code must retain the above copyright
40 * notice, this list of conditions and the following disclaimer.
41 * 2. Redistributions in binary form must reproduce the above copyright
42 * notice, this list of conditions and the following disclaimer in the
43 * documentation and/or other materials provided with the distribution.
44 * 3. The name of the author may not be used to endorse or promote products
45 * derived from this software without specific prior written permission.
47 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
48 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
49 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
50 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
51 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
52 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
53 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
54 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
55 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
56 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
59 * from: NetBSD: sbus.c,v 1.50 2002/06/20 18:26:24 eeh Exp
62 #include <sys/cdefs.h>
63 __FBSDID("$FreeBSD$");
66 * UltraSPARC IOMMU support; used by both the PCI and SBus code.
69 * - Support sub-page boundaries.
70 * - Fix alignment handling for small allocations (the possible page offset
71 * of malloc()ed memory is not handled at all). Revise interaction of
72 * alignment with the load_mbuf and load_uio functions.
73 * - Handle lowaddr and highaddr in some way, and try to work out a way
74 * for filter callbacks to work. Currently, only lowaddr is honored
75 * in that no addresses above it are considered at all.
76 * - Implement BUS_DMA_ALLOCNOW in bus_dma_tag_create as far as possible.
77 * - Check the possible return values and callback error arguments;
78 * the callback currently gets called in error conditions where it should
80 * - When running out of DVMA space, return EINPROGRESS in the non-
81 * BUS_DMA_NOWAIT case and delay the callback until sufficient space
85 #include "opt_iommu.h"
87 #include <sys/param.h>
88 #include <sys/kernel.h>
90 #include <sys/malloc.h>
92 #include <sys/mutex.h>
95 #include <sys/systm.h>
100 #include <vm/vm_map.h>
102 #include <machine/asi.h>
103 #include <machine/bus.h>
104 #include <machine/bus_private.h>
105 #include <machine/iommureg.h>
106 #include <machine/resource.h>
107 #include <machine/ver.h>
109 #include <sys/rman.h>
111 #include <machine/iommuvar.h>
116 #define IOMMU_MAX_PRE (32 * 1024)
117 #define IOMMU_MAX_PRE_SEG 3
119 /* Threshold for using the streaming buffer */
120 #define IOMMU_STREAM_THRESH 128
122 static MALLOC_DEFINE(M_IOMMU, "dvmamem", "IOMMU DVMA Buffers");
124 static int iommu_strbuf_flush_sync(struct iommu_state *);
126 static void iommu_diag(struct iommu_state *, vm_offset_t va);
132 #define IOMMU_READ8(is, reg, off) \
133 bus_space_read_8((is)->is_bustag, (is)->is_bushandle, \
135 #define IOMMU_WRITE8(is, reg, off, v) \
136 bus_space_write_8((is)->is_bustag, (is)->is_bushandle, \
137 (is)->reg + (off), (v))
139 #define IOMMU_HAS_SB(is) \
140 ((is)->is_sb[0] != 0 || (is)->is_sb[1] != 0)
143 * Always overallocate one page; this is needed to handle alignment of the
144 * buffer, so it makes sense using a lazy allocation scheme.
146 #define IOMMU_SIZE_ROUNDUP(sz) \
147 (round_io_page(sz) + IO_PAGE_SIZE)
149 #define IOMMU_SET_TTE(is, va, tte) \
150 ((is)->is_tsb[IOTSBSLOT(va)] = (tte))
151 #define IOMMU_GET_TTE(is, va) \
152 (is)->is_tsb[IOTSBSLOT(va)]
154 /* Resource helpers */
155 #define IOMMU_RES_START(res) \
156 ((bus_addr_t)rman_get_start(res) << IO_PAGE_SHIFT)
157 #define IOMMU_RES_END(res) \
158 ((bus_addr_t)(rman_get_end(res) + 1) << IO_PAGE_SHIFT)
159 #define IOMMU_RES_SIZE(res) \
160 ((bus_size_t)rman_get_size(res) << IO_PAGE_SHIFT)
162 /* Helpers for struct bus_dmamap_res */
163 #define BDR_START(r) IOMMU_RES_START((r)->dr_res)
164 #define BDR_END(r) IOMMU_RES_END((r)->dr_res)
165 #define BDR_SIZE(r) IOMMU_RES_SIZE((r)->dr_res)
168 #define IS_LOCK(is) mtx_lock(&is->is_mtx)
169 #define IS_LOCK_ASSERT(is) mtx_assert(&is->is_mtx, MA_OWNED)
170 #define IS_UNLOCK(is) mtx_unlock(&is->is_mtx)
172 /* Flush a page from the TLB. No locking required, since this is atomic. */
174 iommu_tlb_flush(struct iommu_state *is, bus_addr_t va)
177 if ((is->is_flags & IOMMU_FIRE) != 0)
179 * Direct page flushing is not supported and also not
180 * necessary due to cache snooping.
183 IOMMU_WRITE8(is, is_iommu, IMR_FLUSH, va);
187 * Flush a page from the streaming buffer. No locking required, since this
191 iommu_strbuf_flushpg(struct iommu_state *is, bus_addr_t va)
195 for (i = 0; i < 2; i++)
196 if (is->is_sb[i] != 0)
197 IOMMU_WRITE8(is, is_sb[i], ISR_PGFLUSH, va);
201 * Flush an address from the streaming buffer(s); this is an asynchronous
202 * operation. To make sure that it has completed, iommu_strbuf_sync() needs
203 * to be called. No locking required.
206 iommu_strbuf_flush(struct iommu_state *is, bus_addr_t va)
209 iommu_strbuf_flushpg(is, va);
212 /* Synchronize all outstanding flush operations. */
214 iommu_strbuf_sync(struct iommu_state *is)
218 iommu_strbuf_flush_sync(is);
221 /* LRU queue handling for lazy resource allocation. */
223 iommu_map_insq(struct iommu_state *is, bus_dmamap_t map)
227 if (!SLIST_EMPTY(&map->dm_reslist)) {
229 TAILQ_REMOVE(&is->is_maplruq, map, dm_maplruq);
230 TAILQ_INSERT_TAIL(&is->is_maplruq, map, dm_maplruq);
236 iommu_map_remq(struct iommu_state *is, bus_dmamap_t map)
241 TAILQ_REMOVE(&is->is_maplruq, map, dm_maplruq);
246 * initialise the UltraSPARC IOMMU (PCI or SBus):
247 * - allocate and setup the iotsb.
249 * - initialise the streaming buffers (if they exist)
250 * - create a private DVMA map.
253 iommu_init(const char *name, struct iommu_state *is, u_int tsbsize,
254 uint32_t iovabase, u_int resvpg)
258 uint64_t end, obpmap, obpptsb, tte;
259 u_int maxtsbsize, obptsbentries, obptsbsize, slot, tsbentries;
265 * The sun4u IOMMU is part of the PCI or SBus controller so we
266 * will deal with it here..
268 * The IOMMU address space always ends at 0xffffe000, but the starting
269 * address depends on the size of the map. The map size is 1024 * 2 ^
270 * is->is_tsbsize entries, where each entry is 8 bytes. The start of
271 * the map can be calculated by (0xffffe000 << (8 + is->is_tsbsize)).
273 if ((is->is_flags & IOMMU_FIRE) != 0) {
274 maxtsbsize = IOMMU_TSB512K;
276 * We enable bypass in order to be able to use a physical
277 * address for the event queue base.
279 is->is_cr = IOMMUCR_SE | IOMMUCR_CM_C_TLB_TBW | IOMMUCR_BE;
281 maxtsbsize = IOMMU_TSB128K;
282 is->is_cr = (tsbsize << IOMMUCR_TSBSZ_SHIFT) | IOMMUCR_DE;
284 if (tsbsize > maxtsbsize)
285 panic("%s: unsupported TSB size ", __func__);
286 tsbentries = IOMMU_TSBENTRIES(tsbsize);
287 is->is_cr |= IOMMUCR_EN;
288 is->is_tsbsize = tsbsize;
289 is->is_dvmabase = iovabase;
291 is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize);
293 size = IOTSB_BASESZ << is->is_tsbsize;
294 printf("%s: DVMA map: %#lx to %#lx %d entries%s\n", name,
295 is->is_dvmabase, is->is_dvmabase +
296 (size << (IO_PAGE_SHIFT - IOTTE_SHIFT)) - 1, tsbentries,
297 IOMMU_HAS_SB(is) ? ", streaming buffer" : "");
300 * Set up resource mamangement.
302 mtx_init(&is->is_mtx, "iommu", NULL, MTX_DEF);
303 end = is->is_dvmabase + (size << (IO_PAGE_SHIFT - IOTTE_SHIFT));
304 is->is_dvma_rman.rm_type = RMAN_ARRAY;
305 is->is_dvma_rman.rm_descr = "DVMA Memory";
306 if (rman_init(&is->is_dvma_rman) != 0 ||
307 rman_manage_region(&is->is_dvma_rman,
308 (is->is_dvmabase >> IO_PAGE_SHIFT) + resvpg,
309 (end >> IO_PAGE_SHIFT) - 1) != 0)
310 panic("%s: could not initialize DVMA rman", __func__);
311 TAILQ_INIT(&is->is_maplruq);
314 * Allocate memory for I/O page tables. They need to be
315 * physically contiguous.
317 is->is_tsb = contigmalloc(size, M_DEVBUF, M_NOWAIT, 0, ~0UL,
319 if (is->is_tsb == NULL)
320 panic("%s: contigmalloc failed", __func__);
321 is->is_ptsb = pmap_kextract((vm_offset_t)is->is_tsb);
322 bzero(is->is_tsb, size);
325 * Add the PROM mappings to the kernel IOTSB if desired.
326 * Note that the firmware of certain Darwin boards doesn't set
327 * the TSB size correctly.
329 if ((is->is_flags & IOMMU_FIRE) != 0)
330 obptsbsize = (IOMMU_READ8(is, is_iommu, IMR_TSB) &
331 IOMMUTB_TSBSZ_MASK) >> IOMMUTB_TSBSZ_SHIFT;
333 obptsbsize = (IOMMU_READ8(is, is_iommu, IMR_CTL) &
334 IOMMUCR_TSBSZ_MASK) >> IOMMUCR_TSBSZ_SHIFT;
335 obptsbentries = IOMMU_TSBENTRIES(obptsbsize);
337 printf("%s: PROM IOTSB size: %d (%d entries)\n", name,
338 obptsbsize, obptsbentries);
339 if ((is->is_flags & IOMMU_PRESERVE_PROM) != 0 &&
340 !(PCPU_GET(impl) == CPU_IMPL_ULTRASPARCIIi && obptsbsize == 7)) {
341 if (obptsbentries > tsbentries)
342 panic("%s: PROM IOTSB entries exceed kernel",
344 obpptsb = IOMMU_READ8(is, is_iommu, IMR_TSB) &
346 for (i = 0; i < obptsbentries; i++) {
347 tte = ldxa(obpptsb + i * 8, ASI_PHYS_USE_EC);
348 if ((tte & IOTTE_V) == 0)
350 slot = tsbentries - obptsbentries + i;
352 printf("%s: adding PROM IOTSB slot %d "
353 "(kernel slot %d) TTE: %#lx\n", name,
355 obpmap = (is->is_dvmabase + slot * IO_PAGE_SIZE) >>
357 if (rman_reserve_resource(&is->is_dvma_rman, obpmap,
358 obpmap, IO_PAGE_SIZE >> IO_PAGE_SHIFT, RF_ACTIVE,
360 panic("%s: could not reserve PROM IOTSB slot "
361 "%d (kernel slot %d)", __func__, i, slot);
362 is->is_tsb[slot] = tte;
367 * Initialize streaming buffer, if it is there.
369 if (IOMMU_HAS_SB(is)) {
371 * Find two 64-byte blocks in is_flush that are aligned on
372 * a 64-byte boundary for flushing.
374 offs = roundup2((vm_offset_t)is->is_flush,
375 STRBUF_FLUSHSYNC_NBYTES);
376 for (i = 0; i < 2; i++, offs += STRBUF_FLUSHSYNC_NBYTES) {
377 is->is_flushva[i] = (uint64_t *)offs;
378 is->is_flushpa[i] = pmap_kextract(offs);
383 * Now actually start up the IOMMU.
389 * Streaming buffers don't exist on the UltraSPARC IIi; we should have
390 * detected that already and disabled them. If not, we will notice that
391 * they aren't there when the STRBUF_EN bit does not remain.
394 iommu_reset(struct iommu_state *is)
400 if ((is->is_flags & IOMMU_FIRE) != 0) {
401 tsb |= is->is_tsbsize;
402 IOMMU_WRITE8(is, is_iommu, IMR_CACHE_INVAL, ~0ULL);
404 IOMMU_WRITE8(is, is_iommu, IMR_TSB, tsb);
405 IOMMU_WRITE8(is, is_iommu, IMR_CTL, is->is_cr);
407 for (i = 0; i < 2; i++) {
408 if (is->is_sb[i] != 0) {
409 IOMMU_WRITE8(is, is_sb[i], ISR_CTL, STRBUF_EN |
410 ((is->is_flags & IOMMU_RERUN_DISABLE) != 0 ?
413 /* No streaming buffers? Disable them. */
414 if ((IOMMU_READ8(is, is_sb[i], ISR_CTL) &
420 (void)IOMMU_READ8(is, is_iommu, IMR_CTL);
424 * Enter a mapping into the TSB. No locking required, since each TSB slot is
425 * uniquely assigned to a single map.
428 iommu_enter(struct iommu_state *is, vm_offset_t va, vm_paddr_t pa,
429 int stream, int flags)
433 KASSERT(va >= is->is_dvmabase,
434 ("%s: va %#lx not in DVMA space", __func__, va));
435 KASSERT(pa <= is->is_pmaxaddr,
436 ("%s: XXX: physical address too large (%#lx)", __func__, pa));
438 tte = MAKEIOTTE(pa, !(flags & BUS_DMA_NOWRITE),
439 !(flags & BUS_DMA_NOCACHE), stream);
441 IOMMU_SET_TTE(is, va, tte);
442 iommu_tlb_flush(is, va);
451 * Remove mappings created by iommu_enter(). Flush the streaming buffer,
452 * but do not synchronize it. Returns whether a streaming buffer flush
456 iommu_remove(struct iommu_state *is, vm_offset_t va, vm_size_t len)
458 int slot, streamed = 0;
464 KASSERT(va >= is->is_dvmabase,
465 ("%s: va 0x%lx not in DVMA space", __func__, (u_long)va));
466 KASSERT(va + len >= va,
467 ("%s: va 0x%lx + len 0x%lx wraps", __func__, (long)va, (long)len));
469 va = trunc_io_page(va);
471 if ((IOMMU_GET_TTE(is, va) & IOTTE_STREAM) != 0) {
473 iommu_strbuf_flush(is, va);
475 len -= ulmin(len, IO_PAGE_SIZE);
476 IOMMU_SET_TTE(is, va, 0);
477 iommu_tlb_flush(is, va);
478 if ((is->is_flags & IOMMU_FLUSH_CACHE) != 0) {
479 slot = IOTSBSLOT(va);
480 if (len <= IO_PAGE_SIZE || slot % 8 == 7)
481 IOMMU_WRITE8(is, is_iommu, IMR_CACHE_FLUSH,
482 is->is_ptsb + slot * 8);
489 /* Decode an IOMMU fault for host bridge error handlers. */
491 iommu_decode_fault(struct iommu_state *is, vm_offset_t phys)
496 idx = phys - is->is_ptsb;
497 if (phys < is->is_ptsb ||
498 idx > (PAGE_SIZE << is->is_tsbsize))
500 va = is->is_dvmabase +
501 (((bus_addr_t)idx >> IOTTE_SHIFT) << IO_PAGE_SHIFT);
502 printf("IOMMU fault virtual address %#lx\n", (u_long)va);
506 * A barrier operation which makes sure that all previous streaming buffer
507 * flushes complete before it returns.
510 iommu_strbuf_flush_sync(struct iommu_state *is)
512 struct timeval cur, end;
516 if (!IOMMU_HAS_SB(is))
520 * Streaming buffer flushes:
522 * 1 Tell strbuf to flush by storing va to strbuf_pgflush. If
523 * we're not on a cache line boundary (64-bits):
525 * 3 Store pointer to flag in flushsync
526 * 4 wait till flushsync becomes 0x1
528 * If it takes more than .5 sec, something went wrong.
530 *is->is_flushva[0] = 1;
531 *is->is_flushva[1] = 1;
533 for (i = 0; i < 2; i++) {
534 if (is->is_sb[i] != 0) {
535 *is->is_flushva[i] = 0;
536 IOMMU_WRITE8(is, is_sb[i], ISR_FLUSHSYNC,
544 * 0.5s is the recommended timeout from the U2S manual. The actual
545 * time required should be smaller by at least a factor of 1000.
546 * We have no choice but to busy-wait.
548 end.tv_usec = 500000;
549 timevaladd(&end, &cur);
551 while ((!*is->is_flushva[0] || !*is->is_flushva[1]) &&
552 timevalcmp(&cur, &end, <=))
555 if (!*is->is_flushva[0] || !*is->is_flushva[1]) {
556 panic("%s: flush timeout %ld, %ld at %#lx", __func__,
557 *is->is_flushva[0], *is->is_flushva[1], is->is_flushpa[0]);
563 /* Determine whether we may enable streaming on a mapping. */
565 iommu_use_streaming(struct iommu_state *is, bus_dmamap_t map, bus_size_t size)
568 return (size >= IOMMU_STREAM_THRESH && IOMMU_HAS_SB(is) &&
569 (map->dm_flags & DMF_COHERENT) == 0);
573 * Allocate DVMA virtual memory for a map. The map may not be on a queue,
574 * so that it can be freely modified.
577 iommu_dvma_valloc(bus_dma_tag_t t, struct iommu_state *is, bus_dmamap_t map,
580 struct resource *res;
581 struct bus_dmamap_res *bdr;
582 bus_size_t align, sgsize;
584 KASSERT(!map->dm_onq, ("%s: map on queue!", __func__));
585 if ((bdr = malloc(sizeof(*bdr), M_IOMMU, M_NOWAIT)) == NULL)
588 * If a boundary is specified, a map cannot be larger than it; however
589 * we do not clip currently, as that does not play well with the lazy
591 * Alignment to a page boundary is always enforced.
593 align = (t->dt_alignment + IO_PAGE_MASK) >> IO_PAGE_SHIFT;
594 sgsize = round_io_page(size) >> IO_PAGE_SHIFT;
595 if (t->dt_boundary > 0 && t->dt_boundary < IO_PAGE_SIZE)
596 panic("%s: illegal boundary specified", __func__);
597 res = rman_reserve_resource_bound(&is->is_dvma_rman, 0L,
598 t->dt_lowaddr >> IO_PAGE_SHIFT, sgsize,
599 t->dt_boundary >> IO_PAGE_SHIFT,
600 RF_ACTIVE | rman_make_alignment_flags(align), NULL);
608 SLIST_INSERT_HEAD(&map->dm_reslist, bdr, dr_link);
612 /* Unload the map and mark all resources as unused, but do not free them. */
614 iommu_dvmamap_vunload(struct iommu_state *is, bus_dmamap_t map)
616 struct bus_dmamap_res *r;
619 IS_LOCK_ASSERT(is); /* for iommu_strbuf_sync() below */
620 SLIST_FOREACH(r, &map->dm_reslist, dr_link) {
621 streamed |= iommu_remove(is, BDR_START(r), r->dr_used);
625 iommu_strbuf_sync(is);
628 /* Free a DVMA virtual memory resource. */
630 iommu_dvma_vfree_res(bus_dmamap_t map, struct bus_dmamap_res *r)
633 KASSERT(r->dr_used == 0, ("%s: resource busy!", __func__));
634 if (r->dr_res != NULL && rman_release_resource(r->dr_res) != 0)
635 printf("warning: DVMA space lost\n");
636 SLIST_REMOVE(&map->dm_reslist, r, bus_dmamap_res, dr_link);
640 /* Free all DVMA virtual memory for a map. */
642 iommu_dvma_vfree(struct iommu_state *is, bus_dmamap_t map)
646 iommu_map_remq(is, map);
647 iommu_dvmamap_vunload(is, map);
649 while (!SLIST_EMPTY(&map->dm_reslist))
650 iommu_dvma_vfree_res(map, SLIST_FIRST(&map->dm_reslist));
653 /* Prune a map, freeing all unused DVMA resources. */
655 iommu_dvma_vprune(struct iommu_state *is, bus_dmamap_t map)
657 struct bus_dmamap_res *r, *n;
658 bus_size_t freed = 0;
661 for (r = SLIST_FIRST(&map->dm_reslist); r != NULL; r = n) {
662 n = SLIST_NEXT(r, dr_link);
663 if (r->dr_used == 0) {
664 freed += BDR_SIZE(r);
665 iommu_dvma_vfree_res(map, r);
668 if (SLIST_EMPTY(&map->dm_reslist))
669 iommu_map_remq(is, map);
674 * Try to find a suitably-sized (and if requested, -aligned) slab of DVMA
675 * memory with IO page offset voffs.
678 iommu_dvma_vfindseg(bus_dmamap_t map, vm_offset_t voffs, bus_size_t size,
681 struct bus_dmamap_res *r;
682 bus_addr_t dvmaddr, dvmend;
684 KASSERT(!map->dm_onq, ("%s: map on queue!", __func__));
685 SLIST_FOREACH(r, &map->dm_reslist, dr_link) {
686 dvmaddr = round_io_page(BDR_START(r) + r->dr_used);
687 /* Alignment can only work with voffs == 0. */
688 dvmaddr = (dvmaddr + amask) & ~amask;
690 dvmend = dvmaddr + size;
691 if (dvmend <= BDR_END(r)) {
692 r->dr_used = dvmend - BDR_START(r);
700 * Try to find or allocate a slab of DVMA space; see above.
703 iommu_dvma_vallocseg(bus_dma_tag_t dt, struct iommu_state *is, bus_dmamap_t map,
704 vm_offset_t voffs, bus_size_t size, bus_addr_t amask, bus_addr_t *addr)
706 bus_dmamap_t tm, last;
707 bus_addr_t dvmaddr, freed;
708 int error, complete = 0;
710 dvmaddr = iommu_dvma_vfindseg(map, voffs, size, amask);
712 /* Need to allocate. */
714 while ((error = iommu_dvma_valloc(dt, is, map,
715 voffs + size)) == ENOMEM && !complete) {
717 * Free the allocated DVMA of a few maps until
718 * the required size is reached. This is an
719 * approximation to not have to call the allocation
720 * function too often; most likely one free run
721 * will not suffice if not one map was large enough
722 * itself due to fragmentation.
726 last = TAILQ_LAST(&is->is_maplruq, iommu_maplruq_head);
728 tm = TAILQ_FIRST(&is->is_maplruq);
729 complete = tm == last;
732 freed += iommu_dvma_vprune(is, tm);
733 /* Move to the end. */
734 iommu_map_insq(is, tm);
735 } while (freed < size && !complete);
740 dvmaddr = iommu_dvma_vfindseg(map, voffs, size, amask);
741 KASSERT(dvmaddr != 0, ("%s: allocation failed unexpectedly!",
749 iommu_dvmamem_alloc(bus_dma_tag_t dt, void **vaddr, int flags,
752 struct iommu_state *is = dt->dt_cookie;
756 * XXX: This will break for 32 bit transfers on machines with more
757 * than is->is_pmaxaddr memory.
759 if ((error = sparc64_dma_alloc_map(dt, mapp)) != 0)
762 if ((flags & BUS_DMA_NOWAIT) != 0)
766 if ((flags & BUS_DMA_ZERO) != 0)
769 if ((*vaddr = malloc(dt->dt_maxsize, M_IOMMU, mflags)) == NULL) {
771 sparc64_dma_free_map(dt, *mapp);
774 if ((flags & BUS_DMA_COHERENT) != 0)
775 (*mapp)->dm_flags |= DMF_COHERENT;
777 * Try to preallocate DVMA space. If this fails, it is retried at
780 iommu_dvma_valloc(dt, is, *mapp, IOMMU_SIZE_ROUNDUP(dt->dt_maxsize));
782 iommu_map_insq(is, *mapp);
788 iommu_dvmamem_free(bus_dma_tag_t dt, void *vaddr, bus_dmamap_t map)
790 struct iommu_state *is = dt->dt_cookie;
792 iommu_dvma_vfree(is, map);
793 sparc64_dma_free_map(dt, map);
794 free(vaddr, M_IOMMU);
798 iommu_dvmamap_create(bus_dma_tag_t dt, int flags, bus_dmamap_t *mapp)
800 struct iommu_state *is = dt->dt_cookie;
801 bus_size_t totsz, presz, currsz;
802 int error, i, maxpre;
804 if ((error = sparc64_dma_alloc_map(dt, mapp)) != 0)
806 if ((flags & BUS_DMA_COHERENT) != 0)
807 (*mapp)->dm_flags |= DMF_COHERENT;
809 * Preallocate DVMA space; if this fails now, it is retried at load
810 * time. Through bus_dmamap_load_mbuf() and bus_dmamap_load_uio(),
811 * it is possible to have multiple discontiguous segments in a single
812 * map, which is handled by allocating additional resources, instead
813 * of increasing the size, to avoid fragmentation.
814 * Clamp preallocation to IOMMU_MAX_PRE. In some situations we can
815 * handle more; that case is handled by reallocating at map load time.
817 totsz = ulmin(IOMMU_SIZE_ROUNDUP(dt->dt_maxsize), IOMMU_MAX_PRE);
818 error = iommu_dvma_valloc(dt, is, *mapp, totsz);
822 * Try to be smart about preallocating some additional segments if
825 maxpre = imin(dt->dt_nsegments, IOMMU_MAX_PRE_SEG);
826 presz = dt->dt_maxsize / maxpre;
827 for (i = 1; i < maxpre && totsz < IOMMU_MAX_PRE; i++) {
828 currsz = round_io_page(ulmin(presz, IOMMU_MAX_PRE - totsz));
829 error = iommu_dvma_valloc(dt, is, *mapp, currsz);
835 iommu_map_insq(is, *mapp);
841 iommu_dvmamap_destroy(bus_dma_tag_t dt, bus_dmamap_t map)
843 struct iommu_state *is = dt->dt_cookie;
845 iommu_dvma_vfree(is, map);
846 sparc64_dma_free_map(dt, map);
851 * Utility function to load a physical buffer. segp contains
852 * the starting segment on entrace, and the ending segment on exit.
855 iommu_dvmamap_load_phys(bus_dma_tag_t dt, bus_dmamap_t map, vm_paddr_t buf,
856 bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp)
858 bus_addr_t amask, dvmaddr, dvmoffs;
859 bus_size_t sgsize, esize;
860 struct iommu_state *is;
863 int error, firstpg, sgcnt;
868 if ((map->dm_flags & DMF_LOADED) != 0) {
870 printf("%s: map still in use\n", __func__);
872 bus_dmamap_unload(dt, map);
876 * Make sure that the map is not on a queue so that the
877 * resource list may be safely accessed and modified without
878 * needing the lock to cover the whole operation.
881 iommu_map_remq(is, map);
884 amask = dt->dt_alignment - 1;
887 KASSERT(buflen != 0, ("%s: buflen == 0!", __func__));
888 if (buflen > dt->dt_maxsize)
892 segs = dt->dt_segments;
894 voffs = buf & IO_PAGE_MASK;
896 /* Try to find a slab that is large enough. */
897 error = iommu_dvma_vallocseg(dt, is, map, voffs, buflen, amask,
904 map->dm_flags &= ~DMF_STREAMED;
905 map->dm_flags |= iommu_use_streaming(is, map, buflen) != 0 ?
907 for (; buflen > 0; ) {
911 * Compute the segment size, and adjust counts.
913 sgsize = IO_PAGE_SIZE - ((u_long)buf & IO_PAGE_MASK);
920 dvmoffs = trunc_io_page(dvmaddr);
921 iommu_enter(is, dvmoffs, trunc_io_page(curaddr),
922 (map->dm_flags & DMF_STREAMED) != 0, flags);
923 if ((is->is_flags & IOMMU_FLUSH_CACHE) != 0) {
924 slot = IOTSBSLOT(dvmoffs);
925 if (buflen <= 0 || slot % 8 == 7)
926 IOMMU_WRITE8(is, is_iommu, IMR_CACHE_FLUSH,
927 is->is_ptsb + slot * 8);
931 * Chop the chunk up into segments of at most maxsegsz, but try
932 * to fill each segment as well as possible.
935 esize = ulmin(sgsize,
936 dt->dt_maxsegsz - segs[sgcnt].ds_len);
937 segs[sgcnt].ds_len += esize;
943 if (sgcnt >= dt->dt_nsegments)
946 * No extra alignment here - the common practice in
947 * the busdma code seems to be that only the first
948 * segment needs to satisfy the alignment constraints
949 * (and that only for bus_dmamem_alloc()ed maps).
950 * It is assumed that such tags have maxsegsize >=
953 esize = ulmin(sgsize, dt->dt_maxsegsz);
954 segs[sgcnt].ds_addr = dvmaddr;
955 segs[sgcnt].ds_len = esize;
967 * IOMMU DVMA operations, common to PCI and SBus
970 iommu_dvmamap_load_buffer(bus_dma_tag_t dt, bus_dmamap_t map, void *buf,
971 bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs,
974 bus_addr_t amask, dvmaddr, dvmoffs;
975 bus_size_t sgsize, esize;
976 struct iommu_state *is;
977 vm_offset_t vaddr, voffs;
979 int error, firstpg, sgcnt;
984 if ((map->dm_flags & DMF_LOADED) != 0) {
986 printf("%s: map still in use\n", __func__);
988 bus_dmamap_unload(dt, map);
992 * Make sure that the map is not on a queue so that the
993 * resource list may be safely accessed and modified without
994 * needing the lock to cover the whole operation.
997 iommu_map_remq(is, map);
1000 amask = dt->dt_alignment - 1;
1003 KASSERT(buflen != 0, ("%s: buflen == 0!", __func__));
1004 if (buflen > dt->dt_maxsize)
1008 segs = dt->dt_segments;
1010 vaddr = (vm_offset_t)buf;
1011 voffs = vaddr & IO_PAGE_MASK;
1013 /* Try to find a slab that is large enough. */
1014 error = iommu_dvma_vallocseg(dt, is, map, voffs, buflen, amask,
1021 map->dm_flags &= ~DMF_STREAMED;
1022 map->dm_flags |= iommu_use_streaming(is, map, buflen) != 0 ?
1024 for (; buflen > 0; ) {
1026 * Get the physical address for this page.
1028 if (pmap == kernel_pmap)
1029 curaddr = pmap_kextract(vaddr);
1031 curaddr = pmap_extract(pmap, vaddr);
1034 * Compute the segment size, and adjust counts.
1036 sgsize = IO_PAGE_SIZE - ((u_long)vaddr & IO_PAGE_MASK);
1037 if (buflen < sgsize)
1043 dvmoffs = trunc_io_page(dvmaddr);
1044 iommu_enter(is, dvmoffs, trunc_io_page(curaddr),
1045 (map->dm_flags & DMF_STREAMED) != 0, flags);
1046 if ((is->is_flags & IOMMU_FLUSH_CACHE) != 0) {
1047 slot = IOTSBSLOT(dvmoffs);
1048 if (buflen <= 0 || slot % 8 == 7)
1049 IOMMU_WRITE8(is, is_iommu, IMR_CACHE_FLUSH,
1050 is->is_ptsb + slot * 8);
1054 * Chop the chunk up into segments of at most maxsegsz, but try
1055 * to fill each segment as well as possible.
1058 esize = ulmin(sgsize,
1059 dt->dt_maxsegsz - segs[sgcnt].ds_len);
1060 segs[sgcnt].ds_len += esize;
1064 while (sgsize > 0) {
1066 if (sgcnt >= dt->dt_nsegments)
1069 * No extra alignment here - the common practice in
1070 * the busdma code seems to be that only the first
1071 * segment needs to satisfy the alignment constraints
1072 * (and that only for bus_dmamem_alloc()ed maps).
1073 * It is assumed that such tags have maxsegsize >=
1076 esize = ulmin(sgsize, dt->dt_maxsegsz);
1077 segs[sgcnt].ds_addr = dvmaddr;
1078 segs[sgcnt].ds_len = esize;
1090 iommu_dvmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map,
1091 struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg)
1095 static bus_dma_segment_t *
1096 iommu_dvmamap_complete(bus_dma_tag_t dt, bus_dmamap_t map,
1097 bus_dma_segment_t *segs, int nsegs, int error)
1099 struct iommu_state *is = dt->dt_cookie;
1102 iommu_map_insq(is, map);
1104 iommu_dvmamap_vunload(is, map);
1108 map->dm_flags |= DMF_LOADED;
1111 segs = dt->dt_segments;
1116 iommu_dvmamap_unload(bus_dma_tag_t dt, bus_dmamap_t map)
1118 struct iommu_state *is = dt->dt_cookie;
1120 if ((map->dm_flags & DMF_LOADED) == 0)
1123 iommu_dvmamap_vunload(is, map);
1124 iommu_map_insq(is, map);
1126 map->dm_flags &= ~DMF_LOADED;
1130 iommu_dvmamap_sync(bus_dma_tag_t dt, bus_dmamap_t map, bus_dmasync_op_t op)
1132 struct iommu_state *is = dt->dt_cookie;
1133 struct bus_dmamap_res *r;
1138 if ((map->dm_flags & DMF_LOADED) == 0)
1140 if ((map->dm_flags & DMF_STREAMED) != 0 &&
1141 ((op & BUS_DMASYNC_POSTREAD) != 0 ||
1142 (op & BUS_DMASYNC_PREWRITE) != 0)) {
1144 SLIST_FOREACH(r, &map->dm_reslist, dr_link) {
1145 va = (vm_offset_t)BDR_START(r);
1148 * If we have a streaming buffer, flush it here
1152 if ((IOMMU_GET_TTE(is, va) &
1153 IOTTE_STREAM) != 0) {
1155 iommu_strbuf_flush(is, va);
1157 len -= ulmin(len, IO_PAGE_SIZE);
1162 iommu_strbuf_sync(is);
1165 if ((op & BUS_DMASYNC_PREWRITE) != 0)
1172 * Perform an IOMMU diagnostic access and print the tag belonging to va.
1175 iommu_diag(struct iommu_state *is, vm_offset_t va)
1180 if ((is->is_flags & IOMMU_FIRE) != 0)
1183 IOMMU_WRITE8(is, is_dva, 0, trunc_io_page(va));
1184 membar(StoreStore | StoreLoad);
1185 printf("%s: tte entry %#lx", __func__, IOMMU_GET_TTE(is, va));
1186 if (is->is_dtcmp != 0) {
1187 printf(", tag compare register is %#lx\n",
1188 IOMMU_READ8(is, is_dtcmp, 0));
1191 for (i = 0; i < 16; i++) {
1192 tag = IOMMU_READ8(is, is_dtag, i * 8);
1193 data = IOMMU_READ8(is, is_ddram, i * 8);
1194 printf("%s: tag %d: %#lx, vpn %#lx, err %lx; "
1195 "data %#lx, pa %#lx, v %d, c %d\n", __func__, i,
1196 tag, (tag & IOMMU_DTAG_VPNMASK) << IOMMU_DTAG_VPNSHIFT,
1197 (tag & IOMMU_DTAG_ERRMASK) >> IOMMU_DTAG_ERRSHIFT, data,
1198 (data & IOMMU_DDATA_PGMASK) << IOMMU_DDATA_PGSHIFT,
1199 (data & IOMMU_DDATA_V) != 0, (data & IOMMU_DDATA_C) != 0);
1203 #endif /* IOMMU_DIAG */
1205 struct bus_dma_methods iommu_dma_methods = {
1206 iommu_dvmamap_create,
1207 iommu_dvmamap_destroy,
1208 iommu_dvmamap_load_phys,
1209 iommu_dvmamap_load_buffer,
1210 iommu_dvmamap_waitok,
1211 iommu_dvmamap_complete,
1212 iommu_dvmamap_unload,
1214 iommu_dvmamem_alloc,