2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 2013 The FreeBSD Foundation
7 * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
8 * under sponsorship from the FreeBSD Foundation.
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 #include <sys/cdefs.h>
33 __FBSDID("$FreeBSD$");
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/domainset.h>
38 #include <sys/malloc.h>
41 #include <sys/interrupt.h>
42 #include <sys/kernel.h>
46 #include <sys/memdesc.h>
47 #include <sys/mutex.h>
48 #include <sys/sysctl.h>
50 #include <sys/taskqueue.h>
54 #include <dev/pci/pcireg.h>
55 #include <dev/pci/pcivar.h>
57 #include <vm/vm_extern.h>
58 #include <vm/vm_kern.h>
59 #include <vm/vm_object.h>
60 #include <vm/vm_page.h>
61 #include <vm/vm_map.h>
62 #include <dev/iommu/iommu.h>
63 #include <machine/atomic.h>
64 #include <machine/bus.h>
65 #include <machine/md_var.h>
66 #include <machine/iommu.h>
67 #include <dev/iommu/busdma_iommu.h>
70 * busdma_iommu.c, the implementation of the busdma(9) interface using
71 * IOMMU units from Intel VT-d.
75 iommu_bus_dma_is_dev_disabled(int domain, int bus, int slot, int func)
80 static const char bounce_str[] = "bounce";
81 static const char iommu_str[] = "iommu";
82 static const char dmar_str[] = "dmar"; /* compatibility */
85 env = kern_getenv("hw.busdma.default");
87 if (strcmp(env, bounce_str) == 0)
89 else if (strcmp(env, iommu_str) == 0 ||
90 strcmp(env, dmar_str) == 0)
95 snprintf(str, sizeof(str), "hw.busdma.pci%d.%d.%d.%d",
96 domain, bus, slot, func);
97 env = kern_getenv(str);
99 return (default_bounce != 0);
100 if (strcmp(env, bounce_str) == 0)
102 else if (strcmp(env, iommu_str) == 0 ||
103 strcmp(env, dmar_str) == 0)
106 ret = default_bounce != 0;
112 * Given original device, find the requester ID that will be seen by
113 * the IOMMU unit and used for page table lookup. PCI bridges may take
114 * ownership of transactions from downstream devices, so it may not be
115 * the same as the BSF of the target device. In those cases, all
116 * devices downstream of the bridge must share a single mapping
117 * domain, and must collectively be assigned to use either IOMMU or
121 iommu_get_requester(device_t dev, uint16_t *rid)
123 devclass_t pci_class;
124 device_t l, pci, pcib, pcip, pcibp, requester;
129 pci_class = devclass_find("pci");
132 *rid = pci_get_rid(dev);
135 * Walk the bridge hierarchy from the target device to the
136 * host port to find the translating bridge nearest the IOMMU
140 pci = device_get_parent(l);
141 KASSERT(pci != NULL, ("iommu_get_requester(%s): NULL parent "
142 "for %s", device_get_name(dev), device_get_name(l)));
143 KASSERT(device_get_devclass(pci) == pci_class,
144 ("iommu_get_requester(%s): non-pci parent %s for %s",
145 device_get_name(dev), device_get_name(pci),
146 device_get_name(l)));
148 pcib = device_get_parent(pci);
149 KASSERT(pcib != NULL, ("iommu_get_requester(%s): NULL bridge "
150 "for %s", device_get_name(dev), device_get_name(pci)));
153 * The parent of our "bridge" isn't another PCI bus,
154 * so pcib isn't a PCI->PCI bridge but rather a host
155 * port, and the requester ID won't be translated
158 pcip = device_get_parent(pcib);
159 if (device_get_devclass(pcip) != pci_class)
161 pcibp = device_get_parent(pcip);
163 if (pci_find_cap(l, PCIY_EXPRESS, &cap_offset) == 0) {
165 * Do not stop the loop even if the target
166 * device is PCIe, because it is possible (but
167 * unlikely) to have a PCI->PCIe bridge
168 * somewhere in the hierarchy.
173 * Device is not PCIe, it cannot be seen as a
174 * requester by IOMMU unit. Check whether the
177 bridge_is_pcie = pci_find_cap(pcib, PCIY_EXPRESS,
182 * Check for a buggy PCIe/PCI bridge that
183 * doesn't report the express capability. If
184 * the bridge above it is express but isn't a
185 * PCI bridge, then we know pcib is actually a
188 if (!bridge_is_pcie && pci_find_cap(pcibp,
189 PCIY_EXPRESS, &cap_offset) == 0) {
190 pcie_flags = pci_read_config(pcibp,
191 cap_offset + PCIER_FLAGS, 2);
192 if ((pcie_flags & PCIEM_FLAGS_TYPE) !=
193 PCIEM_TYPE_PCI_BRIDGE)
194 bridge_is_pcie = true;
197 if (bridge_is_pcie) {
199 * The current device is not PCIe, but
200 * the bridge above it is. This is a
201 * PCIe->PCI bridge. Assume that the
202 * requester ID will be the secondary
203 * bus number with slot and function
206 * XXX: Doesn't handle the case where
207 * the bridge is PCIe->PCI-X, and the
208 * bridge will only take ownership of
209 * requests in some cases. We should
210 * provide context entries with the
211 * same page tables for taken and
212 * non-taken transactions.
214 *rid = PCI_RID(pci_get_bus(l), 0, 0);
218 * Neither the device nor the bridge
219 * above it are PCIe. This is a
220 * conventional PCI->PCI bridge, which
221 * will use the bridge's BSF as the
224 *rid = pci_get_rid(pcib);
233 iommu_instantiate_ctx(struct iommu_unit *unit, device_t dev, bool rmrr)
236 struct iommu_ctx *ctx;
240 requester = iommu_get_requester(dev, &rid);
243 * If the user requested the IOMMU disabled for the device, we
244 * cannot disable the IOMMU unit, due to possibility of other
245 * devices on the same IOMMU unit still requiring translation.
246 * Instead provide the identity mapping for the device
249 disabled = iommu_bus_dma_is_dev_disabled(pci_get_domain(requester),
250 pci_get_bus(requester), pci_get_slot(requester),
251 pci_get_function(requester));
252 ctx = iommu_get_ctx(unit, requester, rid, disabled, rmrr);
257 * Keep the first reference on context, release the
261 if ((ctx->flags & IOMMU_CTX_DISABLED) == 0) {
262 ctx->flags |= IOMMU_CTX_DISABLED;
265 iommu_free_ctx_locked(unit, ctx);
273 iommu_get_dev_ctx(device_t dev)
275 struct iommu_unit *unit;
277 unit = iommu_find(dev, bootverbose);
278 /* Not in scope of any IOMMU ? */
281 if (!unit->dma_enabled)
284 #if defined(__amd64__) || defined(__i386__)
285 dmar_quirks_pre_use(unit);
286 dmar_instantiate_rmrr_ctxs(unit);
289 return (iommu_instantiate_ctx(unit, dev, false));
293 iommu_get_dma_tag(device_t dev, device_t child)
295 struct iommu_ctx *ctx;
298 ctx = iommu_get_dev_ctx(child);
302 res = (bus_dma_tag_t)ctx->tag;
307 bus_dma_iommu_set_buswide(device_t dev)
309 struct iommu_unit *unit;
311 u_int busno, slot, func;
313 parent = device_get_parent(dev);
314 if (device_get_devclass(parent) != devclass_find("pci"))
316 unit = iommu_find(dev, bootverbose);
319 busno = pci_get_bus(dev);
320 slot = pci_get_slot(dev);
321 func = pci_get_function(dev);
322 if (slot != 0 || func != 0) {
325 "iommu%d pci%d:%d:%d requested buswide busdma\n",
326 unit->unit, busno, slot, func);
330 iommu_set_buswide_ctx(unit, busno);
335 iommu_set_buswide_ctx(struct iommu_unit *unit, u_int busno)
338 MPASS(busno <= PCI_BUSMAX);
340 unit->buswide_ctxs[busno / NBBY / sizeof(uint32_t)] |=
341 1 << (busno % (NBBY * sizeof(uint32_t)));
346 iommu_is_buswide_ctx(struct iommu_unit *unit, u_int busno)
349 MPASS(busno <= PCI_BUSMAX);
350 return ((unit->buswide_ctxs[busno / NBBY / sizeof(uint32_t)] &
351 (1U << (busno % (NBBY * sizeof(uint32_t))))) != 0);
354 static MALLOC_DEFINE(M_IOMMU_DMAMAP, "iommu_dmamap", "IOMMU DMA Map");
356 static void iommu_bus_schedule_dmamap(struct iommu_unit *unit,
357 struct bus_dmamap_iommu *map);
360 iommu_bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
361 bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr,
362 bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize,
363 int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
364 void *lockfuncarg, bus_dma_tag_t *dmat)
366 struct bus_dma_tag_iommu *newtag, *oldtag;
370 error = common_bus_dma_tag_create(parent != NULL ?
371 &((struct bus_dma_tag_iommu *)parent)->common : NULL, alignment,
372 boundary, lowaddr, highaddr, filter, filterarg, maxsize,
373 nsegments, maxsegsz, flags, lockfunc, lockfuncarg,
374 sizeof(struct bus_dma_tag_iommu), (void **)&newtag);
378 oldtag = (struct bus_dma_tag_iommu *)parent;
379 newtag->common.impl = &bus_dma_iommu_impl;
380 newtag->ctx = oldtag->ctx;
381 newtag->owner = oldtag->owner;
383 *dmat = (bus_dma_tag_t)newtag;
385 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
386 __func__, newtag, (newtag != NULL ? newtag->common.flags : 0),
392 iommu_bus_dma_tag_set_domain(bus_dma_tag_t dmat)
399 iommu_bus_dma_tag_destroy(bus_dma_tag_t dmat1)
401 struct bus_dma_tag_iommu *dmat, *dmat_copy, *parent;
405 dmat_copy = dmat = (struct bus_dma_tag_iommu *)dmat1;
408 if (dmat->map_count != 0) {
412 while (dmat != NULL) {
413 parent = (struct bus_dma_tag_iommu *)dmat->common.parent;
414 if (atomic_fetchadd_int(&dmat->common.ref_count, -1) ==
416 if (dmat == dmat->ctx->tag)
417 iommu_free_ctx(dmat->ctx);
418 free(dmat->segments, M_IOMMU_DMAMAP);
419 free(dmat, M_DEVBUF);
426 CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error);
431 iommu_bus_dma_id_mapped(bus_dma_tag_t dmat, vm_paddr_t buf, bus_size_t buflen)
438 iommu_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
440 struct bus_dma_tag_iommu *tag;
441 struct bus_dmamap_iommu *map;
443 tag = (struct bus_dma_tag_iommu *)dmat;
444 map = malloc_domainset(sizeof(*map), M_IOMMU_DMAMAP,
445 DOMAINSET_PREF(tag->common.domain), M_NOWAIT | M_ZERO);
450 if (tag->segments == NULL) {
451 tag->segments = malloc_domainset(sizeof(bus_dma_segment_t) *
452 tag->common.nsegments, M_IOMMU_DMAMAP,
453 DOMAINSET_PREF(tag->common.domain), M_NOWAIT);
454 if (tag->segments == NULL) {
455 free(map, M_IOMMU_DMAMAP);
460 TAILQ_INIT(&map->map_entries);
463 map->cansleep = false;
465 *mapp = (bus_dmamap_t)map;
471 iommu_bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map1)
473 struct bus_dma_tag_iommu *tag;
474 struct bus_dmamap_iommu *map;
475 struct iommu_domain *domain;
477 tag = (struct bus_dma_tag_iommu *)dmat;
478 map = (struct bus_dmamap_iommu *)map1;
480 domain = tag->ctx->domain;
481 IOMMU_DOMAIN_LOCK(domain);
482 if (!TAILQ_EMPTY(&map->map_entries)) {
483 IOMMU_DOMAIN_UNLOCK(domain);
486 IOMMU_DOMAIN_UNLOCK(domain);
487 free(map, M_IOMMU_DMAMAP);
495 iommu_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
498 struct bus_dma_tag_iommu *tag;
499 struct bus_dmamap_iommu *map;
503 error = iommu_bus_dmamap_create(dmat, flags, mapp);
507 mflags = (flags & BUS_DMA_NOWAIT) != 0 ? M_NOWAIT : M_WAITOK;
508 mflags |= (flags & BUS_DMA_ZERO) != 0 ? M_ZERO : 0;
509 attr = (flags & BUS_DMA_NOCACHE) != 0 ? VM_MEMATTR_UNCACHEABLE :
512 tag = (struct bus_dma_tag_iommu *)dmat;
513 map = (struct bus_dmamap_iommu *)*mapp;
515 if (tag->common.maxsize < PAGE_SIZE &&
516 tag->common.alignment <= tag->common.maxsize &&
517 attr == VM_MEMATTR_DEFAULT) {
518 *vaddr = malloc_domainset(tag->common.maxsize, M_DEVBUF,
519 DOMAINSET_PREF(tag->common.domain), mflags);
520 map->flags |= BUS_DMAMAP_IOMMU_MALLOC;
522 *vaddr = (void *)kmem_alloc_attr_domainset(
523 DOMAINSET_PREF(tag->common.domain), tag->common.maxsize,
524 mflags, 0ul, BUS_SPACE_MAXADDR, attr);
525 map->flags |= BUS_DMAMAP_IOMMU_KMEM_ALLOC;
527 if (*vaddr == NULL) {
528 iommu_bus_dmamap_destroy(dmat, *mapp);
536 iommu_bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map1)
538 struct bus_dma_tag_iommu *tag;
539 struct bus_dmamap_iommu *map;
541 tag = (struct bus_dma_tag_iommu *)dmat;
542 map = (struct bus_dmamap_iommu *)map1;
544 if ((map->flags & BUS_DMAMAP_IOMMU_MALLOC) != 0) {
545 free(vaddr, M_DEVBUF);
546 map->flags &= ~BUS_DMAMAP_IOMMU_MALLOC;
548 KASSERT((map->flags & BUS_DMAMAP_IOMMU_KMEM_ALLOC) != 0,
549 ("iommu_bus_dmamem_free for non alloced map %p", map));
550 kmem_free((vm_offset_t)vaddr, tag->common.maxsize);
551 map->flags &= ~BUS_DMAMAP_IOMMU_KMEM_ALLOC;
554 iommu_bus_dmamap_destroy(dmat, map1);
558 iommu_bus_dmamap_load_something1(struct bus_dma_tag_iommu *tag,
559 struct bus_dmamap_iommu *map, vm_page_t *ma, int offset, bus_size_t buflen,
560 int flags, bus_dma_segment_t *segs, int *segp,
561 struct iommu_map_entries_tailq *unroll_list)
563 struct iommu_ctx *ctx;
564 struct iommu_domain *domain;
565 struct iommu_map_entry *entry;
568 int error, idx, gas_flags, seg;
570 KASSERT(offset < IOMMU_PAGE_SIZE, ("offset %d", offset));
572 segs = tag->segments;
574 domain = ctx->domain;
580 if (seg >= tag->common.nsegments) {
584 buflen1 = buflen > tag->common.maxsegsz ?
585 tag->common.maxsegsz : buflen;
586 size = round_page(offset + buflen1);
589 * (Too) optimistically allow split if there are more
590 * then one segments left.
592 gas_flags = map->cansleep ? IOMMU_MF_CANWAIT : 0;
593 if (seg + 1 < tag->common.nsegments)
594 gas_flags |= IOMMU_MF_CANSPLIT;
596 error = iommu_map(domain, &tag->common, size, offset,
597 IOMMU_MAP_ENTRY_READ |
598 ((flags & BUS_DMA_NOWRITE) == 0 ? IOMMU_MAP_ENTRY_WRITE : 0),
599 gas_flags, ma + idx, &entry);
602 if ((gas_flags & IOMMU_MF_CANSPLIT) != 0) {
603 KASSERT(size >= entry->end - entry->start,
604 ("split increased entry size %jx %jx %jx",
605 (uintmax_t)size, (uintmax_t)entry->start,
606 (uintmax_t)entry->end));
607 size = entry->end - entry->start;
611 KASSERT(entry->end - entry->start == size,
612 ("no split allowed %jx %jx %jx",
613 (uintmax_t)size, (uintmax_t)entry->start,
614 (uintmax_t)entry->end));
616 if (offset + buflen1 > size)
617 buflen1 = size - offset;
618 if (buflen1 > tag->common.maxsegsz)
619 buflen1 = tag->common.maxsegsz;
621 KASSERT(((entry->start + offset) & (tag->common.alignment - 1))
623 ("alignment failed: ctx %p start 0x%jx offset %x "
624 "align 0x%jx", ctx, (uintmax_t)entry->start, offset,
625 (uintmax_t)tag->common.alignment));
626 KASSERT(entry->end <= tag->common.lowaddr ||
627 entry->start >= tag->common.highaddr,
628 ("entry placement failed: ctx %p start 0x%jx end 0x%jx "
629 "lowaddr 0x%jx highaddr 0x%jx", ctx,
630 (uintmax_t)entry->start, (uintmax_t)entry->end,
631 (uintmax_t)tag->common.lowaddr,
632 (uintmax_t)tag->common.highaddr));
633 KASSERT(iommu_test_boundary(entry->start + offset, buflen1,
634 tag->common.boundary),
635 ("boundary failed: ctx %p start 0x%jx end 0x%jx "
636 "boundary 0x%jx", ctx, (uintmax_t)entry->start,
637 (uintmax_t)entry->end, (uintmax_t)tag->common.boundary));
638 KASSERT(buflen1 <= tag->common.maxsegsz,
639 ("segment too large: ctx %p start 0x%jx end 0x%jx "
640 "buflen1 0x%jx maxsegsz 0x%jx", ctx,
641 (uintmax_t)entry->start, (uintmax_t)entry->end,
642 (uintmax_t)buflen1, (uintmax_t)tag->common.maxsegsz));
644 IOMMU_DOMAIN_LOCK(domain);
645 TAILQ_INSERT_TAIL(&map->map_entries, entry, dmamap_link);
646 entry->flags |= IOMMU_MAP_ENTRY_MAP;
647 IOMMU_DOMAIN_UNLOCK(domain);
648 TAILQ_INSERT_TAIL(unroll_list, entry, unroll_link);
650 segs[seg].ds_addr = entry->start + offset;
651 segs[seg].ds_len = buflen1;
653 idx += OFF_TO_IDX(trunc_page(offset + buflen1));
655 offset &= IOMMU_PAGE_MASK;
664 iommu_bus_dmamap_load_something(struct bus_dma_tag_iommu *tag,
665 struct bus_dmamap_iommu *map, vm_page_t *ma, int offset, bus_size_t buflen,
666 int flags, bus_dma_segment_t *segs, int *segp)
668 struct iommu_ctx *ctx;
669 struct iommu_domain *domain;
670 struct iommu_map_entry *entry, *entry1;
671 struct iommu_map_entries_tailq unroll_list;
675 domain = ctx->domain;
676 atomic_add_long(&ctx->loads, 1);
678 TAILQ_INIT(&unroll_list);
679 error = iommu_bus_dmamap_load_something1(tag, map, ma, offset,
680 buflen, flags, segs, segp, &unroll_list);
683 * The busdma interface does not allow us to report
684 * partial buffer load, so unfortunately we have to
685 * revert all work done.
687 IOMMU_DOMAIN_LOCK(domain);
688 TAILQ_FOREACH_SAFE(entry, &unroll_list, unroll_link,
691 * No entries other than what we have created
692 * during the failed run might have been
693 * inserted there in between, since we own ctx
696 TAILQ_REMOVE(&map->map_entries, entry, dmamap_link);
697 TAILQ_REMOVE(&unroll_list, entry, unroll_link);
698 TAILQ_INSERT_TAIL(&domain->unload_entries, entry,
701 IOMMU_DOMAIN_UNLOCK(domain);
702 taskqueue_enqueue(domain->iommu->delayed_taskqueue,
703 &domain->unload_task);
706 if (error == ENOMEM && (flags & BUS_DMA_NOWAIT) == 0 &&
709 if (error == EINPROGRESS)
710 iommu_bus_schedule_dmamap(domain->iommu, map);
715 iommu_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map1,
716 struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags,
717 bus_dma_segment_t *segs, int *segp)
719 struct bus_dma_tag_iommu *tag;
720 struct bus_dmamap_iommu *map;
722 tag = (struct bus_dma_tag_iommu *)dmat;
723 map = (struct bus_dmamap_iommu *)map1;
724 return (iommu_bus_dmamap_load_something(tag, map, ma, ma_offs, tlen,
729 iommu_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map1,
730 vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs,
733 struct bus_dma_tag_iommu *tag;
734 struct bus_dmamap_iommu *map;
736 vm_paddr_t pstart, pend, paddr;
737 int error, i, ma_cnt, mflags, offset;
739 tag = (struct bus_dma_tag_iommu *)dmat;
740 map = (struct bus_dmamap_iommu *)map1;
741 pstart = trunc_page(buf);
742 pend = round_page(buf + buflen);
743 offset = buf & PAGE_MASK;
744 ma_cnt = OFF_TO_IDX(pend - pstart);
745 mflags = map->cansleep ? M_WAITOK : M_NOWAIT;
746 ma = malloc(sizeof(vm_page_t) * ma_cnt, M_DEVBUF, mflags);
750 for (i = 0; i < ma_cnt; i++) {
751 paddr = pstart + ptoa(i);
752 ma[i] = PHYS_TO_VM_PAGE(paddr);
753 if (ma[i] == NULL || VM_PAGE_TO_PHYS(ma[i]) != paddr) {
755 * If PHYS_TO_VM_PAGE() returned NULL or the
756 * vm_page was not initialized we'll use a
760 fma = malloc(sizeof(struct vm_page) * ma_cnt,
761 M_DEVBUF, M_ZERO | mflags);
767 vm_page_initfake(&fma[i], pstart + ptoa(i),
772 error = iommu_bus_dmamap_load_something(tag, map, ma, offset, buflen,
780 iommu_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map1, void *buf,
781 bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs,
784 struct bus_dma_tag_iommu *tag;
785 struct bus_dmamap_iommu *map;
787 vm_paddr_t pstart, pend, paddr;
788 int error, i, ma_cnt, mflags, offset;
790 tag = (struct bus_dma_tag_iommu *)dmat;
791 map = (struct bus_dmamap_iommu *)map1;
792 pstart = trunc_page((vm_offset_t)buf);
793 pend = round_page((vm_offset_t)buf + buflen);
794 offset = (vm_offset_t)buf & PAGE_MASK;
795 ma_cnt = OFF_TO_IDX(pend - pstart);
796 mflags = map->cansleep ? M_WAITOK : M_NOWAIT;
797 ma = malloc(sizeof(vm_page_t) * ma_cnt, M_DEVBUF, mflags);
801 for (i = 0; i < ma_cnt; i++, pstart += PAGE_SIZE) {
802 if (pmap == kernel_pmap)
803 paddr = pmap_kextract(pstart);
805 paddr = pmap_extract(pmap, pstart);
806 ma[i] = PHYS_TO_VM_PAGE(paddr);
807 if (ma[i] == NULL || VM_PAGE_TO_PHYS(ma[i]) != paddr) {
809 * If PHYS_TO_VM_PAGE() returned NULL or the
810 * vm_page was not initialized we'll use a
814 fma = malloc(sizeof(struct vm_page) * ma_cnt,
815 M_DEVBUF, M_ZERO | mflags);
821 vm_page_initfake(&fma[i], paddr, VM_MEMATTR_DEFAULT);
825 error = iommu_bus_dmamap_load_something(tag, map, ma, offset, buflen,
833 iommu_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map1,
834 struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg)
836 struct bus_dmamap_iommu *map;
840 map = (struct bus_dmamap_iommu *)map1;
842 map->tag = (struct bus_dma_tag_iommu *)dmat;
843 map->callback = callback;
844 map->callback_arg = callback_arg;
847 static bus_dma_segment_t *
848 iommu_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map1,
849 bus_dma_segment_t *segs, int nsegs, int error)
851 struct bus_dma_tag_iommu *tag;
852 struct bus_dmamap_iommu *map;
854 tag = (struct bus_dma_tag_iommu *)dmat;
855 map = (struct bus_dmamap_iommu *)map1;
858 KASSERT(map->cansleep,
859 ("map not locked and not sleepable context %p", map));
862 * We are called from the delayed context. Relock the
865 (tag->common.lockfunc)(tag->common.lockfuncarg, BUS_DMA_LOCK);
870 segs = tag->segments;
875 * The limitations of busdma KPI forces the iommu to perform the actual
876 * unload, consisting of the unmapping of the map entries page tables,
877 * from the delayed context on i386, since page table page mapping
878 * might require a sleep to be successfull. The unfortunate
879 * consequence is that the DMA requests can be served some time after
880 * the bus_dmamap_unload() call returned.
882 * On amd64, we assume that sf allocation cannot fail.
885 iommu_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map1)
887 struct bus_dma_tag_iommu *tag;
888 struct bus_dmamap_iommu *map;
889 struct iommu_ctx *ctx;
890 struct iommu_domain *domain;
891 #ifndef IOMMU_DOMAIN_UNLOAD_SLEEP
892 struct iommu_map_entries_tailq entries;
895 tag = (struct bus_dma_tag_iommu *)dmat;
896 map = (struct bus_dmamap_iommu *)map1;
898 domain = ctx->domain;
899 atomic_add_long(&ctx->unloads, 1);
901 #if defined(IOMMU_DOMAIN_UNLOAD_SLEEP)
902 IOMMU_DOMAIN_LOCK(domain);
903 TAILQ_CONCAT(&domain->unload_entries, &map->map_entries, dmamap_link);
904 IOMMU_DOMAIN_UNLOCK(domain);
905 taskqueue_enqueue(domain->iommu->delayed_taskqueue,
906 &domain->unload_task);
908 TAILQ_INIT(&entries);
909 IOMMU_DOMAIN_LOCK(domain);
910 TAILQ_CONCAT(&entries, &map->map_entries, dmamap_link);
911 IOMMU_DOMAIN_UNLOCK(domain);
912 THREAD_NO_SLEEPING();
913 iommu_domain_unload(domain, &entries, false);
914 THREAD_SLEEPING_OK();
915 KASSERT(TAILQ_EMPTY(&entries), ("lazy iommu_ctx_unload %p", ctx));
920 iommu_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map,
925 struct bus_dma_impl bus_dma_iommu_impl = {
926 .tag_create = iommu_bus_dma_tag_create,
927 .tag_destroy = iommu_bus_dma_tag_destroy,
928 .tag_set_domain = iommu_bus_dma_tag_set_domain,
929 .id_mapped = iommu_bus_dma_id_mapped,
930 .map_create = iommu_bus_dmamap_create,
931 .map_destroy = iommu_bus_dmamap_destroy,
932 .mem_alloc = iommu_bus_dmamem_alloc,
933 .mem_free = iommu_bus_dmamem_free,
934 .load_phys = iommu_bus_dmamap_load_phys,
935 .load_buffer = iommu_bus_dmamap_load_buffer,
936 .load_ma = iommu_bus_dmamap_load_ma,
937 .map_waitok = iommu_bus_dmamap_waitok,
938 .map_complete = iommu_bus_dmamap_complete,
939 .map_unload = iommu_bus_dmamap_unload,
940 .map_sync = iommu_bus_dmamap_sync,
944 iommu_bus_task_dmamap(void *arg, int pending)
946 struct bus_dma_tag_iommu *tag;
947 struct bus_dmamap_iommu *map;
948 struct iommu_unit *unit;
952 while ((map = TAILQ_FIRST(&unit->delayed_maps)) != NULL) {
953 TAILQ_REMOVE(&unit->delayed_maps, map, delay_link);
956 map->cansleep = true;
958 bus_dmamap_load_mem((bus_dma_tag_t)tag, (bus_dmamap_t)map,
959 &map->mem, map->callback, map->callback_arg,
961 map->cansleep = false;
963 (tag->common.lockfunc)(tag->common.lockfuncarg,
967 map->cansleep = false;
974 iommu_bus_schedule_dmamap(struct iommu_unit *unit, struct bus_dmamap_iommu *map)
979 TAILQ_INSERT_TAIL(&unit->delayed_maps, map, delay_link);
981 taskqueue_enqueue(unit->delayed_taskqueue, &unit->dmamap_load_task);
985 iommu_init_busdma(struct iommu_unit *unit)
989 unit->dma_enabled = 1;
990 error = TUNABLE_INT_FETCH("hw.iommu.dma", &unit->dma_enabled);
991 if (error == 0) /* compatibility */
992 TUNABLE_INT_FETCH("hw.dmar.dma", &unit->dma_enabled);
993 TAILQ_INIT(&unit->delayed_maps);
994 TASK_INIT(&unit->dmamap_load_task, 0, iommu_bus_task_dmamap, unit);
995 unit->delayed_taskqueue = taskqueue_create("iommu", M_WAITOK,
996 taskqueue_thread_enqueue, &unit->delayed_taskqueue);
997 taskqueue_start_threads(&unit->delayed_taskqueue, 1, PI_DISK,
998 "iommu%d busdma taskq", unit->unit);
1003 iommu_fini_busdma(struct iommu_unit *unit)
1006 if (unit->delayed_taskqueue == NULL)
1009 taskqueue_drain(unit->delayed_taskqueue, &unit->dmamap_load_task);
1010 taskqueue_free(unit->delayed_taskqueue);
1011 unit->delayed_taskqueue = NULL;
1015 bus_dma_iommu_load_ident(bus_dma_tag_t dmat, bus_dmamap_t map1,
1016 vm_paddr_t start, vm_size_t length, int flags)
1018 struct bus_dma_tag_common *tc;
1019 struct bus_dma_tag_iommu *tag;
1020 struct bus_dmamap_iommu *map;
1021 struct iommu_ctx *ctx;
1022 struct iommu_domain *domain;
1023 struct iommu_map_entry *entry;
1029 MPASS((start & PAGE_MASK) == 0);
1030 MPASS((length & PAGE_MASK) == 0);
1032 MPASS(start + length >= start);
1033 MPASS((flags & ~(BUS_DMA_NOWAIT | BUS_DMA_NOWRITE)) == 0);
1035 tc = (struct bus_dma_tag_common *)dmat;
1036 if (tc->impl != &bus_dma_iommu_impl)
1039 tag = (struct bus_dma_tag_iommu *)dmat;
1041 domain = ctx->domain;
1042 map = (struct bus_dmamap_iommu *)map1;
1043 waitok = (flags & BUS_DMA_NOWAIT) != 0;
1045 entry = iommu_map_alloc_entry(domain, waitok ? 0 : IOMMU_PGF_WAITOK);
1048 entry->start = start;
1049 entry->end = start + length;
1050 ma = malloc(sizeof(vm_page_t) * atop(length), M_TEMP, waitok ?
1051 M_WAITOK : M_NOWAIT);
1053 iommu_map_free_entry(domain, entry);
1056 for (i = 0; i < atop(length); i++) {
1057 ma[i] = vm_page_getfake(entry->start + PAGE_SIZE * i,
1058 VM_MEMATTR_DEFAULT);
1060 error = iommu_map_region(domain, entry, IOMMU_MAP_ENTRY_READ |
1061 ((flags & BUS_DMA_NOWRITE) ? 0 : IOMMU_MAP_ENTRY_WRITE),
1062 waitok ? IOMMU_MF_CANWAIT : 0, ma);
1064 IOMMU_DOMAIN_LOCK(domain);
1065 TAILQ_INSERT_TAIL(&map->map_entries, entry, dmamap_link);
1066 entry->flags |= IOMMU_MAP_ENTRY_MAP;
1067 IOMMU_DOMAIN_UNLOCK(domain);
1069 iommu_domain_unload_entry(entry, true);
1071 for (i = 0; i < atop(length); i++)
1072 vm_page_putfake(ma[i]);
1078 iommu_domain_unload_task(void *arg, int pending)
1080 struct iommu_domain *domain;
1081 struct iommu_map_entries_tailq entries;
1084 TAILQ_INIT(&entries);
1087 IOMMU_DOMAIN_LOCK(domain);
1088 TAILQ_SWAP(&domain->unload_entries, &entries,
1089 iommu_map_entry, dmamap_link);
1090 IOMMU_DOMAIN_UNLOCK(domain);
1091 if (TAILQ_EMPTY(&entries))
1093 iommu_domain_unload(domain, &entries, true);
1098 iommu_domain_init(struct iommu_unit *unit, struct iommu_domain *domain,
1099 const struct iommu_domain_map_ops *ops)
1103 domain->iommu = unit;
1105 TASK_INIT(&domain->unload_task, 0, iommu_domain_unload_task, domain);
1106 RB_INIT(&domain->rb_root);
1107 TAILQ_INIT(&domain->unload_entries);
1108 mtx_init(&domain->lock, "iodom", NULL, MTX_DEF);
1112 iommu_domain_fini(struct iommu_domain *domain)
1115 mtx_destroy(&domain->lock);