2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 2013 The FreeBSD Foundation
7 * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
8 * under sponsorship from the FreeBSD Foundation.
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 #include <sys/cdefs.h>
33 __FBSDID("$FreeBSD$");
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/malloc.h>
40 #include <sys/interrupt.h>
41 #include <sys/kernel.h>
45 #include <sys/memdesc.h>
46 #include <sys/mutex.h>
47 #include <sys/sysctl.h>
49 #include <sys/taskqueue.h>
53 #include <dev/pci/pcireg.h>
54 #include <dev/pci/pcivar.h>
56 #include <vm/vm_extern.h>
57 #include <vm/vm_kern.h>
58 #include <vm/vm_object.h>
59 #include <vm/vm_page.h>
60 #include <vm/vm_map.h>
61 #include <machine/atomic.h>
62 #include <machine/bus.h>
63 #include <machine/md_var.h>
64 #include <machine/specialreg.h>
65 #include <x86/include/busdma_impl.h>
66 #include <x86/iommu/intel_reg.h>
67 #include <x86/iommu/busdma_dmar.h>
68 #include <x86/iommu/intel_dmar.h>
71 * busdma_dmar.c, the implementation of the busdma(9) interface using
72 * DMAR units from Intel VT-d.
76 dmar_bus_dma_is_dev_disabled(int domain, int bus, int slot, int func)
81 static const char bounce_str[] = "bounce";
82 static const char dmar_str[] = "dmar";
85 env = kern_getenv("hw.busdma.default");
87 if (strcmp(env, bounce_str) == 0)
89 else if (strcmp(env, dmar_str) == 0)
94 snprintf(str, sizeof(str), "hw.busdma.pci%d.%d.%d.%d",
95 domain, bus, slot, func);
96 env = kern_getenv(str);
98 return (default_bounce != 0);
99 if (strcmp(env, bounce_str) == 0)
101 else if (strcmp(env, dmar_str) == 0)
104 ret = default_bounce != 0;
110 * Given original device, find the requester ID that will be seen by
111 * the DMAR unit and used for page table lookup. PCI bridges may take
112 * ownership of transactions from downstream devices, so it may not be
113 * the same as the BSF of the target device. In those cases, all
114 * devices downstream of the bridge must share a single mapping
115 * domain, and must collectively be assigned to use either DMAR or
119 dmar_get_requester(device_t dev, uint16_t *rid)
121 devclass_t pci_class;
122 device_t l, pci, pcib, pcip, pcibp, requester;
127 pci_class = devclass_find("pci");
130 *rid = pci_get_rid(dev);
133 * Walk the bridge hierarchy from the target device to the
134 * host port to find the translating bridge nearest the DMAR
138 pci = device_get_parent(l);
139 KASSERT(pci != NULL, ("dmar_get_requester(%s): NULL parent "
140 "for %s", device_get_name(dev), device_get_name(l)));
141 KASSERT(device_get_devclass(pci) == pci_class,
142 ("dmar_get_requester(%s): non-pci parent %s for %s",
143 device_get_name(dev), device_get_name(pci),
144 device_get_name(l)));
146 pcib = device_get_parent(pci);
147 KASSERT(pcib != NULL, ("dmar_get_requester(%s): NULL bridge "
148 "for %s", device_get_name(dev), device_get_name(pci)));
151 * The parent of our "bridge" isn't another PCI bus,
152 * so pcib isn't a PCI->PCI bridge but rather a host
153 * port, and the requester ID won't be translated
156 pcip = device_get_parent(pcib);
157 if (device_get_devclass(pcip) != pci_class)
159 pcibp = device_get_parent(pcip);
161 if (pci_find_cap(l, PCIY_EXPRESS, &cap_offset) == 0) {
163 * Do not stop the loop even if the target
164 * device is PCIe, because it is possible (but
165 * unlikely) to have a PCI->PCIe bridge
166 * somewhere in the hierarchy.
171 * Device is not PCIe, it cannot be seen as a
172 * requester by DMAR unit. Check whether the
175 bridge_is_pcie = pci_find_cap(pcib, PCIY_EXPRESS,
180 * Check for a buggy PCIe/PCI bridge that
181 * doesn't report the express capability. If
182 * the bridge above it is express but isn't a
183 * PCI bridge, then we know pcib is actually a
186 if (!bridge_is_pcie && pci_find_cap(pcibp,
187 PCIY_EXPRESS, &cap_offset) == 0) {
188 pcie_flags = pci_read_config(pcibp,
189 cap_offset + PCIER_FLAGS, 2);
190 if ((pcie_flags & PCIEM_FLAGS_TYPE) !=
191 PCIEM_TYPE_PCI_BRIDGE)
192 bridge_is_pcie = true;
195 if (bridge_is_pcie) {
197 * The current device is not PCIe, but
198 * the bridge above it is. This is a
199 * PCIe->PCI bridge. Assume that the
200 * requester ID will be the secondary
201 * bus number with slot and function
204 * XXX: Doesn't handle the case where
205 * the bridge is PCIe->PCI-X, and the
206 * bridge will only take ownership of
207 * requests in some cases. We should
208 * provide context entries with the
209 * same page tables for taken and
210 * non-taken transactions.
212 *rid = PCI_RID(pci_get_bus(l), 0, 0);
216 * Neither the device nor the bridge
217 * above it are PCIe. This is a
218 * conventional PCI->PCI bridge, which
219 * will use the bridge's BSF as the
222 *rid = pci_get_rid(pcib);
231 dmar_instantiate_ctx(struct dmar_unit *dmar, device_t dev, bool rmrr)
234 struct dmar_ctx *ctx;
238 requester = dmar_get_requester(dev, &rid);
241 * If the user requested the IOMMU disabled for the device, we
242 * cannot disable the DMAR, due to possibility of other
243 * devices on the same DMAR still requiring translation.
244 * Instead provide the identity mapping for the device
247 disabled = dmar_bus_dma_is_dev_disabled(pci_get_domain(requester),
248 pci_get_bus(requester), pci_get_slot(requester),
249 pci_get_function(requester));
250 ctx = dmar_get_ctx_for_dev(dmar, requester, rid, disabled, rmrr);
255 * Keep the first reference on context, release the
259 if ((ctx->flags & DMAR_CTX_DISABLED) == 0) {
260 ctx->flags |= DMAR_CTX_DISABLED;
263 dmar_free_ctx_locked(dmar, ctx);
271 dmar_get_dma_tag(device_t dev, device_t child)
273 struct dmar_unit *dmar;
274 struct dmar_ctx *ctx;
277 dmar = dmar_find(child);
278 /* Not in scope of any DMAR ? */
281 if (!dmar->dma_enabled)
283 dmar_quirks_pre_use(dmar);
284 dmar_instantiate_rmrr_ctxs(dmar);
286 ctx = dmar_instantiate_ctx(dmar, child, false);
287 res = ctx == NULL ? NULL : (bus_dma_tag_t)&ctx->ctx_tag;
291 static MALLOC_DEFINE(M_DMAR_DMAMAP, "dmar_dmamap", "Intel DMAR DMA Map");
293 static void dmar_bus_schedule_dmamap(struct dmar_unit *unit,
294 struct bus_dmamap_dmar *map);
297 dmar_bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
298 bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr,
299 bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize,
300 int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
301 void *lockfuncarg, bus_dma_tag_t *dmat)
303 struct bus_dma_tag_dmar *newtag, *oldtag;
307 error = common_bus_dma_tag_create(parent != NULL ?
308 &((struct bus_dma_tag_dmar *)parent)->common : NULL, alignment,
309 boundary, lowaddr, highaddr, filter, filterarg, maxsize,
310 nsegments, maxsegsz, flags, lockfunc, lockfuncarg,
311 sizeof(struct bus_dma_tag_dmar), (void **)&newtag);
315 oldtag = (struct bus_dma_tag_dmar *)parent;
316 newtag->common.impl = &bus_dma_dmar_impl;
317 newtag->ctx = oldtag->ctx;
318 newtag->owner = oldtag->owner;
320 *dmat = (bus_dma_tag_t)newtag;
322 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
323 __func__, newtag, (newtag != NULL ? newtag->common.flags : 0),
329 dmar_bus_dma_tag_set_domain(bus_dma_tag_t dmat)
336 dmar_bus_dma_tag_destroy(bus_dma_tag_t dmat1)
338 struct bus_dma_tag_dmar *dmat, *dmat_copy, *parent;
342 dmat_copy = dmat = (struct bus_dma_tag_dmar *)dmat1;
345 if (dmat->map_count != 0) {
349 while (dmat != NULL) {
350 parent = (struct bus_dma_tag_dmar *)dmat->common.parent;
351 if (atomic_fetchadd_int(&dmat->common.ref_count, -1) ==
353 if (dmat == &dmat->ctx->ctx_tag)
354 dmar_free_ctx(dmat->ctx);
355 free_domain(dmat->segments, M_DMAR_DMAMAP);
356 free(dmat, M_DEVBUF);
363 CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error);
368 dmar_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
370 struct bus_dma_tag_dmar *tag;
371 struct bus_dmamap_dmar *map;
373 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "%s", __func__);
375 tag = (struct bus_dma_tag_dmar *)dmat;
376 map = malloc_domain(sizeof(*map), M_DMAR_DMAMAP,
377 tag->common.domain, M_NOWAIT | M_ZERO);
382 if (tag->segments == NULL) {
383 tag->segments = malloc_domain(sizeof(bus_dma_segment_t) *
384 tag->common.nsegments, M_DMAR_DMAMAP,
385 tag->common.domain, M_NOWAIT);
386 if (tag->segments == NULL) {
387 free_domain(map, M_DMAR_DMAMAP);
392 TAILQ_INIT(&map->map_entries);
395 map->cansleep = false;
397 *mapp = (bus_dmamap_t)map;
403 dmar_bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map1)
405 struct bus_dma_tag_dmar *tag;
406 struct bus_dmamap_dmar *map;
407 struct dmar_domain *domain;
409 tag = (struct bus_dma_tag_dmar *)dmat;
410 map = (struct bus_dmamap_dmar *)map1;
412 domain = tag->ctx->domain;
413 DMAR_DOMAIN_LOCK(domain);
414 if (!TAILQ_EMPTY(&map->map_entries)) {
415 DMAR_DOMAIN_UNLOCK(domain);
418 DMAR_DOMAIN_UNLOCK(domain);
419 free_domain(map, M_DMAR_DMAMAP);
427 dmar_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
430 struct bus_dma_tag_dmar *tag;
431 struct bus_dmamap_dmar *map;
435 error = dmar_bus_dmamap_create(dmat, flags, mapp);
439 mflags = (flags & BUS_DMA_NOWAIT) != 0 ? M_NOWAIT : M_WAITOK;
440 mflags |= (flags & BUS_DMA_ZERO) != 0 ? M_ZERO : 0;
441 attr = (flags & BUS_DMA_NOCACHE) != 0 ? VM_MEMATTR_UNCACHEABLE :
444 tag = (struct bus_dma_tag_dmar *)dmat;
445 map = (struct bus_dmamap_dmar *)*mapp;
447 if (tag->common.maxsize < PAGE_SIZE &&
448 tag->common.alignment <= tag->common.maxsize &&
449 attr == VM_MEMATTR_DEFAULT) {
450 *vaddr = malloc_domain(tag->common.maxsize, M_DEVBUF,
451 tag->common.domain, mflags);
452 map->flags |= BUS_DMAMAP_DMAR_MALLOC;
454 *vaddr = (void *)kmem_alloc_attr_domain(tag->common.domain,
455 tag->common.maxsize, mflags, 0ul, BUS_SPACE_MAXADDR,
457 map->flags |= BUS_DMAMAP_DMAR_KMEM_ALLOC;
459 if (*vaddr == NULL) {
460 dmar_bus_dmamap_destroy(dmat, *mapp);
468 dmar_bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map1)
470 struct bus_dma_tag_dmar *tag;
471 struct bus_dmamap_dmar *map;
473 tag = (struct bus_dma_tag_dmar *)dmat;
474 map = (struct bus_dmamap_dmar *)map1;
476 if ((map->flags & BUS_DMAMAP_DMAR_MALLOC) != 0) {
477 free_domain(vaddr, M_DEVBUF);
478 map->flags &= ~BUS_DMAMAP_DMAR_MALLOC;
480 KASSERT((map->flags & BUS_DMAMAP_DMAR_KMEM_ALLOC) != 0,
481 ("dmar_bus_dmamem_free for non alloced map %p", map));
482 kmem_free(kernel_arena, (vm_offset_t)vaddr, tag->common.maxsize);
483 map->flags &= ~BUS_DMAMAP_DMAR_KMEM_ALLOC;
486 dmar_bus_dmamap_destroy(dmat, map1);
490 dmar_bus_dmamap_load_something1(struct bus_dma_tag_dmar *tag,
491 struct bus_dmamap_dmar *map, vm_page_t *ma, int offset, bus_size_t buflen,
492 int flags, bus_dma_segment_t *segs, int *segp,
493 struct dmar_map_entries_tailq *unroll_list)
495 struct dmar_ctx *ctx;
496 struct dmar_domain *domain;
497 struct dmar_map_entry *entry;
500 int error, idx, gas_flags, seg;
502 KASSERT(offset < DMAR_PAGE_SIZE, ("offset %d", offset));
504 segs = tag->segments;
506 domain = ctx->domain;
512 if (seg >= tag->common.nsegments) {
516 buflen1 = buflen > tag->common.maxsegsz ?
517 tag->common.maxsegsz : buflen;
518 size = round_page(offset + buflen1);
521 * (Too) optimistically allow split if there are more
522 * then one segments left.
524 gas_flags = map->cansleep ? DMAR_GM_CANWAIT : 0;
525 if (seg + 1 < tag->common.nsegments)
526 gas_flags |= DMAR_GM_CANSPLIT;
528 error = dmar_gas_map(domain, &tag->common, size, offset,
529 DMAR_MAP_ENTRY_READ | DMAR_MAP_ENTRY_WRITE,
530 gas_flags, ma + idx, &entry);
533 if ((gas_flags & DMAR_GM_CANSPLIT) != 0) {
534 KASSERT(size >= entry->end - entry->start,
535 ("split increased entry size %jx %jx %jx",
536 (uintmax_t)size, (uintmax_t)entry->start,
537 (uintmax_t)entry->end));
538 size = entry->end - entry->start;
542 KASSERT(entry->end - entry->start == size,
543 ("no split allowed %jx %jx %jx",
544 (uintmax_t)size, (uintmax_t)entry->start,
545 (uintmax_t)entry->end));
547 if (offset + buflen1 > size)
548 buflen1 = size - offset;
549 if (buflen1 > tag->common.maxsegsz)
550 buflen1 = tag->common.maxsegsz;
552 KASSERT(((entry->start + offset) & (tag->common.alignment - 1))
554 ("alignment failed: ctx %p start 0x%jx offset %x "
555 "align 0x%jx", ctx, (uintmax_t)entry->start, offset,
556 (uintmax_t)tag->common.alignment));
557 KASSERT(entry->end <= tag->common.lowaddr ||
558 entry->start >= tag->common.highaddr,
559 ("entry placement failed: ctx %p start 0x%jx end 0x%jx "
560 "lowaddr 0x%jx highaddr 0x%jx", ctx,
561 (uintmax_t)entry->start, (uintmax_t)entry->end,
562 (uintmax_t)tag->common.lowaddr,
563 (uintmax_t)tag->common.highaddr));
564 KASSERT(dmar_test_boundary(entry->start + offset, buflen1,
565 tag->common.boundary),
566 ("boundary failed: ctx %p start 0x%jx end 0x%jx "
567 "boundary 0x%jx", ctx, (uintmax_t)entry->start,
568 (uintmax_t)entry->end, (uintmax_t)tag->common.boundary));
569 KASSERT(buflen1 <= tag->common.maxsegsz,
570 ("segment too large: ctx %p start 0x%jx end 0x%jx "
571 "buflen1 0x%jx maxsegsz 0x%jx", ctx,
572 (uintmax_t)entry->start, (uintmax_t)entry->end,
573 (uintmax_t)buflen1, (uintmax_t)tag->common.maxsegsz));
575 DMAR_DOMAIN_LOCK(domain);
576 TAILQ_INSERT_TAIL(&map->map_entries, entry, dmamap_link);
577 entry->flags |= DMAR_MAP_ENTRY_MAP;
578 DMAR_DOMAIN_UNLOCK(domain);
579 TAILQ_INSERT_TAIL(unroll_list, entry, unroll_link);
581 segs[seg].ds_addr = entry->start + offset;
582 segs[seg].ds_len = buflen1;
584 idx += OFF_TO_IDX(trunc_page(offset + buflen1));
586 offset &= DMAR_PAGE_MASK;
595 dmar_bus_dmamap_load_something(struct bus_dma_tag_dmar *tag,
596 struct bus_dmamap_dmar *map, vm_page_t *ma, int offset, bus_size_t buflen,
597 int flags, bus_dma_segment_t *segs, int *segp)
599 struct dmar_ctx *ctx;
600 struct dmar_domain *domain;
601 struct dmar_map_entry *entry, *entry1;
602 struct dmar_map_entries_tailq unroll_list;
606 domain = ctx->domain;
607 atomic_add_long(&ctx->loads, 1);
609 TAILQ_INIT(&unroll_list);
610 error = dmar_bus_dmamap_load_something1(tag, map, ma, offset,
611 buflen, flags, segs, segp, &unroll_list);
614 * The busdma interface does not allow us to report
615 * partial buffer load, so unfortunately we have to
616 * revert all work done.
618 DMAR_DOMAIN_LOCK(domain);
619 TAILQ_FOREACH_SAFE(entry, &unroll_list, unroll_link,
622 * No entries other than what we have created
623 * during the failed run might have been
624 * inserted there in between, since we own ctx
627 TAILQ_REMOVE(&map->map_entries, entry, dmamap_link);
628 TAILQ_REMOVE(&unroll_list, entry, unroll_link);
629 TAILQ_INSERT_TAIL(&domain->unload_entries, entry,
632 DMAR_DOMAIN_UNLOCK(domain);
633 taskqueue_enqueue(domain->dmar->delayed_taskqueue,
634 &domain->unload_task);
637 if (error == ENOMEM && (flags & BUS_DMA_NOWAIT) == 0 &&
640 if (error == EINPROGRESS)
641 dmar_bus_schedule_dmamap(domain->dmar, map);
646 dmar_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map1,
647 struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags,
648 bus_dma_segment_t *segs, int *segp)
650 struct bus_dma_tag_dmar *tag;
651 struct bus_dmamap_dmar *map;
653 tag = (struct bus_dma_tag_dmar *)dmat;
654 map = (struct bus_dmamap_dmar *)map1;
655 return (dmar_bus_dmamap_load_something(tag, map, ma, ma_offs, tlen,
660 dmar_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map1,
661 vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs,
664 struct bus_dma_tag_dmar *tag;
665 struct bus_dmamap_dmar *map;
667 vm_paddr_t pstart, pend;
668 int error, i, ma_cnt, offset;
670 tag = (struct bus_dma_tag_dmar *)dmat;
671 map = (struct bus_dmamap_dmar *)map1;
672 pstart = trunc_page(buf);
673 pend = round_page(buf + buflen);
674 offset = buf & PAGE_MASK;
675 ma_cnt = OFF_TO_IDX(pend - pstart);
676 ma = malloc(sizeof(vm_page_t) * ma_cnt, M_DEVBUF, map->cansleep ?
677 M_WAITOK : M_NOWAIT);
680 for (i = 0; i < ma_cnt; i++)
681 ma[i] = PHYS_TO_VM_PAGE(pstart + i * PAGE_SIZE);
682 error = dmar_bus_dmamap_load_something(tag, map, ma, offset, buflen,
689 dmar_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map1, void *buf,
690 bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs,
693 struct bus_dma_tag_dmar *tag;
694 struct bus_dmamap_dmar *map;
696 vm_paddr_t pstart, pend, paddr;
697 int error, i, ma_cnt, offset;
699 tag = (struct bus_dma_tag_dmar *)dmat;
700 map = (struct bus_dmamap_dmar *)map1;
701 pstart = trunc_page((vm_offset_t)buf);
702 pend = round_page((vm_offset_t)buf + buflen);
703 offset = (vm_offset_t)buf & PAGE_MASK;
704 ma_cnt = OFF_TO_IDX(pend - pstart);
705 ma = malloc(sizeof(vm_page_t) * ma_cnt, M_DEVBUF, map->cansleep ?
706 M_WAITOK : M_NOWAIT);
711 * If dumping, do not attempt to call
712 * PHYS_TO_VM_PAGE() at all. It may return non-NULL
713 * but the vm_page returned might be not initialized,
714 * e.g. for the kernel itself.
716 KASSERT(pmap == kernel_pmap, ("non-kernel address write"));
717 fma = malloc(sizeof(struct vm_page) * ma_cnt, M_DEVBUF,
718 M_ZERO | (map->cansleep ? M_WAITOK : M_NOWAIT));
723 for (i = 0; i < ma_cnt; i++, pstart += PAGE_SIZE) {
724 paddr = pmap_kextract(pstart);
725 vm_page_initfake(&fma[i], paddr, VM_MEMATTR_DEFAULT);
730 for (i = 0; i < ma_cnt; i++, pstart += PAGE_SIZE) {
731 if (pmap == kernel_pmap)
732 paddr = pmap_kextract(pstart);
734 paddr = pmap_extract(pmap, pstart);
735 ma[i] = PHYS_TO_VM_PAGE(paddr);
736 KASSERT(VM_PAGE_TO_PHYS(ma[i]) == paddr,
737 ("PHYS_TO_VM_PAGE failed %jx %jx m %p",
738 (uintmax_t)paddr, (uintmax_t)VM_PAGE_TO_PHYS(ma[i]),
742 error = dmar_bus_dmamap_load_something(tag, map, ma, offset, buflen,
750 dmar_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map1,
751 struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg)
753 struct bus_dmamap_dmar *map;
757 map = (struct bus_dmamap_dmar *)map1;
759 map->tag = (struct bus_dma_tag_dmar *)dmat;
760 map->callback = callback;
761 map->callback_arg = callback_arg;
764 static bus_dma_segment_t *
765 dmar_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map1,
766 bus_dma_segment_t *segs, int nsegs, int error)
768 struct bus_dma_tag_dmar *tag;
769 struct bus_dmamap_dmar *map;
771 tag = (struct bus_dma_tag_dmar *)dmat;
772 map = (struct bus_dmamap_dmar *)map1;
775 KASSERT(map->cansleep,
776 ("map not locked and not sleepable context %p", map));
779 * We are called from the delayed context. Relock the
782 (tag->common.lockfunc)(tag->common.lockfuncarg, BUS_DMA_LOCK);
787 segs = tag->segments;
792 * The limitations of busdma KPI forces the dmar to perform the actual
793 * unload, consisting of the unmapping of the map entries page tables,
794 * from the delayed context on i386, since page table page mapping
795 * might require a sleep to be successfull. The unfortunate
796 * consequence is that the DMA requests can be served some time after
797 * the bus_dmamap_unload() call returned.
799 * On amd64, we assume that sf allocation cannot fail.
802 dmar_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map1)
804 struct bus_dma_tag_dmar *tag;
805 struct bus_dmamap_dmar *map;
806 struct dmar_ctx *ctx;
807 struct dmar_domain *domain;
808 #if defined(__amd64__)
809 struct dmar_map_entries_tailq entries;
812 tag = (struct bus_dma_tag_dmar *)dmat;
813 map = (struct bus_dmamap_dmar *)map1;
815 domain = ctx->domain;
816 atomic_add_long(&ctx->unloads, 1);
818 #if defined(__i386__)
819 DMAR_DOMAIN_LOCK(domain);
820 TAILQ_CONCAT(&domain->unload_entries, &map->map_entries, dmamap_link);
821 DMAR_DOMAIN_UNLOCK(domain);
822 taskqueue_enqueue(domain->dmar->delayed_taskqueue,
823 &domain->unload_task);
824 #else /* defined(__amd64__) */
825 TAILQ_INIT(&entries);
826 DMAR_DOMAIN_LOCK(domain);
827 TAILQ_CONCAT(&entries, &map->map_entries, dmamap_link);
828 DMAR_DOMAIN_UNLOCK(domain);
829 THREAD_NO_SLEEPING();
830 dmar_domain_unload(domain, &entries, false);
831 THREAD_SLEEPING_OK();
832 KASSERT(TAILQ_EMPTY(&entries), ("lazy dmar_ctx_unload %p", ctx));
837 dmar_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map,
842 struct bus_dma_impl bus_dma_dmar_impl = {
843 .tag_create = dmar_bus_dma_tag_create,
844 .tag_destroy = dmar_bus_dma_tag_destroy,
845 .tag_set_domain = dmar_bus_dma_tag_set_domain,
846 .map_create = dmar_bus_dmamap_create,
847 .map_destroy = dmar_bus_dmamap_destroy,
848 .mem_alloc = dmar_bus_dmamem_alloc,
849 .mem_free = dmar_bus_dmamem_free,
850 .load_phys = dmar_bus_dmamap_load_phys,
851 .load_buffer = dmar_bus_dmamap_load_buffer,
852 .load_ma = dmar_bus_dmamap_load_ma,
853 .map_waitok = dmar_bus_dmamap_waitok,
854 .map_complete = dmar_bus_dmamap_complete,
855 .map_unload = dmar_bus_dmamap_unload,
856 .map_sync = dmar_bus_dmamap_sync,
860 dmar_bus_task_dmamap(void *arg, int pending)
862 struct bus_dma_tag_dmar *tag;
863 struct bus_dmamap_dmar *map;
864 struct dmar_unit *unit;
868 while ((map = TAILQ_FIRST(&unit->delayed_maps)) != NULL) {
869 TAILQ_REMOVE(&unit->delayed_maps, map, delay_link);
872 map->cansleep = true;
874 bus_dmamap_load_mem((bus_dma_tag_t)tag, (bus_dmamap_t)map,
875 &map->mem, map->callback, map->callback_arg,
877 map->cansleep = false;
879 (tag->common.lockfunc)(tag->common.lockfuncarg,
883 map->cansleep = false;
890 dmar_bus_schedule_dmamap(struct dmar_unit *unit, struct bus_dmamap_dmar *map)
895 TAILQ_INSERT_TAIL(&unit->delayed_maps, map, delay_link);
897 taskqueue_enqueue(unit->delayed_taskqueue, &unit->dmamap_load_task);
901 dmar_init_busdma(struct dmar_unit *unit)
904 unit->dma_enabled = 1;
905 TUNABLE_INT_FETCH("hw.dmar.dma", &unit->dma_enabled);
906 TAILQ_INIT(&unit->delayed_maps);
907 TASK_INIT(&unit->dmamap_load_task, 0, dmar_bus_task_dmamap, unit);
908 unit->delayed_taskqueue = taskqueue_create("dmar", M_WAITOK,
909 taskqueue_thread_enqueue, &unit->delayed_taskqueue);
910 taskqueue_start_threads(&unit->delayed_taskqueue, 1, PI_DISK,
911 "dmar%d busdma taskq", unit->unit);
916 dmar_fini_busdma(struct dmar_unit *unit)
919 if (unit->delayed_taskqueue == NULL)
922 taskqueue_drain(unit->delayed_taskqueue, &unit->dmamap_load_task);
923 taskqueue_free(unit->delayed_taskqueue);
924 unit->delayed_taskqueue = NULL;