2 * SPDX-License-Identifier: BSD-2-Clause
4 * Copyright (c) 2020 Ruslan Bukin <br@bsdpad.com>
6 * This software was developed by SRI International and the University of
7 * Cambridge Computer Laboratory (Department of Computer Science and
8 * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
9 * DARPA SSITH research programme.
11 * Portions of this work was supported by Innovate UK project 105694,
12 * "Digital Security by Design (DSbD) Technology Platform Prototype".
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions
17 * 1. Redistributions of source code must retain the above copyright
18 * notice, this list of conditions and the following disclaimer.
19 * 2. Redistributions in binary form must reproduce the above copyright
20 * notice, this list of conditions and the following disclaimer in the
21 * documentation and/or other materials provided with the distribution.
23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
27 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
28 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
29 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
31 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
32 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 #include "opt_platform.h"
38 #include <sys/cdefs.h>
39 __FBSDID("$FreeBSD$");
41 #include <sys/param.h>
43 #include <sys/kernel.h>
44 #include <sys/malloc.h>
45 #include <sys/memdesc.h>
47 #include <sys/taskqueue.h>
49 #include <sys/mutex.h>
51 #include <sys/sysctl.h>
54 #include <dev/pci/pcireg.h>
55 #include <dev/pci/pcivar.h>
56 #include <machine/bus.h>
57 #include <dev/iommu/busdma_iommu.h>
58 #include <machine/vmparam.h>
61 #include <dev/fdt/fdt_common.h>
62 #include <dev/ofw/ofw_bus.h>
63 #include <dev/ofw/ofw_bus_subr.h>
69 static MALLOC_DEFINE(M_IOMMU, "IOMMU", "IOMMU framework");
71 #define IOMMU_LIST_LOCK() sx_xlock(&iommu_sx)
72 #define IOMMU_LIST_UNLOCK() sx_xunlock(&iommu_sx)
73 #define IOMMU_LIST_ASSERT_LOCKED() sx_assert(&iommu_sx, SA_XLOCKED)
75 #define dprintf(fmt, ...)
77 static struct sx iommu_sx;
80 struct iommu_unit *iommu;
81 LIST_ENTRY(iommu_entry) next;
83 static LIST_HEAD(, iommu_entry) iommu_list = LIST_HEAD_INITIALIZER(iommu_list);
86 iommu_domain_unmap_buf(struct iommu_domain *iodom, iommu_gaddr_t base,
87 iommu_gaddr_t size, int flags)
89 struct iommu_unit *iommu;
94 error = IOMMU_UNMAP(iommu->dev, iodom, base, size);
100 iommu_domain_map_buf(struct iommu_domain *iodom, iommu_gaddr_t base,
101 iommu_gaddr_t size, vm_page_t *ma, uint64_t eflags, int flags)
103 struct iommu_unit *iommu;
108 dprintf("%s: base %lx, size %lx\n", __func__, base, size);
111 if (eflags & IOMMU_MAP_ENTRY_READ)
112 prot |= VM_PROT_READ;
113 if (eflags & IOMMU_MAP_ENTRY_WRITE)
114 prot |= VM_PROT_WRITE;
118 iommu = iodom->iommu;
120 error = IOMMU_MAP(iommu->dev, iodom, va, ma, size, prot);
125 static const struct iommu_domain_map_ops domain_map_ops = {
126 .map = iommu_domain_map_buf,
127 .unmap = iommu_domain_unmap_buf,
130 static struct iommu_domain *
131 iommu_domain_alloc(struct iommu_unit *iommu)
133 struct iommu_domain *iodom;
135 iodom = IOMMU_DOMAIN_ALLOC(iommu->dev, iommu);
139 iommu_domain_init(iommu, iodom, &domain_map_ops);
140 iodom->end = VM_MAXUSER_ADDRESS;
141 iodom->iommu = iommu;
142 iommu_gas_init_domain(iodom);
148 iommu_domain_free(struct iommu_domain *iodom)
150 struct iommu_unit *iommu;
152 iommu = iodom->iommu;
156 if ((iodom->flags & IOMMU_DOMAIN_GAS_INITED) != 0) {
157 IOMMU_DOMAIN_LOCK(iodom);
158 iommu_gas_fini_domain(iodom);
159 IOMMU_DOMAIN_UNLOCK(iodom);
162 iommu_domain_fini(iodom);
164 IOMMU_DOMAIN_FREE(iommu->dev, iodom);
171 iommu_tag_init(struct bus_dma_tag_iommu *t)
175 maxaddr = BUS_SPACE_MAXADDR;
177 t->common.ref_count = 0;
178 t->common.impl = &bus_dma_iommu_impl;
179 t->common.alignment = 1;
180 t->common.boundary = 0;
181 t->common.lowaddr = maxaddr;
182 t->common.highaddr = maxaddr;
183 t->common.maxsize = maxaddr;
184 t->common.nsegments = BUS_SPACE_UNRESTRICTED;
185 t->common.maxsegsz = maxaddr;
188 static struct iommu_ctx *
189 iommu_ctx_alloc(device_t requester, struct iommu_domain *iodom, bool disabled)
191 struct iommu_unit *iommu;
192 struct iommu_ctx *ioctx;
194 iommu = iodom->iommu;
196 ioctx = IOMMU_CTX_ALLOC(iommu->dev, iodom, requester, disabled);
200 ioctx->domain = iodom;
206 iommu_ctx_init(device_t requester, struct iommu_ctx *ioctx)
208 struct bus_dma_tag_iommu *tag;
209 struct iommu_domain *iodom;
210 struct iommu_unit *iommu;
213 iodom = ioctx->domain;
214 iommu = iodom->iommu;
216 error = IOMMU_CTX_INIT(iommu->dev, ioctx);
220 tag = ioctx->tag = malloc(sizeof(struct bus_dma_tag_iommu),
221 M_IOMMU, M_WAITOK | M_ZERO);
222 tag->owner = requester;
224 tag->ctx->domain = iodom;
231 static struct iommu_unit *
232 iommu_lookup(device_t dev)
234 struct iommu_entry *entry;
235 struct iommu_unit *iommu;
238 LIST_FOREACH(entry, &iommu_list, next) {
239 iommu = entry->iommu;
240 if (iommu->dev == dev) {
251 iommu_get_ctx_ofw(device_t dev, int channel)
253 struct iommu_domain *iodom;
254 struct iommu_unit *iommu;
255 struct iommu_ctx *ioctx;
256 phandle_t node, parent;
263 node = ofw_bus_get_node(dev);
266 "%s called on not ofw based device.\n", __func__);
270 error = ofw_bus_parse_xref_list_get_length(node,
271 "iommus", "#iommu-cells", &niommus);
273 device_printf(dev, "%s can't get iommu list.\n", __func__);
278 device_printf(dev, "%s iommu list is empty.\n", __func__);
282 error = ofw_bus_parse_xref_list_alloc(node, "iommus", "#iommu-cells",
283 channel, &parent, &ncells, &cells);
285 device_printf(dev, "%s can't get iommu device xref.\n",
290 iommu_dev = OF_device_from_xref(parent);
291 if (iommu_dev == NULL) {
292 device_printf(dev, "%s can't get iommu device.\n", __func__);
296 iommu = iommu_lookup(iommu_dev);
298 device_printf(dev, "%s can't lookup iommu.\n", __func__);
303 * In our current configuration we have a domain per each ctx,
304 * so allocate a domain first.
306 iodom = iommu_domain_alloc(iommu);
308 device_printf(dev, "%s can't allocate domain.\n", __func__);
312 ioctx = iommu_ctx_alloc(dev, iodom, false);
314 iommu_domain_free(iodom);
318 ioctx->domain = iodom;
320 error = IOMMU_OFW_MD_DATA(iommu->dev, ioctx, cells, ncells);
322 device_printf(dev, "%s can't set MD data\n", __func__);
326 error = iommu_ctx_init(dev, ioctx);
328 IOMMU_CTX_FREE(iommu->dev, ioctx);
329 iommu_domain_free(iodom);
337 iommu_get_ctx(struct iommu_unit *iommu, device_t requester,
338 uint16_t rid, bool disabled, bool rmrr)
340 struct iommu_domain *iodom;
341 struct iommu_ctx *ioctx;
345 ioctx = IOMMU_CTX_LOOKUP(iommu->dev, requester);
353 * In our current configuration we have a domain per each ctx.
354 * So allocate a domain first.
356 iodom = iommu_domain_alloc(iommu);
360 ioctx = iommu_ctx_alloc(requester, iodom, disabled);
362 iommu_domain_free(iodom);
366 error = iommu_ctx_init(requester, ioctx);
368 IOMMU_CTX_FREE(iommu->dev, ioctx);
369 iommu_domain_free(iodom);
377 iommu_free_ctx_locked(struct iommu_unit *iommu, struct iommu_ctx *ioctx)
379 struct bus_dma_tag_iommu *tag;
381 IOMMU_ASSERT_LOCKED(iommu);
385 IOMMU_CTX_FREE(iommu->dev, ioctx);
391 iommu_free_ctx(struct iommu_ctx *ioctx)
393 struct iommu_unit *iommu;
394 struct iommu_domain *iodom;
397 iodom = ioctx->domain;
398 iommu = iodom->iommu;
401 iommu_free_ctx_locked(iommu, ioctx);
404 /* Since we have a domain per each ctx, remove the domain too. */
405 error = iommu_domain_free(iodom);
407 device_printf(iommu->dev, "Could not free a domain\n");
411 iommu_domain_free_entry(struct iommu_map_entry *entry, bool free)
413 struct iommu_domain *iodom;
415 iodom = entry->domain;
417 IOMMU_DOMAIN_LOCK(iodom);
418 iommu_gas_free_space(iodom, entry);
419 IOMMU_DOMAIN_UNLOCK(iodom);
422 iommu_gas_free_entry(iodom, entry);
428 iommu_domain_unload(struct iommu_domain *iodom,
429 struct iommu_map_entries_tailq *entries, bool cansleep)
431 struct iommu_map_entry *entry, *entry1;
432 int error __diagused;
434 TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) {
435 KASSERT((entry->flags & IOMMU_MAP_ENTRY_MAP) != 0,
436 ("not mapped entry %p %p", iodom, entry));
437 error = iodom->ops->unmap(iodom, entry->start, entry->end -
438 entry->start, cansleep ? IOMMU_PGF_WAITOK : 0);
439 KASSERT(error == 0, ("unmap %p error %d", iodom, error));
440 TAILQ_REMOVE(entries, entry, dmamap_link);
441 iommu_domain_free_entry(entry, true);
444 if (TAILQ_EMPTY(entries))
447 panic("entries map is not empty");
451 iommu_register(struct iommu_unit *iommu)
453 struct iommu_entry *entry;
455 mtx_init(&iommu->lock, "IOMMU", NULL, MTX_DEF);
457 entry = malloc(sizeof(struct iommu_entry), M_IOMMU, M_WAITOK | M_ZERO);
458 entry->iommu = iommu;
461 LIST_INSERT_HEAD(&iommu_list, entry, next);
464 iommu_init_busdma(iommu);
470 iommu_unregister(struct iommu_unit *iommu)
472 struct iommu_entry *entry, *tmp;
475 LIST_FOREACH_SAFE(entry, &iommu_list, next, tmp) {
476 if (entry->iommu == iommu) {
477 LIST_REMOVE(entry, next);
478 free(entry, M_IOMMU);
483 iommu_fini_busdma(iommu);
485 mtx_destroy(&iommu->lock);
491 iommu_find(device_t dev, bool verbose)
493 struct iommu_entry *entry;
494 struct iommu_unit *iommu;
498 LIST_FOREACH(entry, &iommu_list, next) {
499 iommu = entry->iommu;
500 error = IOMMU_FIND(iommu->dev, dev);
503 return (entry->iommu);
512 iommu_domain_unload_entry(struct iommu_map_entry *entry, bool free,
513 bool cansleep __unused)
516 dprintf("%s\n", __func__);
518 iommu_domain_free_entry(entry, free);
525 sx_init(&iommu_sx, "IOMMU list");
528 SYSINIT(iommu, SI_SUB_DRIVERS, SI_ORDER_FIRST, iommu_init, NULL);