2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 2013 The FreeBSD Foundation
7 * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
8 * under sponsorship from the FreeBSD Foundation.
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 #include <sys/cdefs.h>
33 __FBSDID("$FreeBSD$");
37 #include <sys/param.h>
39 #include <sys/kernel.h>
40 #include <sys/malloc.h>
41 #include <sys/memdesc.h>
42 #include <sys/module.h>
44 #include <sys/taskqueue.h>
47 #include <machine/bus.h>
48 #include <contrib/dev/acpica/include/acpi.h>
49 #include <contrib/dev/acpica/include/accommon.h>
50 #include <dev/acpica/acpivar.h>
51 #include <dev/pci/pcireg.h>
52 #include <dev/pci/pcivar.h>
54 #include <vm/vm_extern.h>
55 #include <vm/vm_kern.h>
56 #include <vm/vm_page.h>
57 #include <vm/vm_map.h>
58 #include <x86/include/busdma_impl.h>
59 #include <x86/iommu/intel_reg.h>
60 #include <x86/iommu/busdma_dmar.h>
61 #include <x86/iommu/intel_dmar.h>
64 * Fault interrupt handling for DMARs. If advanced fault logging is
65 * not implemented by hardware, the code emulates it. Fast interrupt
66 * handler flushes the fault registers into circular buffer at
67 * unit->fault_log, and schedules a task.
69 * The fast handler is used since faults usually come in bursts, and
70 * number of fault log registers is limited, e.g. down to one for 5400
71 * MCH. We are trying to reduce the latency for clearing the fault
72 * register file. The task is usually long-running, since printf() is
73 * slow, but this is not problematic because bursts are rare.
75 * For the same reason, each translation unit task is executed in its
78 * XXXKIB It seems there is no hardware available which implements
79 * advanced fault logging, so the code to handle AFL is not written.
83 dmar_fault_next(struct dmar_unit *unit, int faultp)
87 if (faultp == unit->fault_log_size)
93 dmar_fault_intr_clear(struct dmar_unit *unit, uint32_t fsts)
98 if ((fsts & DMAR_FSTS_ITE) != 0) {
99 printf("DMAR%d: Invalidation timed out\n", unit->unit);
100 clear |= DMAR_FSTS_ITE;
102 if ((fsts & DMAR_FSTS_ICE) != 0) {
103 printf("DMAR%d: Invalidation completion error\n",
105 clear |= DMAR_FSTS_ICE;
107 if ((fsts & DMAR_FSTS_IQE) != 0) {
108 printf("DMAR%d: Invalidation queue error\n",
110 clear |= DMAR_FSTS_IQE;
112 if ((fsts & DMAR_FSTS_APF) != 0) {
113 printf("DMAR%d: Advanced pending fault\n", unit->unit);
114 clear |= DMAR_FSTS_APF;
116 if ((fsts & DMAR_FSTS_AFO) != 0) {
117 printf("DMAR%d: Advanced fault overflow\n", unit->unit);
118 clear |= DMAR_FSTS_AFO;
121 dmar_write4(unit, DMAR_FSTS_REG, clear);
125 dmar_fault_intr(void *arg)
127 struct dmar_unit *unit;
128 uint64_t fault_rec[2];
130 int fri, frir, faultp;
135 fsts = dmar_read4(unit, DMAR_FSTS_REG);
136 dmar_fault_intr_clear(unit, fsts);
138 if ((fsts & DMAR_FSTS_PPF) == 0)
141 fri = DMAR_FSTS_FRI(fsts);
143 frir = (DMAR_CAP_FRO(unit->hw_cap) + fri) * 16;
144 fault_rec[1] = dmar_read8(unit, frir + 8);
145 if ((fault_rec[1] & DMAR_FRCD2_F) == 0)
147 fault_rec[0] = dmar_read8(unit, frir);
148 dmar_write4(unit, frir + 12, DMAR_FRCD2_F32);
149 DMAR_FAULT_LOCK(unit);
150 faultp = unit->fault_log_head;
151 if (dmar_fault_next(unit, faultp) == unit->fault_log_tail) {
152 /* XXXKIB log overflow */
154 unit->fault_log[faultp] = fault_rec[0];
155 unit->fault_log[faultp + 1] = fault_rec[1];
156 unit->fault_log_head = dmar_fault_next(unit, faultp);
159 DMAR_FAULT_UNLOCK(unit);
161 if (fri >= DMAR_CAP_NFR(unit->hw_cap))
167 * On SandyBridge, due to errata BJ124, IvyBridge errata
168 * BV100, and Haswell errata HSD40, "Spurious Intel VT-d
169 * Interrupts May Occur When the PFO Bit is Set". Handle the
170 * cases by clearing overflow bit even if no fault is
173 * On IvyBridge, errata BV30 states that clearing clear
174 * DMAR_FRCD2_F bit in the fault register causes spurious
175 * interrupt. Do nothing.
178 if ((fsts & DMAR_FSTS_PFO) != 0) {
179 printf("DMAR%d: Fault Overflow\n", unit->unit);
180 dmar_write4(unit, DMAR_FSTS_REG, DMAR_FSTS_PFO);
184 taskqueue_enqueue(unit->fault_taskqueue,
187 return (FILTER_HANDLED);
191 dmar_fault_task(void *arg, int pending __unused)
193 struct dmar_unit *unit;
194 struct dmar_ctx *ctx;
195 uint64_t fault_rec[2];
196 int sid, bus, slot, func, faultp;
199 DMAR_FAULT_LOCK(unit);
201 faultp = unit->fault_log_tail;
202 if (faultp == unit->fault_log_head)
205 fault_rec[0] = unit->fault_log[faultp];
206 fault_rec[1] = unit->fault_log[faultp + 1];
207 unit->fault_log_tail = dmar_fault_next(unit, faultp);
208 DMAR_FAULT_UNLOCK(unit);
210 sid = DMAR_FRCD2_SID(fault_rec[1]);
211 printf("DMAR%d: ", unit->unit);
213 ctx = dmar_find_ctx_locked(unit, sid);
215 printf("<unknown dev>:");
218 * Note that the slot and function will not be correct
219 * if ARI is in use, but without a ctx entry we have
220 * no way of knowing whether ARI is in use or not.
222 bus = PCI_RID2BUS(sid);
223 slot = PCI_RID2SLOT(sid);
224 func = PCI_RID2FUNC(sid);
226 ctx->flags |= DMAR_CTX_FAULTED;
227 ctx->last_fault_rec[0] = fault_rec[0];
228 ctx->last_fault_rec[1] = fault_rec[1];
229 device_print_prettyname(ctx->ctx_tag.owner);
230 bus = pci_get_bus(ctx->ctx_tag.owner);
231 slot = pci_get_slot(ctx->ctx_tag.owner);
232 func = pci_get_function(ctx->ctx_tag.owner);
236 "pci%d:%d:%d sid %x fault acc %x adt 0x%x reason 0x%x "
238 bus, slot, func, sid, DMAR_FRCD2_T(fault_rec[1]),
239 DMAR_FRCD2_AT(fault_rec[1]), DMAR_FRCD2_FR(fault_rec[1]),
240 (uintmax_t)fault_rec[0]);
241 DMAR_FAULT_LOCK(unit);
243 DMAR_FAULT_UNLOCK(unit);
247 dmar_clear_faults(struct dmar_unit *unit)
249 uint32_t frec, frir, fsts;
252 for (i = 0; i < DMAR_CAP_NFR(unit->hw_cap); i++) {
253 frir = (DMAR_CAP_FRO(unit->hw_cap) + i) * 16;
254 frec = dmar_read4(unit, frir + 12);
255 if ((frec & DMAR_FRCD2_F32) == 0)
257 dmar_write4(unit, frir + 12, DMAR_FRCD2_F32);
259 fsts = dmar_read4(unit, DMAR_FSTS_REG);
260 dmar_write4(unit, DMAR_FSTS_REG, fsts);
264 dmar_init_fault_log(struct dmar_unit *unit)
267 mtx_init(&unit->fault_lock, "dmarflt", NULL, MTX_SPIN);
268 unit->fault_log_size = 256; /* 128 fault log entries */
269 TUNABLE_INT_FETCH("hw.dmar.fault_log_size", &unit->fault_log_size);
270 if (unit->fault_log_size % 2 != 0)
271 panic("hw.dmar_fault_log_size must be even");
272 unit->fault_log = malloc(sizeof(uint64_t) * unit->fault_log_size,
273 M_DEVBUF, M_WAITOK | M_ZERO);
275 TASK_INIT(&unit->fault_task, 0, dmar_fault_task, unit);
276 unit->fault_taskqueue = taskqueue_create_fast("dmarff", M_WAITOK,
277 taskqueue_thread_enqueue, &unit->fault_taskqueue);
278 taskqueue_start_threads(&unit->fault_taskqueue, 1, PI_AV,
279 "dmar%d fault taskq", unit->unit);
282 dmar_disable_fault_intr(unit);
283 dmar_clear_faults(unit);
284 dmar_enable_fault_intr(unit);
291 dmar_fini_fault_log(struct dmar_unit *unit)
295 dmar_disable_fault_intr(unit);
298 if (unit->fault_taskqueue == NULL)
301 taskqueue_drain(unit->fault_taskqueue, &unit->fault_task);
302 taskqueue_free(unit->fault_taskqueue);
303 unit->fault_taskqueue = NULL;
304 mtx_destroy(&unit->fault_lock);
306 free(unit->fault_log, M_DEVBUF);
307 unit->fault_log = NULL;
308 unit->fault_log_head = unit->fault_log_tail = 0;
312 dmar_enable_fault_intr(struct dmar_unit *unit)
316 DMAR_ASSERT_LOCKED(unit);
317 fectl = dmar_read4(unit, DMAR_FECTL_REG);
318 fectl &= ~DMAR_FECTL_IM;
319 dmar_write4(unit, DMAR_FECTL_REG, fectl);
323 dmar_disable_fault_intr(struct dmar_unit *unit)
327 DMAR_ASSERT_LOCKED(unit);
328 fectl = dmar_read4(unit, DMAR_FECTL_REG);
329 dmar_write4(unit, DMAR_FECTL_REG, fectl | DMAR_FECTL_IM);