2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4 * Copyright (c) 2013 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com>
5 * Copyright (c) 2013 Neel Natu <neel@freebsd.org>
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
17 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 #include <sys/cdefs.h>
33 __FBSDID("$FreeBSD$");
35 #include <sys/param.h>
36 #include <sys/queue.h>
38 #include <sys/mutex.h>
39 #include <sys/systm.h>
40 #include <sys/kernel.h>
41 #include <sys/malloc.h>
43 #include <x86/apicreg.h>
44 #include <machine/vmm.h>
47 #include "vmm_lapic.h"
54 #define REDIR_ENTRIES 32
55 #define RTBL_RO_BITS ((uint64_t)(IOART_REM_IRR | IOART_DELIVS))
64 int acnt; /* sum of pin asserts (+1) and deasserts (-1) */
65 } rtbl[REDIR_ENTRIES];
68 #define VIOAPIC_LOCK(vioapic) mtx_lock_spin(&((vioapic)->mtx))
69 #define VIOAPIC_UNLOCK(vioapic) mtx_unlock_spin(&((vioapic)->mtx))
70 #define VIOAPIC_LOCKED(vioapic) mtx_owned(&((vioapic)->mtx))
72 static MALLOC_DEFINE(M_VIOAPIC, "vioapic", "bhyve virtual ioapic");
74 #define VIOAPIC_CTR1(vioapic, fmt, a1) \
75 VM_CTR1((vioapic)->vm, fmt, a1)
77 #define VIOAPIC_CTR2(vioapic, fmt, a1, a2) \
78 VM_CTR2((vioapic)->vm, fmt, a1, a2)
80 #define VIOAPIC_CTR3(vioapic, fmt, a1, a2, a3) \
81 VM_CTR3((vioapic)->vm, fmt, a1, a2, a3)
83 #define VIOAPIC_CTR4(vioapic, fmt, a1, a2, a3, a4) \
84 VM_CTR4((vioapic)->vm, fmt, a1, a2, a3, a4)
88 pinstate_str(bool asserted)
94 return ("deasserted");
99 vioapic_send_intr(struct vioapic *vioapic, int pin)
102 uint32_t low, high, dest;
105 KASSERT(pin >= 0 && pin < REDIR_ENTRIES,
106 ("vioapic_set_pinstate: invalid pin number %d", pin));
108 KASSERT(VIOAPIC_LOCKED(vioapic),
109 ("vioapic_set_pinstate: vioapic is not locked"));
111 low = vioapic->rtbl[pin].reg;
112 high = vioapic->rtbl[pin].reg >> 32;
114 if ((low & IOART_INTMASK) == IOART_INTMSET) {
115 VIOAPIC_CTR1(vioapic, "ioapic pin%d: masked", pin);
119 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY);
120 delmode = low & IOART_DELMOD;
121 level = low & IOART_TRGRLVL ? true : false;
123 vioapic->rtbl[pin].reg |= IOART_REM_IRR;
125 vector = low & IOART_INTVEC;
126 dest = high >> APIC_ID_SHIFT;
127 vlapic_deliver_intr(vioapic->vm, level, dest, phys, delmode, vector);
131 vioapic_set_pinstate(struct vioapic *vioapic, int pin, bool newstate)
136 KASSERT(pin >= 0 && pin < REDIR_ENTRIES,
137 ("vioapic_set_pinstate: invalid pin number %d", pin));
139 KASSERT(VIOAPIC_LOCKED(vioapic),
140 ("vioapic_set_pinstate: vioapic is not locked"));
142 oldcnt = vioapic->rtbl[pin].acnt;
144 vioapic->rtbl[pin].acnt++;
146 vioapic->rtbl[pin].acnt--;
147 newcnt = vioapic->rtbl[pin].acnt;
150 VIOAPIC_CTR2(vioapic, "ioapic pin%d: bad acnt %d",
155 if (oldcnt == 0 && newcnt == 1) {
157 VIOAPIC_CTR1(vioapic, "ioapic pin%d: asserted", pin);
158 } else if (oldcnt == 1 && newcnt == 0) {
159 VIOAPIC_CTR1(vioapic, "ioapic pin%d: deasserted", pin);
161 VIOAPIC_CTR3(vioapic, "ioapic pin%d: %s, ignored, acnt %d",
162 pin, pinstate_str(newstate), newcnt);
166 vioapic_send_intr(vioapic, pin);
176 vioapic_set_irqstate(struct vm *vm, int irq, enum irqstate irqstate)
178 struct vioapic *vioapic;
180 if (irq < 0 || irq >= REDIR_ENTRIES)
183 vioapic = vm_ioapic(vm);
185 VIOAPIC_LOCK(vioapic);
187 case IRQSTATE_ASSERT:
188 vioapic_set_pinstate(vioapic, irq, true);
190 case IRQSTATE_DEASSERT:
191 vioapic_set_pinstate(vioapic, irq, false);
194 vioapic_set_pinstate(vioapic, irq, true);
195 vioapic_set_pinstate(vioapic, irq, false);
198 panic("vioapic_set_irqstate: invalid irqstate %d", irqstate);
200 VIOAPIC_UNLOCK(vioapic);
206 vioapic_assert_irq(struct vm *vm, int irq)
209 return (vioapic_set_irqstate(vm, irq, IRQSTATE_ASSERT));
213 vioapic_deassert_irq(struct vm *vm, int irq)
216 return (vioapic_set_irqstate(vm, irq, IRQSTATE_DEASSERT));
220 vioapic_pulse_irq(struct vm *vm, int irq)
223 return (vioapic_set_irqstate(vm, irq, IRQSTATE_PULSE));
227 * Reset the vlapic's trigger-mode register to reflect the ioapic pin
231 vioapic_update_tmr(struct vm *vm, int vcpuid, void *arg)
233 struct vioapic *vioapic;
234 struct vlapic *vlapic;
235 uint32_t low, high, dest;
236 int delmode, pin, vector;
239 vlapic = vm_lapic(vm, vcpuid);
240 vioapic = vm_ioapic(vm);
242 VIOAPIC_LOCK(vioapic);
244 * Reset all vectors to be edge-triggered.
246 vlapic_reset_tmr(vlapic);
247 for (pin = 0; pin < REDIR_ENTRIES; pin++) {
248 low = vioapic->rtbl[pin].reg;
249 high = vioapic->rtbl[pin].reg >> 32;
251 level = low & IOART_TRGRLVL ? true : false;
256 * For a level-triggered 'pin' let the vlapic figure out if
257 * an assertion on this 'pin' would result in an interrupt
258 * being delivered to it. If yes, then it will modify the
259 * TMR bit associated with this vector to level-triggered.
261 phys = ((low & IOART_DESTMOD) == IOART_DESTPHY);
262 delmode = low & IOART_DELMOD;
263 vector = low & IOART_INTVEC;
264 dest = high >> APIC_ID_SHIFT;
265 vlapic_set_tmr_level(vlapic, dest, phys, delmode, vector);
267 VIOAPIC_UNLOCK(vioapic);
271 vioapic_read(struct vioapic *vioapic, int vcpuid, uint32_t addr)
273 int regnum, pin, rshift;
275 regnum = addr & 0xff;
278 return (vioapic->id);
281 return (((REDIR_ENTRIES - 1) << MAXREDIRSHIFT) | 0x11);
284 return (vioapic->id);
290 /* redirection table entries */
291 if (regnum >= IOAPIC_REDTBL &&
292 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) {
293 pin = (regnum - IOAPIC_REDTBL) / 2;
294 if ((regnum - IOAPIC_REDTBL) % 2)
299 return (vioapic->rtbl[pin].reg >> rshift);
306 vioapic_write(struct vioapic *vioapic, int vcpuid, uint32_t addr, uint32_t data)
308 uint64_t data64, mask64;
309 uint64_t last, changed;
310 int regnum, pin, lshift;
313 regnum = addr & 0xff;
316 vioapic->id = data & APIC_ID_MASK;
326 /* redirection table entries */
327 if (regnum >= IOAPIC_REDTBL &&
328 regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) {
329 pin = (regnum - IOAPIC_REDTBL) / 2;
330 if ((regnum - IOAPIC_REDTBL) % 2)
335 last = vioapic->rtbl[pin].reg;
337 data64 = (uint64_t)data << lshift;
338 mask64 = (uint64_t)0xffffffff << lshift;
339 vioapic->rtbl[pin].reg &= ~mask64 | RTBL_RO_BITS;
340 vioapic->rtbl[pin].reg |= data64 & ~RTBL_RO_BITS;
342 VIOAPIC_CTR2(vioapic, "ioapic pin%d: redir table entry %#lx",
343 pin, vioapic->rtbl[pin].reg);
346 * If any fields in the redirection table entry (except mask
347 * or polarity) have changed then rendezvous all the vcpus
348 * to update their vlapic trigger-mode registers.
350 changed = last ^ vioapic->rtbl[pin].reg;
351 if (changed & ~(IOART_INTMASK | IOART_INTPOL)) {
352 VIOAPIC_CTR1(vioapic, "ioapic pin%d: recalculate "
353 "vlapic trigger-mode register", pin);
354 VIOAPIC_UNLOCK(vioapic);
355 allvcpus = vm_active_cpus(vioapic->vm);
356 vm_smp_rendezvous(vioapic->vm, vcpuid, allvcpus,
357 vioapic_update_tmr, NULL);
358 VIOAPIC_LOCK(vioapic);
362 * Generate an interrupt if the following conditions are met:
363 * - pin is not masked
364 * - previous interrupt has been EOIed
365 * - pin level is asserted
367 if ((vioapic->rtbl[pin].reg & IOART_INTMASK) == IOART_INTMCLR &&
368 (vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0 &&
369 (vioapic->rtbl[pin].acnt > 0)) {
370 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at rtbl "
371 "write, acnt %d", pin, vioapic->rtbl[pin].acnt);
372 vioapic_send_intr(vioapic, pin);
378 vioapic_mmio_rw(struct vioapic *vioapic, int vcpuid, uint64_t gpa,
379 uint64_t *data, int size, bool doread)
383 offset = gpa - VIOAPIC_BASE;
386 * The IOAPIC specification allows 32-bit wide accesses to the
387 * IOREGSEL (offset 0) and IOWIN (offset 16) registers.
389 if (size != 4 || (offset != IOREGSEL && offset != IOWIN)) {
395 VIOAPIC_LOCK(vioapic);
396 if (offset == IOREGSEL) {
398 *data = vioapic->ioregsel;
400 vioapic->ioregsel = *data;
403 *data = vioapic_read(vioapic, vcpuid,
406 vioapic_write(vioapic, vcpuid, vioapic->ioregsel,
410 VIOAPIC_UNLOCK(vioapic);
416 vioapic_mmio_read(void *vm, int vcpuid, uint64_t gpa, uint64_t *rval,
420 struct vioapic *vioapic;
422 vioapic = vm_ioapic(vm);
423 error = vioapic_mmio_rw(vioapic, vcpuid, gpa, rval, size, true);
428 vioapic_mmio_write(void *vm, int vcpuid, uint64_t gpa, uint64_t wval,
432 struct vioapic *vioapic;
434 vioapic = vm_ioapic(vm);
435 error = vioapic_mmio_rw(vioapic, vcpuid, gpa, &wval, size, false);
440 vioapic_process_eoi(struct vm *vm, int vcpuid, int vector)
442 struct vioapic *vioapic;
445 KASSERT(vector >= 0 && vector < 256,
446 ("vioapic_process_eoi: invalid vector %d", vector));
448 vioapic = vm_ioapic(vm);
449 VIOAPIC_CTR1(vioapic, "ioapic processing eoi for vector %d", vector);
452 * XXX keep track of the pins associated with this vector instead
453 * of iterating on every single pin each time.
455 VIOAPIC_LOCK(vioapic);
456 for (pin = 0; pin < REDIR_ENTRIES; pin++) {
457 if ((vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0)
459 if ((vioapic->rtbl[pin].reg & IOART_INTVEC) != vector)
461 vioapic->rtbl[pin].reg &= ~IOART_REM_IRR;
462 if (vioapic->rtbl[pin].acnt > 0) {
463 VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at eoi, "
464 "acnt %d", pin, vioapic->rtbl[pin].acnt);
465 vioapic_send_intr(vioapic, pin);
468 VIOAPIC_UNLOCK(vioapic);
472 vioapic_init(struct vm *vm)
475 struct vioapic *vioapic;
477 vioapic = malloc(sizeof(struct vioapic), M_VIOAPIC, M_WAITOK | M_ZERO);
480 mtx_init(&vioapic->mtx, "vioapic lock", NULL, MTX_SPIN);
482 /* Initialize all redirection entries to mask all interrupts */
483 for (i = 0; i < REDIR_ENTRIES; i++)
484 vioapic->rtbl[i].reg = 0x0001000000010000UL;
490 vioapic_cleanup(struct vioapic *vioapic)
493 free(vioapic, M_VIOAPIC);
497 vioapic_pincount(struct vm *vm)
500 return (REDIR_ENTRIES);