/*-
 * Copyright (c) 1989, 1990 William F. Jolitz.
 * Copyright (c) 1990 The Regents of the University of California.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	from: vector.s, 386BSD 0.1 unknown origin
 * $FreeBSD$
 */

/*
 * Interrupt entry points for external interrupts triggered by I/O APICs
 * as well as IPI handlers.
 */

#include "opt_smp.h"

#include <machine/asmacros.h>
#include <x86/apicreg.h>

#include "assym.s"

#ifdef SMP
#define LK	lock ;
#else
#define LK
#endif

/*
 * I/O Interrupt Entry Point.  Rather than having one entry point for
 * each interrupt source, we use one entry point for each 32-bit word
 * in the ISR.  The handler determines the highest bit set in the ISR,
 * translates that into a vector, and passes the vector to the
 * lapic_handle_intr() function.
 */
#define	ISR_VEC(index, vec_name)					\
	.text ;								\
	SUPERALIGN_TEXT ;						\
IDTVEC(vec_name) ;							\
	PUSH_FRAME ;							\
	FAKE_MCOUNT(TF_RIP(%rsp)) ;					\
	movq	lapic, %rdx ;	/* pointer to local APIC */		\
	movl	LA_ISR + 16 * (index)(%rdx), %eax ;	/* load ISR */	\
	bsrl	%eax, %eax ;	/* index of highest set bit in ISR */	\
	jz	1f ;							\
	addl	$(32 * index),%eax ;					\
	movq	%rsp, %rsi	;                                       \
	movl	%eax, %edi ;	/* pass the IRQ */			\
	call	lapic_handle_intr ;					\
1: ;									\
	MEXITCOUNT ;							\
	jmp	doreti

/*
 * Handle "spurious INTerrupts".
 * Notes:
 *  This is different than the "spurious INTerrupt" generated by an
 *   8259 PIC for missing INTs.  See the APIC documentation for details.
 *  This routine should NOT do an 'EOI' cycle.
 */
	.text
	SUPERALIGN_TEXT
IDTVEC(spuriousint)

	/* No EOI cycle used here */

	jmp	doreti_iret

	ISR_VEC(1, apic_isr1)
	ISR_VEC(2, apic_isr2)
	ISR_VEC(3, apic_isr3)
	ISR_VEC(4, apic_isr4)
	ISR_VEC(5, apic_isr5)
	ISR_VEC(6, apic_isr6)
	ISR_VEC(7, apic_isr7)

/*
 * Local APIC periodic timer handler.
 */
	.text
	SUPERALIGN_TEXT
IDTVEC(timerint)
	PUSH_FRAME
	FAKE_MCOUNT(TF_RIP(%rsp))
	movq	%rsp, %rdi
	call	lapic_handle_timer
	MEXITCOUNT
	jmp	doreti

/*
 * Local APIC CMCI handler.
 */
	.text
	SUPERALIGN_TEXT
IDTVEC(cmcint)
	PUSH_FRAME
	FAKE_MCOUNT(TF_RIP(%rsp))
	call	lapic_handle_cmc
	MEXITCOUNT
	jmp	doreti

/*
 * Local APIC error interrupt handler.
 */
	.text
	SUPERALIGN_TEXT
IDTVEC(errorint)
	PUSH_FRAME
	FAKE_MCOUNT(TF_RIP(%rsp))
	call	lapic_handle_error
	MEXITCOUNT
	jmp	doreti

#ifdef XENHVM
/*
 * Xen event channel upcall interrupt handler.
 * Only used when the hypervisor supports direct vector callbacks.
 */
	.text
	SUPERALIGN_TEXT
IDTVEC(xen_intr_upcall)
	PUSH_FRAME
	FAKE_MCOUNT(TF_RIP(%rsp))
	movq	%rsp, %rdi
	call	xen_intr_handle_upcall
	MEXITCOUNT
	jmp	doreti
#endif

#ifdef SMP
/*
 * Global address space TLB shootdown.
 */
	.text

#define	NAKE_INTR_CS	24

	SUPERALIGN_TEXT
global_invltlb:
	movq	%cr4,%rax
	andq	$~0x80,%rax	/* PGE */
	movq	%rax,%cr4
	orq	$0x80,%rax
	movq	%rax,%cr4
invltlb_ret_clear_pm_save:
	movq	smp_tlb_pmap,%rdx
	testq	%rdx,%rdx
	jz	invltlb_ret_rdx
	testb	$SEL_RPL_MASK,NAKE_INTR_CS(%rsp)
	jz	1f
	swapgs
1:
	movl	PCPU(CPUID),%eax
	jz	2f
	swapgs
2:
	LK btcl	%eax,PM_SAVE(%rdx)
	SUPERALIGN_TEXT
invltlb_ret_rdx:
	popq	%rdx
invltlb_ret_rax:
	movq	lapic, %rax
	movl	$0, LA_EOI(%rax)	/* End Of Interrupt to APIC */
	LK incl	smp_tlb_wait
	popq	%rax
	jmp	doreti_iret

	SUPERALIGN_TEXT
IDTVEC(invltlb_pcid)
#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
	PUSH_FRAME
	movl	PCPU(CPUID), %eax
#ifdef COUNT_XINVLTLB_HITS
	incl	xhits_gbl(,%rax,4)
#endif
#ifdef COUNT_IPIS
	movq	ipi_invltlb_counts(,%rax,8),%rax
	incq	(%rax)
#endif
	POP_FRAME
#endif

	pushq	%rax
	pushq	%rdx

	movq	%cr3,%rax

	movq	$smp_tlb_invpcid,%rdx
	cmpl	$0,(%rdx)
	je	global_invltlb
	cmpl	$-1,(%rdx)
	je	global_invltlb

	/*
	 * Only invalidate TLB for entries with current PCID.
	 */
	cmpl	$0,invpcid_works
	je	1f
	/* Use invpcid if available. */
	movl	$1,%eax /* INVPCID_CTX */
	/* invpcid (%rdx),%rax */
	.byte 0x66,0x0f,0x38,0x82,0x02
	jmp	invltlb_ret_clear_pm_save
1:
	/* Otherwise reload %cr3 twice. */
	movq	pcid_cr3,%rdx
	cmpq	%rax,%rdx
	je	2f
	movq	%rdx,%cr3	/* Invalidate, bit 63 is zero. */
	btsq	$63,%rax
2:
	movq	%rax,%cr3
	jmp	invltlb_ret_clear_pm_save

	SUPERALIGN_TEXT
IDTVEC(invltlb)
#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
	PUSH_FRAME
	movl	PCPU(CPUID), %eax
#ifdef COUNT_XINVLTLB_HITS
	incl	xhits_gbl(,%rax,4)
#endif
#ifdef COUNT_IPIS
	movq	ipi_invltlb_counts(,%rax,8),%rax
	incq	(%rax)
#endif
	POP_FRAME
#endif

	pushq	%rax
	movq	%cr3, %rax		/* invalidate the TLB */
	movq	%rax, %cr3
	jmp	invltlb_ret_rax

/*
 * Single page TLB shootdown
 */
	.text
	SUPERALIGN_TEXT
IDTVEC(invlpg_pcid)
#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
	PUSH_FRAME
	movl	PCPU(CPUID), %eax
#ifdef COUNT_XINVLTLB_HITS
	incl	xhits_pg(,%rax,4)
#endif
#ifdef COUNT_IPIS
	movq	ipi_invlpg_counts(,%rax,8),%rax
	incq	(%rax)
#endif
	POP_FRAME
#endif

	pushq	%rax
	pushq	%rdx
	movq	$smp_tlb_invpcid,%rdx
	cmpl	$0,invpcid_works
	jne	2f

	/* kernel pmap - use invlpg to invalidate global mapping */
	cmpl	$0,(%rdx)
	je	3f
	cmpl	$-1,(%rdx)
	je	global_invltlb

	/*
	 * PCID supported, but INVPCID is not.
	 * Temporarily switch to the target address space and do INVLPG.
	 */
	pushq	%rcx
	movq	%cr3,%rcx
	movq	pcid_cr3,%rax
	cmp	%rcx,%rax
	je	1f
	btsq	$63,%rax
	movq	%rax,%cr3
1:	movq	8(%rdx),%rax
	invlpg	(%rax)
	btsq	$63,%rcx
	movq	%rcx,%cr3
	popq	%rcx
	jmp	invltlb_ret_rdx

	/*
	 * Invalidate the TLB entry using INVPCID_ADDR.
	 */
2:
	xorl	%eax,%eax
/*	invpcid	(%rdx),%rax */
	.byte	0x66,0x0f,0x38,0x82,0x02
	jmp	invltlb_ret_rdx

	/*
	 * PCID is not supported or kernel pmap.
	 * Invalidate single page using INVLPG.
	 */
3:
	movq	8(%rdx),%rax
	invlpg	(%rax)
	jmp	invltlb_ret_rdx

	SUPERALIGN_TEXT
IDTVEC(invlpg)
#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
	PUSH_FRAME
	movl	PCPU(CPUID), %eax
#ifdef COUNT_XINVLTLB_HITS
	incl	xhits_pg(,%rax,4)
#endif
#ifdef COUNT_IPIS
	movq	ipi_invlpg_counts(,%rax,8),%rax
	incq	(%rax)
#endif
	POP_FRAME
#endif

	pushq	%rax
	movq	smp_tlb_invpcid+8,%rax
	invlpg	(%rax)			/* invalidate single page */
	jmp	invltlb_ret_rax

/*
 * Page range TLB shootdown.
 */
	.text
	SUPERALIGN_TEXT
IDTVEC(invlrng)
#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
	PUSH_FRAME
	movl	PCPU(CPUID), %eax
#ifdef COUNT_XINVLTLB_HITS
	incl	xhits_rng(,%rax,4)
#endif
#ifdef COUNT_IPIS
	movq	ipi_invlrng_counts(,%rax,8),%rax
	incq	(%rax)
#endif
	POP_FRAME
#endif

	pushq	%rax
	pushq	%rdx
	movq	$smp_tlb_invpcid,%rdx
	cmpl	$0,pmap_pcid_enabled
	je	invlrng_single_page

	/* kernel pmap - use invlpg to invalidate global mapping */
	cmpl	$0,(%rdx)
	je	invlrng_single_page
	cmpl	$-1,(%rdx)
	je	global_invltlb
	cmpl	$0,invpcid_works
	jne	invlrng_invpcid

	pushq	%rcx
	movq	%cr3,%rcx
	movq	pcid_cr3,%rax
	cmpq	%rcx,%rax
	je	1f
	btsq	$63,%rax
	movq	%rax,%cr3
1:
	movq	8(%rdx),%rdx
	movq	smp_tlb_addr2,%rax
2:
	invlpg	(%rdx)
	addq	$PAGE_SIZE,%rdx
	cmpq	%rax,%rdx
	jb	2b
	btsq	$63,%rcx
	movq	%rcx,%cr3
	popq	%rcx
	jmp	invltlb_ret_rdx

invlrng_invpcid:
	pushq	%rcx
	subq	$16,%rsp
	movq	(%rdx),%rcx
	movq	%rcx,(%rsp)
	movq	8(%rdx),%rax
	movq	%rax,8(%rsp)
	movq	smp_tlb_addr2,%rcx
	subq	%rax,%rcx
	shrq	$PAGE_SHIFT,%rcx
1:
//	invpcid	(%rdx),%rax
	.byte	0x66,0x0f,0x38,0x82,0x02
	addq	$PAGE_SIZE,8(%rsp)
	dec	%rcx
	jne	1b
	addq	$16,%rsp
	popq	%rcx
	jmp	invltlb_ret_rdx

invlrng_single_page:
	movq	8(%rdx),%rdx
	movq	smp_tlb_addr2,%rax
1:	invlpg	(%rdx)			/* invalidate single page */
	addq	$PAGE_SIZE,%rdx
	cmpq	%rax,%rdx
	jb	1b
	jmp	invltlb_ret_rdx

/*
 * Invalidate cache.
 */
	.text
	SUPERALIGN_TEXT
IDTVEC(invlcache)
#ifdef COUNT_IPIS
	PUSH_FRAME
	movl	PCPU(CPUID), %eax
	movq	ipi_invlcache_counts(,%rax,8),%rax
	incq	(%rax)
	POP_FRAME
#endif

	pushq	%rax
	wbinvd
	jmp	invltlb_ret_rax

/*
 * Handler for IPIs sent via the per-cpu IPI bitmap.
 */
	.text
	SUPERALIGN_TEXT
IDTVEC(ipi_intr_bitmap_handler)		
	PUSH_FRAME

	movq	lapic, %rdx
	movl	$0, LA_EOI(%rdx)	/* End Of Interrupt to APIC */
	
	FAKE_MCOUNT(TF_RIP(%rsp))

	call	ipi_bitmap_handler
	MEXITCOUNT
	jmp	doreti

/*
 * Executed by a CPU when it receives an IPI_STOP from another CPU.
 */
	.text
	SUPERALIGN_TEXT
IDTVEC(cpustop)
	PUSH_FRAME

	movq	lapic, %rax
	movl	$0, LA_EOI(%rax)	/* End Of Interrupt to APIC */

	call	cpustop_handler
	jmp	doreti

/*
 * Executed by a CPU when it receives an IPI_SUSPEND from another CPU.
 */
	.text
	SUPERALIGN_TEXT
IDTVEC(cpususpend)
	PUSH_FRAME

	call	cpususpend_handler
	movq	lapic, %rax
	movl	$0, LA_EOI(%rax)	/* End Of Interrupt to APIC */
	jmp	doreti

/*
 * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
 *
 * - Calls the generic rendezvous action function.
 */
	.text
	SUPERALIGN_TEXT
IDTVEC(rendezvous)
	PUSH_FRAME
#ifdef COUNT_IPIS
	movl	PCPU(CPUID), %eax
	movq	ipi_rendezvous_counts(,%rax,8), %rax
	incq	(%rax)
#endif
	call	smp_rendezvous_action
	movq	lapic, %rax
	movl	$0, LA_EOI(%rax)	/* End Of Interrupt to APIC */
	jmp	doreti
#endif /* SMP */