sys/i386/i386/apic_vector.s

   1 /*
   2  *      from: vector.s, 386BSD 0.1 unknown origin
   3  * $FreeBSD$
   4  */
   5
   6
   7 #include <machine/apic.h>
   8 #include <machine/smp.h>
   9
  10 #include "i386/isa/intr_machdep.h"
  11
  12 /* convert an absolute IRQ# into a bitmask */
  13 #define IRQ_BIT(irq_num)        (1 << (irq_num))
  14
  15 /* make an index into the IO APIC from the IRQ# */
  16 #define REDTBL_IDX(irq_num)     (0x10 + ((irq_num) * 2))
  17
  18 /*
  19  *
  20  */
  21 #define PUSH_FRAME                                                      \
  22         pushl   $0 ;            /* dummy error code */                  \
  23         pushl   $0 ;            /* dummy trap type */                   \
  24         pushal ;                                                        \
  25         pushl   %ds ;           /* save data and extra segments ... */  \
  26         pushl   %es ;                                                   \
  27         pushl   %fs
  28
  29 #define POP_FRAME                                                       \
  30         popl    %fs ;                                                   \
  31         popl    %es ;                                                   \
  32         popl    %ds ;                                                   \
  33         popal ;                                                         \
  34         addl    $4+4,%esp
  35
  36 /*
  37  * Macros for interrupt entry, call to handler, and exit.
  38  */
  39
  40 #define FAST_INTR(irq_num, vec_name)                                    \
  41         .text ;                                                         \
  42         SUPERALIGN_TEXT ;                                               \
  43 IDTVEC(vec_name) ;                                                      \
  44         PUSH_FRAME ;                                                    \
  45         movl    $KDSEL,%eax ;                                           \
  46         mov     %ax,%ds ;                                               \
  47         mov     %ax,%es ;                                               \
  48         movl    $KPSEL,%eax ;                                           \
  49         mov     %ax,%fs ;                                               \
  50         FAKE_MCOUNT(13*4(%esp)) ;                                       \
  51         movl    PCPU(CURPROC),%ebx ;                                    \
  52         incl    P_INTR_NESTING_LEVEL(%ebx) ;                            \
  53         pushl   intr_unit + (irq_num) * 4 ;                             \
  54         call    *intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \
  55         addl    $4, %esp ;                                              \
  56         movl    $0, lapic+LA_EOI ;                                      \
  57         lock ;                                                          \
  58         incl    cnt+V_INTR ;    /* book-keeping can wait */             \
  59         movl    intr_countp + (irq_num) * 4, %eax ;                     \
  60         lock ;                                                          \
  61         incl    (%eax) ;                                                \
  62         decl    P_INTR_NESTING_LEVEL(%ebx) ;                            \
  63         MEXITCOUNT ;                                                    \
  64         jmp     doreti
  65
  66 #define IOAPICADDR(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 8
  67 #define REDIRIDX(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 12
  68
  69 #define MASK_IRQ(irq_num)                                               \
  70         IMASK_LOCK ;                            /* into critical reg */ \
  71         testl   $IRQ_BIT(irq_num), apic_imen ;                          \
  72         jne     7f ;                    /* masked, don't mask */        \
  73         orl     $IRQ_BIT(irq_num), apic_imen ;  /* set the mask bit */  \
  74         movl    IOAPICADDR(irq_num), %ecx ;     /* ioapic addr */       \
  75         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
  76         movl    %eax, (%ecx) ;                  /* write the index */   \
  77         movl    IOAPIC_WINDOW(%ecx), %eax ;     /* current value */     \
  78         orl     $IOART_INTMASK, %eax ;          /* set the mask */      \
  79         movl    %eax, IOAPIC_WINDOW(%ecx) ;     /* new value */         \
  80 7: ;                                            /* already masked */    \
  81         IMASK_UNLOCK
  82 /*
  83  * Test to see whether we are handling an edge or level triggered INT.
  84  *  Level-triggered INTs must still be masked as we don't clear the source,
  85  *  and the EOI cycle would cause redundant INTs to occur.
  86  */
  87 #define MASK_LEVEL_IRQ(irq_num)                                         \
  88         testl   $IRQ_BIT(irq_num), apic_pin_trigger ;                   \
  89         jz      9f ;                            /* edge, don't mask */  \
  90         MASK_IRQ(irq_num) ;                                             \
  91 9:
  92
  93
  94 #ifdef APIC_INTR_REORDER
  95 #define EOI_IRQ(irq_num)                                                \
  96         movl    apic_isrbit_location + 8 * (irq_num), %eax ;            \
  97         movl    (%eax), %eax ;                                          \
  98         testl   apic_isrbit_location + 4 + 8 * (irq_num), %eax ;        \
  99         jz      9f ;                            /* not active */        \
 100         movl    $0, lapic+LA_EOI ;                                      \
 101 9:
 102
 103 #else
 104 #define EOI_IRQ(irq_num)                                                \
 105         testl   $IRQ_BIT(irq_num), lapic+LA_ISR1;                       \
 106         jz      9f      ;                       /* not active */        \
 107         movl    $0, lapic+LA_EOI;                                       \
 108 9:
 109 #endif
 110
 111
 112 /*
 113  * Test to see if the source is currently masked, clear if so.
 114  */
 115 #define UNMASK_IRQ(irq_num)                                     \
 116         IMASK_LOCK ;                            /* into critical reg */ \
 117         testl   $IRQ_BIT(irq_num), _apic_imen ;                         \
 118         je      7f ;                    /* bit clear, not masked */     \
 119         andl    $~IRQ_BIT(irq_num), _apic_imen ;/* clear mask bit */    \
 120         movl    IOAPICADDR(irq_num), %ecx ;     /* ioapic addr */       \
 121         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
 122         movl    %eax, (%ecx) ;                  /* write the index */   \
 123         movl    IOAPIC_WINDOW(%ecx), %eax ;     /* current value */     \
 124         andl    $~IOART_INTMASK, %eax ;         /* clear the mask */    \
 125         movl    %eax, IOAPIC_WINDOW(%ecx) ;     /* new value */         \
 126 7: ;                                            /* already unmasked */  \
 127         IMASK_UNLOCK
 128
 129 /*
 130  * Slow, threaded interrupts.
 131  *
 132  * XXX Most of the parameters here are obsolete.  Fix this when we're
 133  * done.
 134  * XXX we really shouldn't return via doreti if we just schedule the
 135  * interrupt handler and don't run anything.  We could just do an
 136  * iret.  FIXME.
 137  */
 138 #define INTR(irq_num, vec_name, maybe_extra_ipending)                   \
 139         .text ;                                                         \
 140         SUPERALIGN_TEXT ;                                               \
 141 /* _XintrNN: entry point used by IDT/HWIs via _vec[]. */                \
 142 IDTVEC(vec_name) ;                                                      \
 143         PUSH_FRAME ;                                                    \
 144         movl    $KDSEL, %eax ;  /* reload with kernel's data segment */ \
 145         mov     %ax, %ds ;                                              \
 146         mov     %ax, %es ;                                              \
 147         movl    $KPSEL, %eax ;                                          \
 148         mov     %ax, %fs ;                                              \
 149 ;                                                                       \
 150         maybe_extra_ipending ;                                          \
 151 ;                                                                       \
 152         MASK_LEVEL_IRQ(irq_num) ;                                       \
 153         EOI_IRQ(irq_num) ;                                              \
 154 0: ;                                                                    \
 155         movl    PCPU(CURPROC),%ebx ;                                    \
 156         incl    P_INTR_NESTING_LEVEL(%ebx) ;                            \
 157 ;                                                                       \
 158   /* entry point used by doreti_unpend for HWIs. */                     \
 159 __CONCAT(Xresume,irq_num): ;                                            \
 160         FAKE_MCOUNT(13*4(%esp)) ;               /* XXX avoid dbl cnt */ \
 161         pushl   $irq_num;                       /* pass the IRQ */      \
 162         call    sched_ithd ;                                            \
 163         addl    $4, %esp ;              /* discard the parameter */     \
 164 ;                                                                       \
 165         decl    P_INTR_NESTING_LEVEL(%ebx) ;                            \
 166         MEXITCOUNT ;                                                    \
 167         jmp     doreti
 168
 169 /*
 170  * Handle "spurious INTerrupts".
 171  * Notes:
 172  *  This is different than the "spurious INTerrupt" generated by an
 173  *   8259 PIC for missing INTs.  See the APIC documentation for details.
 174  *  This routine should NOT do an 'EOI' cycle.
 175  */
 176         .text
 177         SUPERALIGN_TEXT
 178         .globl Xspuriousint
 179 Xspuriousint:
 180
 181         /* No EOI cycle used here */
 182
 183         iret
 184
 185 /*
 186  * Handle TLB shootdowns.
 187  */
 188         .text
 189         SUPERALIGN_TEXT
 190         .globl  Xinvltlb
 191 Xinvltlb:
 192         pushl   %eax
 193
 194 #ifdef COUNT_XINVLTLB_HITS
 195         pushl   %fs
 196         movl    $KPSEL, %eax
 197         mov     %ax, %fs
 198         movl    PCPU(CPUID), %eax
 199         popl    %fs
 200         ss
 201         incl    _xhits(,%eax,4)
 202 #endif /* COUNT_XINVLTLB_HITS */
 203
 204         movl    %cr3, %eax              /* invalidate the TLB */
 205         movl    %eax, %cr3
 206
 207         ss                              /* stack segment, avoid %ds load */
 208         movl    $0, lapic+LA_EOI        /* End Of Interrupt to APIC */
 209
 210         popl    %eax
 211         iret
 212
 213 /*
 214  * Forward hardclock to another CPU.  Pushes a trapframe and calls
 215  * forwarded_hardclock().
 216  */
 217         .text
 218         SUPERALIGN_TEXT
 219         .globl Xhardclock
 220 Xhardclock:
 221         PUSH_FRAME
 222         movl    $KDSEL, %eax    /* reload with kernel's data segment */
 223         mov     %ax, %ds
 224         mov     %ax, %es
 225         movl    $KPSEL, %eax
 226         mov     %ax, %fs
 227
 228         movl    $0, lapic+LA_EOI        /* End Of Interrupt to APIC */
 229
 230         movl    PCPU(CURPROC),%ebx
 231         incl    P_INTR_NESTING_LEVEL(%ebx)
 232         call    forwarded_hardclock
 233         decl    P_INTR_NESTING_LEVEL(%ebx)
 234         MEXITCOUNT
 235         jmp     doreti
 236
 237 /*
 238  * Forward statclock to another CPU.  Pushes a trapframe and calls
 239  * forwarded_statclock().
 240  */
 241         .text
 242         SUPERALIGN_TEXT
 243         .globl Xstatclock
 244 Xstatclock:
 245         PUSH_FRAME
 246         movl    $KDSEL, %eax    /* reload with kernel's data segment */
 247         mov     %ax, %ds
 248         mov     %ax, %es
 249         movl    $KPSEL, %eax
 250         mov     %ax, %fs
 251
 252         movl    $0, lapic+LA_EOI        /* End Of Interrupt to APIC */
 253
 254         FAKE_MCOUNT(13*4(%esp))
 255         movl    PCPU(CURPROC),%ebx
 256         incl    P_INTR_NESTING_LEVEL(%ebx)
 257         call    forwarded_statclock
 258         decl    P_INTR_NESTING_LEVEL(%ebx)
 259         MEXITCOUNT
 260         jmp     doreti
 261
 262 /*
 263  * Executed by a CPU when it receives an Xcpuast IPI from another CPU,
 264  *
 265  * The other CPU has already executed aston() or need_resched() on our
 266  * current process, so we simply need to ack the interrupt and return
 267  * via doreti to run ast().
 268  */
 269
 270         .text
 271         SUPERALIGN_TEXT
 272         .globl Xcpuast
 273 Xcpuast:
 274         PUSH_FRAME
 275         movl    $KDSEL, %eax
 276         mov     %ax, %ds                /* use KERNEL data segment */
 277         mov     %ax, %es
 278         movl    $KPSEL, %eax
 279         mov     %ax, %fs
 280
 281         movl    $0, lapic+LA_EOI        /* End Of Interrupt to APIC */
 282
 283         FAKE_MCOUNT(13*4(%esp))
 284
 285         MEXITCOUNT
 286         jmp     doreti
 287
 288 /*
 289  * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
 290  *
 291  *  - Signals its receipt.
 292  *  - Waits for permission to restart.
 293  *  - Signals its restart.
 294  */
 295         .text
 296         SUPERALIGN_TEXT
 297         .globl Xcpustop
 298 Xcpustop:
 299         pushl   %ebp
 300         movl    %esp, %ebp
 301         pushl   %eax
 302         pushl   %ecx
 303         pushl   %edx
 304         pushl   %ds                     /* save current data segment */
 305         pushl   %fs
 306
 307         movl    $KDSEL, %eax
 308         mov     %ax, %ds                /* use KERNEL data segment */
 309         movl    $KPSEL, %eax
 310         mov     %ax, %fs
 311
 312         movl    $0, lapic+LA_EOI        /* End Of Interrupt to APIC */
 313
 314         movl    PCPU(CPUID), %eax
 315         imull   $PCB_SIZE, %eax
 316         leal    CNAME(stoppcbs)(%eax), %eax
 317         pushl   %eax
 318         call    CNAME(savectx)          /* Save process context */
 319         addl    $4, %esp
 320
 321         movl    PCPU(CPUID), %eax
 322
 323         lock
 324         btsl    %eax, CNAME(stopped_cpus) /* stopped_cpus |= (1<<id) */
 325 1:
 326         btl     %eax, CNAME(started_cpus) /* while (!(started_cpus & (1<<id))) */
 327         jnc     1b
 328
 329         lock
 330         btrl    %eax, CNAME(started_cpus) /* started_cpus &= ~(1<<id) */
 331         lock
 332         btrl    %eax, CNAME(stopped_cpus) /* stopped_cpus &= ~(1<<id) */
 333
 334         test    %eax, %eax
 335         jnz     2f
 336
 337         movl    CNAME(cpustop_restartfunc), %eax
 338         test    %eax, %eax
 339         jz      2f
 340         movl    $0, CNAME(cpustop_restartfunc)  /* One-shot */
 341
 342         call    *%eax
 343 2:
 344         popl    %fs
 345         popl    %ds                     /* restore previous data segment */
 346         popl    %edx
 347         popl    %ecx
 348         popl    %eax
 349         movl    %ebp, %esp
 350         popl    %ebp
 351         iret
 352
 353
 354 MCOUNT_LABEL(bintr)
 355         FAST_INTR(0,fastintr0)
 356         FAST_INTR(1,fastintr1)
 357         FAST_INTR(2,fastintr2)
 358         FAST_INTR(3,fastintr3)
 359         FAST_INTR(4,fastintr4)
 360         FAST_INTR(5,fastintr5)
 361         FAST_INTR(6,fastintr6)
 362         FAST_INTR(7,fastintr7)
 363         FAST_INTR(8,fastintr8)
 364         FAST_INTR(9,fastintr9)
 365         FAST_INTR(10,fastintr10)
 366         FAST_INTR(11,fastintr11)
 367         FAST_INTR(12,fastintr12)
 368         FAST_INTR(13,fastintr13)
 369         FAST_INTR(14,fastintr14)
 370         FAST_INTR(15,fastintr15)
 371         FAST_INTR(16,fastintr16)
 372         FAST_INTR(17,fastintr17)
 373         FAST_INTR(18,fastintr18)
 374         FAST_INTR(19,fastintr19)
 375         FAST_INTR(20,fastintr20)
 376         FAST_INTR(21,fastintr21)
 377         FAST_INTR(22,fastintr22)
 378         FAST_INTR(23,fastintr23)
 379         FAST_INTR(24,fastintr24)
 380         FAST_INTR(25,fastintr25)
 381         FAST_INTR(26,fastintr26)
 382         FAST_INTR(27,fastintr27)
 383         FAST_INTR(28,fastintr28)
 384         FAST_INTR(29,fastintr29)
 385         FAST_INTR(30,fastintr30)
 386         FAST_INTR(31,fastintr31)
 387 #define CLKINTR_PENDING movl $1,CNAME(clkintr_pending)
 388 /* Threaded interrupts */
 389         INTR(0,intr0, CLKINTR_PENDING)
 390         INTR(1,intr1,)
 391         INTR(2,intr2,)
 392         INTR(3,intr3,)
 393         INTR(4,intr4,)
 394         INTR(5,intr5,)
 395         INTR(6,intr6,)
 396         INTR(7,intr7,)
 397         INTR(8,intr8,)
 398         INTR(9,intr9,)
 399         INTR(10,intr10,)
 400         INTR(11,intr11,)
 401         INTR(12,intr12,)
 402         INTR(13,intr13,)
 403         INTR(14,intr14,)
 404         INTR(15,intr15,)
 405         INTR(16,intr16,)
 406         INTR(17,intr17,)
 407         INTR(18,intr18,)
 408         INTR(19,intr19,)
 409         INTR(20,intr20,)
 410         INTR(21,intr21,)
 411         INTR(22,intr22,)
 412         INTR(23,intr23,)
 413         INTR(24,intr24,)
 414         INTR(25,intr25,)
 415         INTR(26,intr26,)
 416         INTR(27,intr27,)
 417         INTR(28,intr28,)
 418         INTR(29,intr29,)
 419         INTR(30,intr30,)
 420         INTR(31,intr31,)
 421 MCOUNT_LABEL(eintr)
 422
 423 /*
 424  * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
 425  *
 426  * - Calls the generic rendezvous action function.
 427  */
 428         .text
 429         SUPERALIGN_TEXT
 430         .globl  Xrendezvous
 431 Xrendezvous:
 432         PUSH_FRAME
 433         movl    $KDSEL, %eax
 434         mov     %ax, %ds                /* use KERNEL data segment */
 435         mov     %ax, %es
 436         movl    $KPSEL, %eax
 437         mov     %ax, %fs
 438
 439         call    smp_rendezvous_action
 440
 441         movl    $0, lapic+LA_EOI        /* End Of Interrupt to APIC */
 442         POP_FRAME
 443         iret
 444
 445
 446         .data
 447
 448 #ifdef COUNT_XINVLTLB_HITS
 449         .globl  _xhits
 450 _xhits:
 451         .space  (NCPU * 4), 0
 452 #endif /* COUNT_XINVLTLB_HITS */
 453
 454         .globl  apic_pin_trigger
 455 apic_pin_trigger:
 456         .long   0
 457
 458         .text