sys/i386/isa/apic_vector.s

   1 /*
   2  *      from: vector.s, 386BSD 0.1 unknown origin
   3  * $FreeBSD$
   4  */
   5
   6
   7 #include <machine/apic.h>
   8 #include <machine/smp.h>
   9
  10 #include "i386/isa/intr_machdep.h"
  11
  12 /* convert an absolute IRQ# into a bitmask */
  13 #define IRQ_BIT(irq_num)        (1 << (irq_num))
  14
  15 /* make an index into the IO APIC from the IRQ# */
  16 #define REDTBL_IDX(irq_num)     (0x10 + ((irq_num) * 2))
  17
  18 /*
  19  *
  20  */
  21 #define PUSH_FRAME                                                      \
  22         pushl   $0 ;            /* dummy error code */                  \
  23         pushl   $0 ;            /* dummy trap type */                   \
  24         pushal ;                                                        \
  25         pushl   %ds ;           /* save data and extra segments ... */  \
  26         pushl   %es ;                                                   \
  27         pushl   %fs
  28
  29 #define POP_FRAME                                                       \
  30         popl    %fs ;                                                   \
  31         popl    %es ;                                                   \
  32         popl    %ds ;                                                   \
  33         popal ;                                                         \
  34         addl    $4+4,%esp
  35
  36 /*
  37  * Macros for interrupt entry, call to handler, and exit.
  38  */
  39
  40 #define FAST_INTR(irq_num, vec_name)                                    \
  41         .text ;                                                         \
  42         SUPERALIGN_TEXT ;                                               \
  43 IDTVEC(vec_name) ;                                                      \
  44         PUSH_FRAME ;                                                    \
  45         movl    $KDSEL,%eax ;                                           \
  46         mov     %ax,%ds ;                                               \
  47         mov     %ax,%es ;                                               \
  48         movl    $KPSEL,%eax ;                                           \
  49         mov     %ax,%fs ;                                               \
  50         FAKE_MCOUNT(13*4(%esp)) ;                                       \
  51         movl    PCPU(CURPROC),%ebx ;                                    \
  52         incl    P_INTR_NESTING_LEVEL(%ebx) ;                            \
  53         pushl   intr_unit + (irq_num) * 4 ;                             \
  54         call    *intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \
  55         addl    $4, %esp ;                                              \
  56         movl    $0, lapic+LA_EOI ;                                      \
  57         lock ;                                                          \
  58         incl    cnt+V_INTR ;    /* book-keeping can wait */             \
  59         movl    intr_countp + (irq_num) * 4, %eax ;                     \
  60         lock ;                                                          \
  61         incl    (%eax) ;                                                \
  62         decl    P_INTR_NESTING_LEVEL(%ebx) ;                            \
  63         MEXITCOUNT ;                                                    \
  64         jmp     doreti
  65
  66 #define IOAPICADDR(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 8
  67 #define REDIRIDX(irq_num) CNAME(int_to_apicintpin) + 16 * (irq_num) + 12
  68
  69 #define MASK_IRQ(irq_num)                                               \
  70         IMASK_LOCK ;                            /* into critical reg */ \
  71         testl   $IRQ_BIT(irq_num), apic_imen ;                          \
  72         jne     7f ;                    /* masked, don't mask */        \
  73         orl     $IRQ_BIT(irq_num), apic_imen ;  /* set the mask bit */  \
  74         movl    IOAPICADDR(irq_num), %ecx ;     /* ioapic addr */       \
  75         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
  76         movl    %eax, (%ecx) ;                  /* write the index */   \
  77         movl    IOAPIC_WINDOW(%ecx), %eax ;     /* current value */     \
  78         orl     $IOART_INTMASK, %eax ;          /* set the mask */      \
  79         movl    %eax, IOAPIC_WINDOW(%ecx) ;     /* new value */         \
  80 7: ;                                            /* already masked */    \
  81         IMASK_UNLOCK
  82 /*
  83  * Test to see whether we are handling an edge or level triggered INT.
  84  *  Level-triggered INTs must still be masked as we don't clear the source,
  85  *  and the EOI cycle would cause redundant INTs to occur.
  86  */
  87 #define MASK_LEVEL_IRQ(irq_num)                                         \
  88         testl   $IRQ_BIT(irq_num), apic_pin_trigger ;                   \
  89         jz      9f ;                            /* edge, don't mask */  \
  90         MASK_IRQ(irq_num) ;                                             \
  91 9:
  92
  93
  94 #ifdef APIC_INTR_REORDER
  95 #define EOI_IRQ(irq_num)                                                \
  96         movl    apic_isrbit_location + 8 * (irq_num), %eax ;            \
  97         movl    (%eax), %eax ;                                          \
  98         testl   apic_isrbit_location + 4 + 8 * (irq_num), %eax ;        \
  99         jz      9f ;                            /* not active */        \
 100         movl    $0, lapic+LA_EOI ;                                      \
 101 9:
 102
 103 #else
 104 #define EOI_IRQ(irq_num)                                                \
 105         testl   $IRQ_BIT(irq_num), lapic+LA_ISR1;                       \
 106         jz      9f      ;                       /* not active */        \
 107         movl    $0, lapic+LA_EOI;                                       \
 108 9:
 109 #endif
 110
 111
 112 /*
 113  * Test to see if the source is currently masked, clear if so.
 114  */
 115 #define UNMASK_IRQ(irq_num)                                     \
 116         IMASK_LOCK ;                            /* into critical reg */ \
 117         testl   $IRQ_BIT(irq_num), _apic_imen ;                         \
 118         je      7f ;                    /* bit clear, not masked */     \
 119         andl    $~IRQ_BIT(irq_num), _apic_imen ;/* clear mask bit */    \
 120         movl    IOAPICADDR(irq_num), %ecx ;     /* ioapic addr */       \
 121         movl    REDIRIDX(irq_num), %eax ;       /* get the index */     \
 122         movl    %eax, (%ecx) ;                  /* write the index */   \
 123         movl    IOAPIC_WINDOW(%ecx), %eax ;     /* current value */     \
 124         andl    $~IOART_INTMASK, %eax ;         /* clear the mask */    \
 125         movl    %eax, IOAPIC_WINDOW(%ecx) ;     /* new value */         \
 126 7: ;                                            /* already unmasked */  \
 127         IMASK_UNLOCK
 128
 129 /*
 130  * Slow, threaded interrupts.
 131  *
 132  * XXX Most of the parameters here are obsolete.  Fix this when we're
 133  * done.
 134  * XXX we really shouldn't return via doreti if we just schedule the
 135  * interrupt handler and don't run anything.  We could just do an
 136  * iret.  FIXME.
 137  */
 138 #define INTR(irq_num, vec_name, maybe_extra_ipending)                   \
 139         .text ;                                                         \
 140         SUPERALIGN_TEXT ;                                               \
 141 /* _XintrNN: entry point used by IDT/HWIs via _vec[]. */                \
 142 IDTVEC(vec_name) ;                                                      \
 143         PUSH_FRAME ;                                                    \
 144         movl    $KDSEL, %eax ;  /* reload with kernel's data segment */ \
 145         mov     %ax, %ds ;                                              \
 146         mov     %ax, %es ;                                              \
 147         movl    $KPSEL, %eax ;                                          \
 148         mov     %ax, %fs ;                                              \
 149 ;                                                                       \
 150         maybe_extra_ipending ;                                          \
 151 ;                                                                       \
 152         MASK_LEVEL_IRQ(irq_num) ;                                       \
 153         EOI_IRQ(irq_num) ;                                              \
 154 0: ;                                                                    \
 155         movl    PCPU(CURPROC),%ebx ;                                    \
 156         incl    P_INTR_NESTING_LEVEL(%ebx) ;                            \
 157 ;                                                                       \
 158   /* entry point used by doreti_unpend for HWIs. */                     \
 159 __CONCAT(Xresume,irq_num): ;                                            \
 160         FAKE_MCOUNT(13*4(%esp)) ;               /* XXX avoid dbl cnt */ \
 161         pushl   $irq_num;                       /* pass the IRQ */      \
 162         sti ;                                                           \
 163         call    sched_ithd ;                                            \
 164         addl    $4, %esp ;              /* discard the parameter */     \
 165 ;                                                                       \
 166         decl    P_INTR_NESTING_LEVEL(%ebx) ;                            \
 167         MEXITCOUNT ;                                                    \
 168         jmp     doreti
 169
 170 /*
 171  * Handle "spurious INTerrupts".
 172  * Notes:
 173  *  This is different than the "spurious INTerrupt" generated by an
 174  *   8259 PIC for missing INTs.  See the APIC documentation for details.
 175  *  This routine should NOT do an 'EOI' cycle.
 176  */
 177         .text
 178         SUPERALIGN_TEXT
 179         .globl Xspuriousint
 180 Xspuriousint:
 181
 182         /* No EOI cycle used here */
 183
 184         iret
 185
 186
 187 /*
 188  * Handle TLB shootdowns.
 189  */
 190         .text
 191         SUPERALIGN_TEXT
 192         .globl  Xinvltlb
 193 Xinvltlb:
 194         pushl   %eax
 195
 196 #ifdef COUNT_XINVLTLB_HITS
 197         pushl   %fs
 198         movl    $KPSEL, %eax
 199         mov     %ax, %fs
 200         movl    PCPU(CPUID), %eax
 201         popl    %fs
 202         ss
 203         incl    _xhits(,%eax,4)
 204 #endif /* COUNT_XINVLTLB_HITS */
 205
 206         movl    %cr3, %eax              /* invalidate the TLB */
 207         movl    %eax, %cr3
 208
 209         ss                              /* stack segment, avoid %ds load */
 210         movl    $0, lapic+LA_EOI        /* End Of Interrupt to APIC */
 211
 212         popl    %eax
 213         iret
 214
 215
 216 #ifdef BETTER_CLOCK
 217
 218 /*
 219  * Executed by a CPU when it receives an Xcpucheckstate IPI from another CPU,
 220  *
 221  *  - Stores current cpu state in checkstate_cpustate[cpuid]
 222  *      0 == user, 1 == sys, 2 == intr
 223  *  - Stores current process in checkstate_curproc[cpuid]
 224  *
 225  *  - Signals its receipt by setting bit cpuid in checkstate_probed_cpus.
 226  *
 227  * stack: 0->ds, 4->fs, 8->ebx, 12->eax, 16->eip, 20->cs, 24->eflags
 228  */
 229
 230         .text
 231         SUPERALIGN_TEXT
 232         .globl Xcpucheckstate
 233         .globl checkstate_cpustate
 234         .globl checkstate_curproc
 235         .globl checkstate_pc
 236 Xcpucheckstate:
 237         pushl   %eax
 238         pushl   %ebx
 239         pushl   %ds                     /* save current data segment */
 240         pushl   %fs
 241
 242         movl    $KDSEL, %eax
 243         mov     %ax, %ds                /* use KERNEL data segment */
 244         movl    $KPSEL, %eax
 245         mov     %ax, %fs
 246
 247         movl    $0, lapic+LA_EOI        /* End Of Interrupt to APIC */
 248
 249         movl    $0, %ebx
 250         movl    20(%esp), %eax
 251         andl    $3, %eax
 252         cmpl    $3, %eax
 253         je      1f
 254         testl   $PSL_VM, 24(%esp)
 255         jne     1f
 256         incl    %ebx                    /* system or interrupt */
 257 1:
 258         movl    PCPU(CPUID), %eax
 259         movl    %ebx, checkstate_cpustate(,%eax,4)
 260         movl    PCPU(CURPROC), %ebx
 261         movl    %ebx, checkstate_curproc(,%eax,4)
 262
 263         movl    16(%esp), %ebx
 264         movl    %ebx, checkstate_pc(,%eax,4)
 265
 266         lock                            /* checkstate_probed_cpus |= (1<<id) */
 267         btsl    %eax, checkstate_probed_cpus
 268
 269         popl    %fs
 270         popl    %ds                     /* restore previous data segment */
 271         popl    %ebx
 272         popl    %eax
 273         iret
 274
 275 #endif /* BETTER_CLOCK */
 276
 277 /*
 278  * Executed by a CPU when it receives an Xcpuast IPI from another CPU,
 279  *
 280  *  - Signals its receipt by clearing bit cpuid in checkstate_need_ast.
 281  *
 282  *  - We need a better method of triggering asts on other cpus.
 283  */
 284
 285         .text
 286         SUPERALIGN_TEXT
 287         .globl Xcpuast
 288 Xcpuast:
 289         PUSH_FRAME
 290         movl    $KDSEL, %eax
 291         mov     %ax, %ds                /* use KERNEL data segment */
 292         mov     %ax, %es
 293         movl    $KPSEL, %eax
 294         mov     %ax, %fs
 295
 296         movl    PCPU(CPUID), %eax
 297         lock                            /* checkstate_need_ast &= ~(1<<id) */
 298         btrl    %eax, checkstate_need_ast
 299         movl    $0, lapic+LA_EOI        /* End Of Interrupt to APIC */
 300
 301         lock
 302         btsl    %eax, checkstate_pending_ast
 303         jc      1f
 304
 305         FAKE_MCOUNT(13*4(%esp))
 306
 307         MTX_LOCK_SPIN(sched_lock, 0)
 308         movl    PCPU(CURPROC),%ebx
 309         orl     $PS_ASTPENDING, P_SFLAG(%ebx)
 310
 311         movl    PCPU(CPUID), %eax
 312         lock
 313         btrl    %eax, checkstate_pending_ast
 314         lock
 315         btrl    %eax, CNAME(resched_cpus)
 316         jnc     2f
 317         orl     $PS_NEEDRESCHED, P_SFLAG(%ebx)
 318         lock
 319         incl    CNAME(want_resched_cnt)
 320 2:
 321         MTX_UNLOCK_SPIN(sched_lock)
 322         lock
 323         incl    CNAME(cpuast_cnt)
 324         MEXITCOUNT
 325         jmp     doreti
 326 1:
 327         /* We are already in the process of delivering an ast for this CPU */
 328         POP_FRAME
 329         iret
 330
 331 /*
 332  * Executed by a CPU when it receives an Xcpustop IPI from another CPU,
 333  *
 334  *  - Signals its receipt.
 335  *  - Waits for permission to restart.
 336  *  - Signals its restart.
 337  */
 338
 339         .text
 340         SUPERALIGN_TEXT
 341         .globl Xcpustop
 342 Xcpustop:
 343         pushl   %ebp
 344         movl    %esp, %ebp
 345         pushl   %eax
 346         pushl   %ecx
 347         pushl   %edx
 348         pushl   %ds                     /* save current data segment */
 349         pushl   %fs
 350
 351         movl    $KDSEL, %eax
 352         mov     %ax, %ds                /* use KERNEL data segment */
 353         movl    $KPSEL, %eax
 354         mov     %ax, %fs
 355
 356         movl    $0, lapic+LA_EOI        /* End Of Interrupt to APIC */
 357
 358         movl    PCPU(CPUID), %eax
 359         imull   $PCB_SIZE, %eax
 360         leal    CNAME(stoppcbs)(%eax), %eax
 361         pushl   %eax
 362         call    CNAME(savectx)          /* Save process context */
 363         addl    $4, %esp
 364
 365
 366         movl    PCPU(CPUID), %eax
 367
 368         lock
 369         btsl    %eax, stopped_cpus      /* stopped_cpus |= (1<<id) */
 370 1:
 371         btl     %eax, started_cpus      /* while (!(started_cpus & (1<<id))) */
 372         jnc     1b
 373
 374         lock
 375         btrl    %eax, started_cpus      /* started_cpus &= ~(1<<id) */
 376         lock
 377         btrl    %eax, stopped_cpus      /* stopped_cpus &= ~(1<<id) */
 378
 379         test    %eax, %eax
 380         jnz     2f
 381
 382         movl    CNAME(cpustop_restartfunc), %eax
 383         test    %eax, %eax
 384         jz      2f
 385         movl    $0, CNAME(cpustop_restartfunc)  /* One-shot */
 386
 387         call    *%eax
 388 2:
 389         popl    %fs
 390         popl    %ds                     /* restore previous data segment */
 391         popl    %edx
 392         popl    %ecx
 393         popl    %eax
 394         movl    %ebp, %esp
 395         popl    %ebp
 396         iret
 397
 398
 399 MCOUNT_LABEL(bintr)
 400         FAST_INTR(0,fastintr0)
 401         FAST_INTR(1,fastintr1)
 402         FAST_INTR(2,fastintr2)
 403         FAST_INTR(3,fastintr3)
 404         FAST_INTR(4,fastintr4)
 405         FAST_INTR(5,fastintr5)
 406         FAST_INTR(6,fastintr6)
 407         FAST_INTR(7,fastintr7)
 408         FAST_INTR(8,fastintr8)
 409         FAST_INTR(9,fastintr9)
 410         FAST_INTR(10,fastintr10)
 411         FAST_INTR(11,fastintr11)
 412         FAST_INTR(12,fastintr12)
 413         FAST_INTR(13,fastintr13)
 414         FAST_INTR(14,fastintr14)
 415         FAST_INTR(15,fastintr15)
 416         FAST_INTR(16,fastintr16)
 417         FAST_INTR(17,fastintr17)
 418         FAST_INTR(18,fastintr18)
 419         FAST_INTR(19,fastintr19)
 420         FAST_INTR(20,fastintr20)
 421         FAST_INTR(21,fastintr21)
 422         FAST_INTR(22,fastintr22)
 423         FAST_INTR(23,fastintr23)
 424         FAST_INTR(24,fastintr24)
 425         FAST_INTR(25,fastintr25)
 426         FAST_INTR(26,fastintr26)
 427         FAST_INTR(27,fastintr27)
 428         FAST_INTR(28,fastintr28)
 429         FAST_INTR(29,fastintr29)
 430         FAST_INTR(30,fastintr30)
 431         FAST_INTR(31,fastintr31)
 432 #define CLKINTR_PENDING movl $1,CNAME(clkintr_pending)
 433 /* Threaded interrupts */
 434         INTR(0,intr0, CLKINTR_PENDING)
 435         INTR(1,intr1,)
 436         INTR(2,intr2,)
 437         INTR(3,intr3,)
 438         INTR(4,intr4,)
 439         INTR(5,intr5,)
 440         INTR(6,intr6,)
 441         INTR(7,intr7,)
 442         INTR(8,intr8,)
 443         INTR(9,intr9,)
 444         INTR(10,intr10,)
 445         INTR(11,intr11,)
 446         INTR(12,intr12,)
 447         INTR(13,intr13,)
 448         INTR(14,intr14,)
 449         INTR(15,intr15,)
 450         INTR(16,intr16,)
 451         INTR(17,intr17,)
 452         INTR(18,intr18,)
 453         INTR(19,intr19,)
 454         INTR(20,intr20,)
 455         INTR(21,intr21,)
 456         INTR(22,intr22,)
 457         INTR(23,intr23,)
 458         INTR(24,intr24,)
 459         INTR(25,intr25,)
 460         INTR(26,intr26,)
 461         INTR(27,intr27,)
 462         INTR(28,intr28,)
 463         INTR(29,intr29,)
 464         INTR(30,intr30,)
 465         INTR(31,intr31,)
 466 MCOUNT_LABEL(eintr)
 467
 468 /*
 469  * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
 470  *
 471  * - Calls the generic rendezvous action function.
 472  */
 473         .text
 474         SUPERALIGN_TEXT
 475         .globl  Xrendezvous
 476 Xrendezvous:
 477         PUSH_FRAME
 478         movl    $KDSEL, %eax
 479         mov     %ax, %ds                /* use KERNEL data segment */
 480         mov     %ax, %es
 481         movl    $KPSEL, %eax
 482         mov     %ax, %fs
 483
 484         call    smp_rendezvous_action
 485
 486         movl    $0, lapic+LA_EOI        /* End Of Interrupt to APIC */
 487         POP_FRAME
 488         iret
 489
 490
 491         .data
 492
 493 #ifdef COUNT_XINVLTLB_HITS
 494         .globl  _xhits
 495 _xhits:
 496         .space  (NCPU * 4), 0
 497 #endif /* COUNT_XINVLTLB_HITS */
 498
 499 /* variables used by stop_cpus()/restart_cpus()/Xcpustop */
 500         .globl stopped_cpus, started_cpus
 501 stopped_cpus:
 502         .long   0
 503 started_cpus:
 504         .long   0
 505
 506 #ifdef BETTER_CLOCK
 507         .globl checkstate_probed_cpus
 508 checkstate_probed_cpus:
 509         .long   0
 510 #endif /* BETTER_CLOCK */
 511         .globl checkstate_need_ast
 512 checkstate_need_ast:
 513         .long   0
 514 checkstate_pending_ast:
 515         .long   0
 516         .globl CNAME(resched_cpus)
 517         .globl CNAME(want_resched_cnt)
 518         .globl CNAME(cpuast_cnt)
 519         .globl CNAME(cpustop_restartfunc)
 520 CNAME(resched_cpus):
 521         .long 0
 522 CNAME(want_resched_cnt):
 523         .long 0
 524 CNAME(cpuast_cnt):
 525         .long 0
 526 CNAME(cpustop_restartfunc):
 527         .long 0
 528
 529         .globl  apic_pin_trigger
 530 apic_pin_trigger:
 531         .long   0
 532
 533         .text