sys/ia64/ia64/interrupt.c

   1 /* $FreeBSD$ */
   2 /* $NetBSD: interrupt.c,v 1.23 1998/02/24 07:38:01 thorpej Exp $ */
   3
   4 /*-
   5  * Copyright (c) 1994, 1995, 1996 Carnegie-Mellon University.
   6  * All rights reserved.
   7  *
   8  * Authors: Keith Bostic, Chris G. Demetriou
   9  *
  10  * Permission to use, copy, modify and distribute this software and
  11  * its documentation is hereby granted, provided that both the copyright
  12  * notice and this permission notice appear in all copies of the
  13  * software, derivative works or modified versions, and any portions
  14  * thereof, and that both notices appear in supporting documentation.
  15  *
  16  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  17  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
  18  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  19  *
  20  * Carnegie Mellon requests users of this software to return to
  21  *
  22  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  23  *  School of Computer Science
  24  *  Carnegie Mellon University
  25  *  Pittsburgh PA 15213-3890
  26  *
  27  * any improvements or extensions that they make and grant Carnegie the
  28  * rights to redistribute these changes.
  29  */
  30 /*-
  31  * Additional Copyright (c) 1997 by Matthew Jacob for NASA/Ames Research Center.
  32  * Redistribute and modify at will, leaving only this additional copyright
  33  * notice.
  34  */
  35
  36 #include "opt_ddb.h"
  37
  38 #include <sys/cdefs.h>                  /* RCS ID & Copyright macro defns */
  39
  40 #include <sys/param.h>
  41 #include <sys/systm.h>
  42 #include <sys/kernel.h>
  43 #include <sys/proc.h>
  44 #include <sys/vmmeter.h>
  45 #include <sys/bus.h>
  46 #include <sys/malloc.h>
  47 #include <sys/ktr.h>
  48 #include <sys/lock.h>
  49 #include <sys/mutex.h>
  50 #include <sys/smp.h>
  51 #include <sys/sysctl.h>
  52 #include <sys/syslog.h>
  53
  54 #include <machine/clock.h>
  55 #include <machine/cpu.h>
  56 #include <machine/fpu.h>
  57 #include <machine/frame.h>
  58 #include <machine/intr.h>
  59 #include <machine/md_var.h>
  60 #include <machine/pcb.h>
  61 #include <machine/reg.h>
  62 #include <machine/sapicvar.h>
  63 #include <machine/smp.h>
  64
  65 #ifdef EVCNT_COUNTERS
  66 struct evcnt clock_intr_evcnt;  /* event counter for clock intrs. */
  67 #else
  68 #include <sys/interrupt.h>
  69 #include <machine/intrcnt.h>
  70 #endif
  71
  72 #ifdef DDB
  73 #include <ddb/ddb.h>
  74 #endif
  75
  76 #ifdef SMP
  77 extern int mp_ipi_test;
  78 #endif
  79
  80 static void ia64_dispatch_intr(void *, u_int);
  81
  82 static void
  83 dummy_perf(unsigned long vector, struct trapframe *tf)
  84 {
  85         printf("performance interrupt!\n");
  86 }
  87
  88 void (*perf_irq)(unsigned long, struct trapframe *) = dummy_perf;
  89
  90 static unsigned int ints[MAXCPU];
  91 SYSCTL_OPAQUE(_debug, OID_AUTO, ints, CTLFLAG_RW, &ints, sizeof(ints), "IU",
  92     "");
  93
  94 static unsigned int clks[MAXCPU];
  95 #ifdef SMP
  96 SYSCTL_OPAQUE(_debug, OID_AUTO, clks, CTLFLAG_RW, &clks, sizeof(clks), "IU",
  97     "");
  98 #else
  99 SYSCTL_INT(_debug, OID_AUTO, clks, CTLFLAG_RW, clks, 0, "");
 100 #endif
 101
 102 #ifdef SMP
 103 static unsigned int asts[MAXCPU];
 104 SYSCTL_OPAQUE(_debug, OID_AUTO, asts, CTLFLAG_RW, &asts, sizeof(asts), "IU",
 105     "");
 106
 107 static unsigned int rdvs[MAXCPU];
 108 SYSCTL_OPAQUE(_debug, OID_AUTO, rdvs, CTLFLAG_RW, &rdvs, sizeof(rdvs), "IU",
 109     "");
 110 #endif
 111
 112 SYSCTL_NODE(_debug, OID_AUTO, clock, CTLFLAG_RW, 0, "clock statistics");
 113
 114 static int adjust_edges = 0;
 115 SYSCTL_INT(_debug_clock, OID_AUTO, adjust_edges, CTLFLAG_RD,
 116     &adjust_edges, 0, "Number of times ITC got more than 12.5% behind");
 117
 118 static int adjust_excess = 0;
 119 SYSCTL_INT(_debug_clock, OID_AUTO, adjust_excess, CTLFLAG_RD,
 120     &adjust_excess, 0, "Total number of ignored ITC interrupts");
 121
 122 static int adjust_lost = 0;
 123 SYSCTL_INT(_debug_clock, OID_AUTO, adjust_lost, CTLFLAG_RD,
 124     &adjust_lost, 0, "Total number of lost ITC interrupts");
 125
 126 static int adjust_ticks = 0;
 127 SYSCTL_INT(_debug_clock, OID_AUTO, adjust_ticks, CTLFLAG_RD,
 128     &adjust_ticks, 0, "Total number of ITC interrupts with adjustment");
 129
 130 void
 131 interrupt(struct trapframe *tf)
 132 {
 133         struct thread *td;
 134         volatile struct ia64_interrupt_block *ib = IA64_INTERRUPT_BLOCK;
 135         uint64_t adj, clk, itc;
 136         int64_t delta;
 137         u_int vector;
 138         int count;
 139         uint8_t inta;
 140         ia64_set_fpsr(IA64_FPSR_DEFAULT);
 141
 142         td = curthread;
 143         atomic_add_int(&td->td_intr_nesting_level, 1);
 144
 145         vector = tf->tf_special.ifa;
 146
 147  next:
 148         /*
 149          * Handle ExtINT interrupts by generating an INTA cycle to
 150          * read the vector.
 151          */
 152         if (vector == 0) {
 153                 inta = ib->ib_inta;
 154                 printf("ExtINT interrupt: vector=%u\n", (int)inta);
 155                 if (inta == 15) {
 156                         __asm __volatile("mov cr.eoi = r0;; srlz.d");
 157                         goto stray;
 158                 }
 159                 vector = (int)inta;
 160         } else if (vector == 15)
 161                 goto stray;
 162
 163         if (vector == CLOCK_VECTOR) {/* clock interrupt */
 164                 /* CTR0(KTR_INTR, "clock interrupt"); */
 165
 166                 itc = ia64_get_itc();
 167
 168                 PCPU_INC(cnt.v_intr);
 169 #ifdef EVCNT_COUNTERS
 170                 clock_intr_evcnt.ev_count++;
 171 #else
 172                 intrcnt[INTRCNT_CLOCK]++;
 173 #endif
 174                 clks[PCPU_GET(cpuid)]++;
 175
 176                 critical_enter();
 177
 178                 adj = PCPU_GET(clockadj);
 179                 clk = PCPU_GET(clock);
 180                 delta = itc - clk;
 181                 count = 0;
 182                 while (delta >= ia64_clock_reload) {
 183                         /* Only the BSP runs the real clock */
 184                         if (PCPU_GET(cpuid) == 0)
 185                                 hardclock(TRAPF_USERMODE(tf), TRAPF_PC(tf));
 186                         else
 187                                 hardclock_cpu(TRAPF_USERMODE(tf));
 188                         if (profprocs != 0)
 189                                 profclock(TRAPF_USERMODE(tf), TRAPF_PC(tf));
 190                         statclock(TRAPF_USERMODE(tf));
 191                         delta -= ia64_clock_reload;
 192                         clk += ia64_clock_reload;
 193                         if (adj != 0)
 194                                 adjust_ticks++;
 195                         count++;
 196                 }
 197                 ia64_set_itm(itc + ia64_clock_reload - adj);
 198                 if (count > 0) {
 199                         adjust_lost += count - 1;
 200                         if (delta > (ia64_clock_reload >> 3)) {
 201                                 if (adj == 0)
 202                                         adjust_edges++;
 203                                 adj = ia64_clock_reload >> 4;
 204                         } else
 205                                 adj = 0;
 206                 } else {
 207                         adj = 0;
 208                         adjust_excess++;
 209                 }
 210                 PCPU_SET(clock, clk);
 211                 PCPU_SET(clockadj, adj);
 212                 critical_exit();
 213                 ia64_srlz_d();
 214
 215 #ifdef SMP
 216         } else if (vector == ipi_vector[IPI_AST]) {
 217                 asts[PCPU_GET(cpuid)]++;
 218                 CTR1(KTR_SMP, "IPI_AST, cpuid=%d", PCPU_GET(cpuid));
 219         } else if (vector == ipi_vector[IPI_HIGH_FP]) {
 220                 struct thread *thr = PCPU_GET(fpcurthread);
 221                 if (thr != NULL) {
 222                         mtx_lock_spin(&thr->td_md.md_highfp_mtx);
 223                         save_high_fp(&thr->td_pcb->pcb_high_fp);
 224                         thr->td_pcb->pcb_fpcpu = NULL;
 225                         PCPU_SET(fpcurthread, NULL);
 226                         mtx_unlock_spin(&thr->td_md.md_highfp_mtx);
 227                 }
 228         } else if (vector == ipi_vector[IPI_RENDEZVOUS]) {
 229                 rdvs[PCPU_GET(cpuid)]++;
 230                 CTR1(KTR_SMP, "IPI_RENDEZVOUS, cpuid=%d", PCPU_GET(cpuid));
 231                 smp_rendezvous_action();
 232         } else if (vector == ipi_vector[IPI_STOP]) {
 233                 cpumask_t mybit = PCPU_GET(cpumask);
 234
 235                 savectx(PCPU_PTR(pcb));
 236                 atomic_set_int(&stopped_cpus, mybit);
 237                 while ((started_cpus & mybit) == 0)
 238                         cpu_spinwait();
 239                 atomic_clear_int(&started_cpus, mybit);
 240                 atomic_clear_int(&stopped_cpus, mybit);
 241         } else if (vector == ipi_vector[IPI_TEST]) {
 242                 CTR1(KTR_SMP, "IPI_TEST, cpuid=%d", PCPU_GET(cpuid));
 243                 mp_ipi_test++;
 244 #endif
 245         } else {
 246                 ints[PCPU_GET(cpuid)]++;
 247                 ia64_dispatch_intr(tf, vector);
 248         }
 249
 250         __asm __volatile("mov cr.eoi = r0;; srlz.d");
 251         vector = ia64_get_ivr();
 252         if (vector != 15)
 253                 goto next;
 254
 255 stray:
 256         atomic_subtract_int(&td->td_intr_nesting_level, 1);
 257
 258         if (TRAPF_USERMODE(tf)) {
 259                 enable_intr();
 260                 userret(td, tf);
 261                 mtx_assert(&Giant, MA_NOTOWNED);
 262                 do_ast(tf);
 263         }
 264 }
 265
 266 /*
 267  * Hardware irqs have vectors starting at this offset.
 268  */
 269 #define IA64_HARDWARE_IRQ_BASE  0x20
 270
 271 struct ia64_intr {
 272         struct intr_event *event;       /* interrupt event */
 273         volatile long *cntp;            /* interrupt counter */
 274         struct sapic *sapic;
 275         u_int   irq;
 276 };
 277
 278 static struct ia64_intr *ia64_intrs[256];
 279
 280 static void
 281 ia64_intr_eoi(void *arg)
 282 {
 283         u_int vector = (uintptr_t)arg;
 284         struct ia64_intr *i;
 285
 286         i = ia64_intrs[vector];
 287         if (i != NULL)
 288                 sapic_eoi(i->sapic, vector);
 289 }
 290
 291 static void
 292 ia64_intr_mask(void *arg)
 293 {
 294         u_int vector = (uintptr_t)arg;
 295         struct ia64_intr *i;
 296
 297         i = ia64_intrs[vector];
 298         if (i != NULL) {
 299                 sapic_mask(i->sapic, i->irq);
 300                 sapic_eoi(i->sapic, vector);
 301         }
 302 }
 303
 304 static void
 305 ia64_intr_unmask(void *arg)
 306 {
 307         u_int vector = (uintptr_t)arg;
 308         struct ia64_intr *i;
 309
 310         i = ia64_intrs[vector];
 311         if (i != NULL)
 312                 sapic_unmask(i->sapic, i->irq);
 313 }
 314
 315 int
 316 ia64_setup_intr(const char *name, int irq, driver_filter_t filter,
 317     driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep)
 318 {
 319         struct ia64_intr *i;
 320         struct sapic *sa;
 321         char *intrname;
 322         u_int vector;
 323         int error;
 324
 325         /* Get the I/O SAPIC that corresponds to the IRQ. */
 326         sa = sapic_lookup(irq);
 327         if (sa == NULL)
 328                 return (EINVAL);
 329
 330         /*
 331          * XXX - There's a priority implied by the choice of vector.
 332          * We should therefore relate the vector to the interrupt type.
 333          */
 334         vector = irq + IA64_HARDWARE_IRQ_BASE;
 335
 336         i = ia64_intrs[vector];
 337         if (i == NULL) {
 338                 i = malloc(sizeof(struct ia64_intr), M_DEVBUF, M_NOWAIT);
 339                 if (i == NULL)
 340                         return (ENOMEM);
 341
 342                 error = intr_event_create(&i->event, (void *)(uintptr_t)vector,
 343                     0, ia64_intr_unmask,
 344 #ifdef INTR_FILTER
 345                     ia64_intr_eoi, ia64_intr_mask,
 346 #endif
 347                     NULL, "irq%u:", irq);
 348                 if (error) {
 349                         free(i, M_DEVBUF);
 350                         return (error);
 351                 }
 352
 353                 if (!atomic_cmpset_ptr(&ia64_intrs[vector], NULL, i)) {
 354                         intr_event_destroy(i->event);
 355                         free(i, M_DEVBUF);
 356                         i = ia64_intrs[vector];
 357                 } else {
 358                         i->sapic = sa;
 359                         i->irq = irq;
 360
 361                         i->cntp = intrcnt + irq + INTRCNT_ISA_IRQ;
 362                         if (name != NULL && *name != '\0') {
 363                                 /* XXX needs abstraction. Too error prone. */
 364                                 intrname = intrnames +
 365                                     (irq + INTRCNT_ISA_IRQ) * INTRNAME_LEN;
 366                                 memset(intrname, ' ', INTRNAME_LEN - 1);
 367                                 bcopy(name, intrname, strlen(name));
 368                         }
 369
 370                         sapic_enable(i->sapic, irq, vector);
 371                 }
 372         }
 373
 374         error = intr_event_add_handler(i->event, name, filter, handler, arg,
 375             intr_priority(flags), flags, cookiep);
 376         return (error);
 377 }
 378
 379 int
 380 ia64_teardown_intr(void *cookie)
 381 {
 382
 383         return (intr_event_remove_handler(cookie));
 384 }
 385
 386 static void
 387 ia64_dispatch_intr(void *frame, u_int vector)
 388 {
 389         struct ia64_intr *i;
 390         struct intr_event *ie;                  /* our interrupt event */
 391 #ifndef INTR_FILTER
 392         struct intr_handler *ih;
 393         int error, thread, ret;
 394 #endif
 395
 396         /*
 397          * Find the interrupt thread for this vector.
 398          */
 399         i = ia64_intrs[vector];
 400         KASSERT(i != NULL, ("%s: unassigned vector", __func__));
 401
 402         (*i->cntp)++;
 403
 404         ie = i->event;
 405         KASSERT(ie != NULL, ("%s: interrupt without event", __func__));
 406
 407 #ifdef INTR_FILTER
 408         if (intr_event_handle(ie, frame) != 0) {
 409                 ia64_intr_mask((void *)(uintptr_t)vector);
 410                 log(LOG_ERR, "stray irq%u\n", i->irq);
 411         }
 412 #else
 413         /*
 414          * As an optimization, if an event has no handlers, don't
 415          * schedule it to run.
 416          */
 417         if (TAILQ_EMPTY(&ie->ie_handlers))
 418                 return;
 419
 420         /*
 421          * Execute all fast interrupt handlers directly without Giant.  Note
 422          * that this means that any fast interrupt handler must be MP safe.
 423          */
 424         ret = 0;
 425         thread = 0;
 426         critical_enter();
 427         TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) {
 428                 if (ih->ih_filter == NULL) {
 429                         thread = 1;
 430                         continue;
 431                 }
 432                 CTR4(KTR_INTR, "%s: exec %p(%p) for %s", __func__,
 433                     ih->ih_filter, ih->ih_argument, ih->ih_name);
 434                 ret = ih->ih_filter(ih->ih_argument);
 435                 /*
 436                  * Wrapper handler special case: see
 437                  * i386/intr_machdep.c::intr_execute_handlers()
 438                  */
 439                 if (!thread) {
 440                         if (ret == FILTER_SCHEDULE_THREAD)
 441                                 thread = 1;
 442                 }
 443         }
 444
 445         if (thread) {
 446                 ia64_intr_mask((void *)(uintptr_t)vector);
 447                 error = intr_event_schedule_thread(ie);
 448                 KASSERT(error == 0, ("%s: impossible stray", __func__));
 449         } else
 450                 ia64_intr_eoi((void *)(uintptr_t)vector);
 451         critical_exit();
 452 #endif
 453 }
 454
 455 #ifdef DDB
 456
 457 static void
 458 db_print_vector(u_int vector, int always)
 459 {
 460         struct ia64_intr *i;
 461
 462         i = ia64_intrs[vector];
 463         if (i != NULL) {
 464                 db_printf("vector %u (%p): ", vector, i);
 465                 sapic_print(i->sapic, i->irq);
 466         } else if (always)
 467                 db_printf("vector %u: unassigned\n", vector);
 468 }
 469
 470 DB_SHOW_COMMAND(vector, db_show_vector)
 471 {
 472         u_int vector;
 473
 474         if (have_addr) {
 475                 vector = ((addr >> 4) % 16) * 10 + (addr % 16);
 476                 if (vector >= 256)
 477                         db_printf("error: vector %u not in range [0..255]\n",
 478                             vector);
 479                 else
 480                         db_print_vector(vector, 1);
 481         } else {
 482                 for (vector = 0; vector < 256; vector++)
 483                         db_print_vector(vector, 0);
 484         }
 485 }
 486
 487 #endif