sys/ia64/ia64/interrupt.c

   1 /* $FreeBSD$ */
   2 /* $NetBSD: interrupt.c,v 1.23 1998/02/24 07:38:01 thorpej Exp $ */
   3
   4 /*-
   5  * Copyright (c) 1994, 1995, 1996 Carnegie-Mellon University.
   6  * All rights reserved.
   7  *
   8  * Authors: Keith Bostic, Chris G. Demetriou
   9  *
  10  * Permission to use, copy, modify and distribute this software and
  11  * its documentation is hereby granted, provided that both the copyright
  12  * notice and this permission notice appear in all copies of the
  13  * software, derivative works or modified versions, and any portions
  14  * thereof, and that both notices appear in supporting documentation.
  15  *
  16  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  17  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
  18  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  19  *
  20  * Carnegie Mellon requests users of this software to return to
  21  *
  22  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  23  *  School of Computer Science
  24  *  Carnegie Mellon University
  25  *  Pittsburgh PA 15213-3890
  26  *
  27  * any improvements or extensions that they make and grant Carnegie the
  28  * rights to redistribute these changes.
  29  */
  30 /*-
  31  * Additional Copyright (c) 1997 by Matthew Jacob for NASA/Ames Research Center.
  32  * Redistribute and modify at will, leaving only this additional copyright
  33  * notice.
  34  */
  35
  36 #include "opt_ddb.h"
  37
  38 #include <sys/cdefs.h>                  /* RCS ID & Copyright macro defns */
  39
  40 #include <sys/param.h>
  41 #include <sys/systm.h>
  42 #include <sys/kernel.h>
  43 #include <sys/proc.h>
  44 #include <sys/vmmeter.h>
  45 #include <sys/bus.h>
  46 #include <sys/malloc.h>
  47 #include <sys/ktr.h>
  48 #include <sys/lock.h>
  49 #include <sys/mutex.h>
  50 #include <sys/sched.h>
  51 #include <sys/smp.h>
  52 #include <sys/sysctl.h>
  53 #include <sys/syslog.h>
  54
  55 #include <machine/clock.h>
  56 #include <machine/cpu.h>
  57 #include <machine/fpu.h>
  58 #include <machine/frame.h>
  59 #include <machine/intr.h>
  60 #include <machine/md_var.h>
  61 #include <machine/pcb.h>
  62 #include <machine/reg.h>
  63 #include <machine/sapicvar.h>
  64 #include <machine/smp.h>
  65
  66 #ifdef EVCNT_COUNTERS
  67 struct evcnt clock_intr_evcnt;  /* event counter for clock intrs. */
  68 #else
  69 #include <sys/interrupt.h>
  70 #include <machine/intrcnt.h>
  71 #endif
  72
  73 #ifdef DDB
  74 #include <ddb/ddb.h>
  75 #endif
  76
  77 #ifdef SMP
  78 extern int mp_ipi_test;
  79 #endif
  80
  81 static void ia64_dispatch_intr(void *, u_int);
  82
  83 static void
  84 dummy_perf(unsigned long vector, struct trapframe *tf)
  85 {
  86         printf("performance interrupt!\n");
  87 }
  88
  89 void (*perf_irq)(unsigned long, struct trapframe *) = dummy_perf;
  90
  91 static unsigned int ints[MAXCPU];
  92 SYSCTL_OPAQUE(_debug, OID_AUTO, ints, CTLFLAG_RW, &ints, sizeof(ints), "IU",
  93     "");
  94
  95 static unsigned int clks[MAXCPU];
  96 #ifdef SMP
  97 SYSCTL_OPAQUE(_debug, OID_AUTO, clks, CTLFLAG_RW, &clks, sizeof(clks), "IU",
  98     "");
  99 #else
 100 SYSCTL_INT(_debug, OID_AUTO, clks, CTLFLAG_RW, clks, 0, "");
 101 #endif
 102
 103 #ifdef SMP
 104 static unsigned int asts[MAXCPU];
 105 SYSCTL_OPAQUE(_debug, OID_AUTO, asts, CTLFLAG_RW, &asts, sizeof(asts), "IU",
 106     "");
 107
 108 static unsigned int rdvs[MAXCPU];
 109 SYSCTL_OPAQUE(_debug, OID_AUTO, rdvs, CTLFLAG_RW, &rdvs, sizeof(rdvs), "IU",
 110     "");
 111 #endif
 112
 113 SYSCTL_NODE(_debug, OID_AUTO, clock, CTLFLAG_RW, 0, "clock statistics");
 114
 115 static int adjust_edges = 0;
 116 SYSCTL_INT(_debug_clock, OID_AUTO, adjust_edges, CTLFLAG_RD,
 117     &adjust_edges, 0, "Number of times ITC got more than 12.5% behind");
 118
 119 static int adjust_excess = 0;
 120 SYSCTL_INT(_debug_clock, OID_AUTO, adjust_excess, CTLFLAG_RD,
 121     &adjust_excess, 0, "Total number of ignored ITC interrupts");
 122
 123 static int adjust_lost = 0;
 124 SYSCTL_INT(_debug_clock, OID_AUTO, adjust_lost, CTLFLAG_RD,
 125     &adjust_lost, 0, "Total number of lost ITC interrupts");
 126
 127 static int adjust_ticks = 0;
 128 SYSCTL_INT(_debug_clock, OID_AUTO, adjust_ticks, CTLFLAG_RD,
 129     &adjust_ticks, 0, "Total number of ITC interrupts with adjustment");
 130
 131 void
 132 interrupt(struct trapframe *tf)
 133 {
 134         struct thread *td;
 135         volatile struct ia64_interrupt_block *ib = IA64_INTERRUPT_BLOCK;
 136         uint64_t adj, clk, itc;
 137         int64_t delta;
 138         u_int vector;
 139         int count;
 140         uint8_t inta;
 141         ia64_set_fpsr(IA64_FPSR_DEFAULT);
 142
 143         td = curthread;
 144         atomic_add_int(&td->td_intr_nesting_level, 1);
 145
 146         vector = tf->tf_special.ifa;
 147
 148  next:
 149         /*
 150          * Handle ExtINT interrupts by generating an INTA cycle to
 151          * read the vector.
 152          */
 153         if (vector == 0) {
 154                 inta = ib->ib_inta;
 155                 printf("ExtINT interrupt: vector=%u\n", (int)inta);
 156                 if (inta == 15) {
 157                         __asm __volatile("mov cr.eoi = r0;; srlz.d");
 158                         goto stray;
 159                 }
 160                 vector = (int)inta;
 161         } else if (vector == 15)
 162                 goto stray;
 163
 164         if (vector == CLOCK_VECTOR) {/* clock interrupt */
 165                 /* CTR0(KTR_INTR, "clock interrupt"); */
 166
 167                 itc = ia64_get_itc();
 168
 169                 PCPU_INC(cnt.v_intr);
 170 #ifdef EVCNT_COUNTERS
 171                 clock_intr_evcnt.ev_count++;
 172 #else
 173                 intrcnt[INTRCNT_CLOCK]++;
 174 #endif
 175                 clks[PCPU_GET(cpuid)]++;
 176
 177                 critical_enter();
 178
 179                 adj = PCPU_GET(clockadj);
 180                 clk = PCPU_GET(clock);
 181                 delta = itc - clk;
 182                 count = 0;
 183                 while (delta >= ia64_clock_reload) {
 184                         /* Only the BSP runs the real clock */
 185                         if (PCPU_GET(cpuid) == 0)
 186                                 hardclock(TRAPF_USERMODE(tf), TRAPF_PC(tf));
 187                         else
 188                                 hardclock_cpu(TRAPF_USERMODE(tf));
 189                         if (profprocs != 0)
 190                                 profclock(TRAPF_USERMODE(tf), TRAPF_PC(tf));
 191                         statclock(TRAPF_USERMODE(tf));
 192                         delta -= ia64_clock_reload;
 193                         clk += ia64_clock_reload;
 194                         if (adj != 0)
 195                                 adjust_ticks++;
 196                         count++;
 197                 }
 198                 ia64_set_itm(itc + ia64_clock_reload - adj);
 199                 if (count > 0) {
 200                         adjust_lost += count - 1;
 201                         if (delta > (ia64_clock_reload >> 3)) {
 202                                 if (adj == 0)
 203                                         adjust_edges++;
 204                                 adj = ia64_clock_reload >> 4;
 205                         } else
 206                                 adj = 0;
 207                 } else {
 208                         adj = 0;
 209                         adjust_excess++;
 210                 }
 211                 PCPU_SET(clock, clk);
 212                 PCPU_SET(clockadj, adj);
 213                 critical_exit();
 214                 ia64_srlz_d();
 215
 216 #ifdef SMP
 217         } else if (vector == ipi_vector[IPI_AST]) {
 218                 asts[PCPU_GET(cpuid)]++;
 219                 CTR1(KTR_SMP, "IPI_AST, cpuid=%d", PCPU_GET(cpuid));
 220         } else if (vector == ipi_vector[IPI_HIGH_FP]) {
 221                 struct thread *thr = PCPU_GET(fpcurthread);
 222                 if (thr != NULL) {
 223                         mtx_lock_spin(&thr->td_md.md_highfp_mtx);
 224                         save_high_fp(&thr->td_pcb->pcb_high_fp);
 225                         thr->td_pcb->pcb_fpcpu = NULL;
 226                         PCPU_SET(fpcurthread, NULL);
 227                         mtx_unlock_spin(&thr->td_md.md_highfp_mtx);
 228                 }
 229         } else if (vector == ipi_vector[IPI_RENDEZVOUS]) {
 230                 rdvs[PCPU_GET(cpuid)]++;
 231                 CTR1(KTR_SMP, "IPI_RENDEZVOUS, cpuid=%d", PCPU_GET(cpuid));
 232                 smp_rendezvous_action();
 233         } else if (vector == ipi_vector[IPI_STOP]) {
 234                 cpumask_t mybit = PCPU_GET(cpumask);
 235
 236                 savectx(PCPU_PTR(pcb));
 237                 atomic_set_int(&stopped_cpus, mybit);
 238                 while ((started_cpus & mybit) == 0)
 239                         cpu_spinwait();
 240                 atomic_clear_int(&started_cpus, mybit);
 241                 atomic_clear_int(&stopped_cpus, mybit);
 242         } else if (vector == ipi_vector[IPI_TEST]) {
 243                 CTR1(KTR_SMP, "IPI_TEST, cpuid=%d", PCPU_GET(cpuid));
 244                 mp_ipi_test++;
 245         } else if (vector == ipi_vector[IPI_PREEMPT]) {
 246                 CTR1(KTR_SMP, "IPI_PREEMPT, cpuid=%d", PCPU_GET(cpuid));
 247                 sched_preempt(curthread);
 248 #endif
 249         } else {
 250                 ints[PCPU_GET(cpuid)]++;
 251                 ia64_dispatch_intr(tf, vector);
 252         }
 253
 254         __asm __volatile("mov cr.eoi = r0;; srlz.d");
 255         vector = ia64_get_ivr();
 256         if (vector != 15)
 257                 goto next;
 258
 259 stray:
 260         atomic_subtract_int(&td->td_intr_nesting_level, 1);
 261
 262         if (TRAPF_USERMODE(tf)) {
 263                 enable_intr();
 264                 userret(td, tf);
 265                 mtx_assert(&Giant, MA_NOTOWNED);
 266                 do_ast(tf);
 267         }
 268 }
 269
 270 /*
 271  * Hardware irqs have vectors starting at this offset.
 272  */
 273 #define IA64_HARDWARE_IRQ_BASE  0x20
 274
 275 struct ia64_intr {
 276         struct intr_event *event;       /* interrupt event */
 277         volatile long *cntp;            /* interrupt counter */
 278         struct sapic *sapic;
 279         u_int   irq;
 280 };
 281
 282 static struct ia64_intr *ia64_intrs[256];
 283
 284 static void
 285 ia64_intr_eoi(void *arg)
 286 {
 287         u_int vector = (uintptr_t)arg;
 288         struct ia64_intr *i;
 289
 290         i = ia64_intrs[vector];
 291         if (i != NULL)
 292                 sapic_eoi(i->sapic, vector);
 293 }
 294
 295 static void
 296 ia64_intr_mask(void *arg)
 297 {
 298         u_int vector = (uintptr_t)arg;
 299         struct ia64_intr *i;
 300
 301         i = ia64_intrs[vector];
 302         if (i != NULL) {
 303                 sapic_mask(i->sapic, i->irq);
 304                 sapic_eoi(i->sapic, vector);
 305         }
 306 }
 307
 308 static void
 309 ia64_intr_unmask(void *arg)
 310 {
 311         u_int vector = (uintptr_t)arg;
 312         struct ia64_intr *i;
 313
 314         i = ia64_intrs[vector];
 315         if (i != NULL)
 316                 sapic_unmask(i->sapic, i->irq);
 317 }
 318
 319 int
 320 ia64_setup_intr(const char *name, int irq, driver_filter_t filter,
 321     driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep)
 322 {
 323         struct ia64_intr *i;
 324         struct sapic *sa;
 325         char *intrname;
 326         u_int vector;
 327         int error;
 328
 329         /* Get the I/O SAPIC that corresponds to the IRQ. */
 330         sa = sapic_lookup(irq);
 331         if (sa == NULL)
 332                 return (EINVAL);
 333
 334         /*
 335          * XXX - There's a priority implied by the choice of vector.
 336          * We should therefore relate the vector to the interrupt type.
 337          */
 338         vector = irq + IA64_HARDWARE_IRQ_BASE;
 339
 340         i = ia64_intrs[vector];
 341         if (i == NULL) {
 342                 i = malloc(sizeof(struct ia64_intr), M_DEVBUF, M_NOWAIT);
 343                 if (i == NULL)
 344                         return (ENOMEM);
 345
 346                 error = intr_event_create(&i->event, (void *)(uintptr_t)vector,
 347                     0, irq, ia64_intr_mask, ia64_intr_unmask, ia64_intr_eoi,
 348                     NULL, "irq%u:", irq);
 349                 if (error) {
 350                         free(i, M_DEVBUF);
 351                         return (error);
 352                 }
 353
 354                 if (!atomic_cmpset_ptr(&ia64_intrs[vector], NULL, i)) {
 355                         intr_event_destroy(i->event);
 356                         free(i, M_DEVBUF);
 357                         i = ia64_intrs[vector];
 358                 } else {
 359                         i->sapic = sa;
 360                         i->irq = irq;
 361
 362                         i->cntp = intrcnt + irq + INTRCNT_ISA_IRQ;
 363                         if (name != NULL && *name != '\0') {
 364                                 /* XXX needs abstraction. Too error prone. */
 365                                 intrname = intrnames +
 366                                     (irq + INTRCNT_ISA_IRQ) * INTRNAME_LEN;
 367                                 memset(intrname, ' ', INTRNAME_LEN - 1);
 368                                 bcopy(name, intrname, strlen(name));
 369                         }
 370
 371                         sapic_enable(i->sapic, irq, vector);
 372                 }
 373         }
 374
 375         error = intr_event_add_handler(i->event, name, filter, handler, arg,
 376             intr_priority(flags), flags, cookiep);
 377         return (error);
 378 }
 379
 380 int
 381 ia64_teardown_intr(void *cookie)
 382 {
 383
 384         return (intr_event_remove_handler(cookie));
 385 }
 386
 387 static void
 388 ia64_dispatch_intr(void *frame, u_int vector)
 389 {
 390         struct ia64_intr *i;
 391         struct intr_event *ie;                  /* our interrupt event */
 392
 393         /*
 394          * Find the interrupt thread for this vector.
 395          */
 396         i = ia64_intrs[vector];
 397         KASSERT(i != NULL, ("%s: unassigned vector", __func__));
 398
 399         (*i->cntp)++;
 400
 401         ie = i->event;
 402         KASSERT(ie != NULL, ("%s: interrupt without event", __func__));
 403
 404         if (intr_event_handle(ie, frame) != 0) {
 405                 /*
 406                  * XXX: The pre-INTR_FILTER code didn't mask stray
 407                  * interrupts.
 408                  */
 409                 ia64_intr_mask((void *)(uintptr_t)vector);
 410                 log(LOG_ERR, "stray irq%u\n", i->irq);
 411         }
 412 }
 413
 414 #ifdef DDB
 415
 416 static void
 417 db_print_vector(u_int vector, int always)
 418 {
 419         struct ia64_intr *i;
 420
 421         i = ia64_intrs[vector];
 422         if (i != NULL) {
 423                 db_printf("vector %u (%p): ", vector, i);
 424                 sapic_print(i->sapic, i->irq);
 425         } else if (always)
 426                 db_printf("vector %u: unassigned\n", vector);
 427 }
 428
 429 DB_SHOW_COMMAND(vector, db_show_vector)
 430 {
 431         u_int vector;
 432
 433         if (have_addr) {
 434                 vector = ((addr >> 4) % 16) * 10 + (addr % 16);
 435                 if (vector >= 256)
 436                         db_printf("error: vector %u not in range [0..255]\n",
 437                             vector);
 438                 else
 439                         db_print_vector(vector, 1);
 440         } else {
 441                 for (vector = 0; vector < 256; vector++)
 442                         db_print_vector(vector, 0);
 443         }
 444 }
 445
 446 #endif