sys/ia64/ia64/interrupt.c

   1 /* $FreeBSD$ */
   2 /* $NetBSD: interrupt.c,v 1.23 1998/02/24 07:38:01 thorpej Exp $ */
   3
   4 /*-
   5  * Copyright (c) 1994, 1995, 1996 Carnegie-Mellon University.
   6  * All rights reserved.
   7  *
   8  * Authors: Keith Bostic, Chris G. Demetriou
   9  *
  10  * Permission to use, copy, modify and distribute this software and
  11  * its documentation is hereby granted, provided that both the copyright
  12  * notice and this permission notice appear in all copies of the
  13  * software, derivative works or modified versions, and any portions
  14  * thereof, and that both notices appear in supporting documentation.
  15  *
  16  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  17  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
  18  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  19  *
  20  * Carnegie Mellon requests users of this software to return to
  21  *
  22  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  23  *  School of Computer Science
  24  *  Carnegie Mellon University
  25  *  Pittsburgh PA 15213-3890
  26  *
  27  * any improvements or extensions that they make and grant Carnegie the
  28  * rights to redistribute these changes.
  29  */
  30 /*-
  31  * Additional Copyright (c) 1997 by Matthew Jacob for NASA/Ames Research Center.
  32  * Redistribute and modify at will, leaving only this additional copyright
  33  * notice.
  34  */
  35
  36 #include "opt_ddb.h"
  37
  38 #include <sys/cdefs.h>                  /* RCS ID & Copyright macro defns */
  39
  40 #include <sys/param.h>
  41 #include <sys/systm.h>
  42 #include <sys/kernel.h>
  43 #include <sys/proc.h>
  44 #include <sys/vmmeter.h>
  45 #include <sys/bus.h>
  46 #include <sys/malloc.h>
  47 #include <sys/ktr.h>
  48 #include <sys/lock.h>
  49 #include <sys/mutex.h>
  50 #include <sys/sched.h>
  51 #include <sys/smp.h>
  52 #include <sys/sysctl.h>
  53 #include <sys/syslog.h>
  54
  55 #include <machine/clock.h>
  56 #include <machine/cpu.h>
  57 #include <machine/fpu.h>
  58 #include <machine/frame.h>
  59 #include <machine/intr.h>
  60 #include <machine/md_var.h>
  61 #include <machine/pcb.h>
  62 #include <machine/reg.h>
  63 #include <machine/sapicvar.h>
  64 #include <machine/smp.h>
  65
  66 #ifdef EVCNT_COUNTERS
  67 struct evcnt clock_intr_evcnt;  /* event counter for clock intrs. */
  68 #else
  69 #include <sys/interrupt.h>
  70 #include <machine/intrcnt.h>
  71 #endif
  72
  73 #ifdef DDB
  74 #include <ddb/ddb.h>
  75 #endif
  76
  77 static void ia64_dispatch_intr(void *, u_int);
  78
  79 static void
  80 dummy_perf(unsigned long vector, struct trapframe *tf)
  81 {
  82         printf("performance interrupt!\n");
  83 }
  84
  85 void (*perf_irq)(unsigned long, struct trapframe *) = dummy_perf;
  86
  87 static unsigned int ints[MAXCPU];
  88 SYSCTL_OPAQUE(_debug, OID_AUTO, ints, CTLFLAG_RW, &ints, sizeof(ints), "IU",
  89     "");
  90
  91 static unsigned int clks[MAXCPU];
  92 #ifdef SMP
  93 SYSCTL_OPAQUE(_debug, OID_AUTO, clks, CTLFLAG_RW, &clks, sizeof(clks), "IU",
  94     "");
  95 #else
  96 SYSCTL_INT(_debug, OID_AUTO, clks, CTLFLAG_RW, clks, 0, "");
  97 #endif
  98
  99 #ifdef SMP
 100 static unsigned int asts[MAXCPU];
 101 SYSCTL_OPAQUE(_debug, OID_AUTO, asts, CTLFLAG_RW, &asts, sizeof(asts), "IU",
 102     "");
 103
 104 static unsigned int rdvs[MAXCPU];
 105 SYSCTL_OPAQUE(_debug, OID_AUTO, rdvs, CTLFLAG_RW, &rdvs, sizeof(rdvs), "IU",
 106     "");
 107 #endif
 108
 109 SYSCTL_NODE(_debug, OID_AUTO, clock, CTLFLAG_RW, 0, "clock statistics");
 110
 111 static int adjust_edges = 0;
 112 SYSCTL_INT(_debug_clock, OID_AUTO, adjust_edges, CTLFLAG_RD,
 113     &adjust_edges, 0, "Number of times ITC got more than 12.5% behind");
 114
 115 static int adjust_excess = 0;
 116 SYSCTL_INT(_debug_clock, OID_AUTO, adjust_excess, CTLFLAG_RD,
 117     &adjust_excess, 0, "Total number of ignored ITC interrupts");
 118
 119 static int adjust_lost = 0;
 120 SYSCTL_INT(_debug_clock, OID_AUTO, adjust_lost, CTLFLAG_RD,
 121     &adjust_lost, 0, "Total number of lost ITC interrupts");
 122
 123 static int adjust_ticks = 0;
 124 SYSCTL_INT(_debug_clock, OID_AUTO, adjust_ticks, CTLFLAG_RD,
 125     &adjust_ticks, 0, "Total number of ITC interrupts with adjustment");
 126
 127 void
 128 interrupt(struct trapframe *tf)
 129 {
 130         struct thread *td;
 131         volatile struct ia64_interrupt_block *ib = IA64_INTERRUPT_BLOCK;
 132         uint64_t adj, clk, itc;
 133         int64_t delta;
 134         u_int vector;
 135         int count;
 136         uint8_t inta;
 137
 138         ia64_set_fpsr(IA64_FPSR_DEFAULT);
 139
 140         td = curthread;
 141
 142         vector = tf->tf_special.ifa;
 143
 144  next:
 145         /*
 146          * Handle ExtINT interrupts by generating an INTA cycle to
 147          * read the vector.
 148          */
 149         if (vector == 0) {
 150                 inta = ib->ib_inta;
 151                 printf("ExtINT interrupt: vector=%u\n", (int)inta);
 152                 if (inta == 15) {
 153                         __asm __volatile("mov cr.eoi = r0;; srlz.d");
 154                         goto stray;
 155                 }
 156                 vector = (int)inta;
 157         } else if (vector == 15)
 158                 goto stray;
 159
 160         if (vector == CLOCK_VECTOR) {/* clock interrupt */
 161                 /* CTR0(KTR_INTR, "clock interrupt"); */
 162
 163                 itc = ia64_get_itc();
 164
 165                 PCPU_INC(cnt.v_intr);
 166 #ifdef EVCNT_COUNTERS
 167                 clock_intr_evcnt.ev_count++;
 168 #else
 169                 intrcnt[INTRCNT_CLOCK]++;
 170 #endif
 171                 clks[PCPU_GET(cpuid)]++;
 172
 173                 critical_enter();
 174
 175                 adj = PCPU_GET(clockadj);
 176                 clk = PCPU_GET(clock);
 177                 delta = itc - clk;
 178                 count = 0;
 179                 while (delta >= ia64_clock_reload) {
 180                         /* Only the BSP runs the real clock */
 181                         if (PCPU_GET(cpuid) == 0)
 182                                 hardclock(TRAPF_USERMODE(tf), TRAPF_PC(tf));
 183                         else
 184                                 hardclock_cpu(TRAPF_USERMODE(tf));
 185                         if (profprocs != 0)
 186                                 profclock(TRAPF_USERMODE(tf), TRAPF_PC(tf));
 187                         statclock(TRAPF_USERMODE(tf));
 188                         delta -= ia64_clock_reload;
 189                         clk += ia64_clock_reload;
 190                         if (adj != 0)
 191                                 adjust_ticks++;
 192                         count++;
 193                 }
 194                 ia64_set_itm(ia64_get_itc() + ia64_clock_reload - adj);
 195                 if (count > 0) {
 196                         adjust_lost += count - 1;
 197                         if (delta > (ia64_clock_reload >> 3)) {
 198                                 if (adj == 0)
 199                                         adjust_edges++;
 200                                 adj = ia64_clock_reload >> 4;
 201                         } else
 202                                 adj = 0;
 203                 } else {
 204                         adj = 0;
 205                         adjust_excess++;
 206                 }
 207                 PCPU_SET(clock, clk);
 208                 PCPU_SET(clockadj, adj);
 209                 critical_exit();
 210                 ia64_srlz_d();
 211
 212 #ifdef SMP
 213         } else if (vector == ipi_vector[IPI_AST]) {
 214                 asts[PCPU_GET(cpuid)]++;
 215                 CTR1(KTR_SMP, "IPI_AST, cpuid=%d", PCPU_GET(cpuid));
 216         } else if (vector == ipi_vector[IPI_HIGH_FP]) {
 217                 struct thread *thr = PCPU_GET(fpcurthread);
 218                 if (thr != NULL) {
 219                         mtx_lock_spin(&thr->td_md.md_highfp_mtx);
 220                         save_high_fp(&thr->td_pcb->pcb_high_fp);
 221                         thr->td_pcb->pcb_fpcpu = NULL;
 222                         PCPU_SET(fpcurthread, NULL);
 223                         mtx_unlock_spin(&thr->td_md.md_highfp_mtx);
 224                 }
 225         } else if (vector == ipi_vector[IPI_RENDEZVOUS]) {
 226                 rdvs[PCPU_GET(cpuid)]++;
 227                 CTR1(KTR_SMP, "IPI_RENDEZVOUS, cpuid=%d", PCPU_GET(cpuid));
 228                 enable_intr();
 229                 smp_rendezvous_action();
 230                 disable_intr();
 231         } else if (vector == ipi_vector[IPI_STOP]) {
 232                 cpumask_t mybit = PCPU_GET(cpumask);
 233
 234                 savectx(PCPU_PTR(pcb));
 235                 atomic_set_int(&stopped_cpus, mybit);
 236                 while ((started_cpus & mybit) == 0)
 237                         cpu_spinwait();
 238                 atomic_clear_int(&started_cpus, mybit);
 239                 atomic_clear_int(&stopped_cpus, mybit);
 240         } else if (vector == ipi_vector[IPI_PREEMPT]) {
 241                 CTR1(KTR_SMP, "IPI_PREEMPT, cpuid=%d", PCPU_GET(cpuid));
 242                 __asm __volatile("mov cr.eoi = r0;; srlz.d");
 243                 enable_intr();
 244                 sched_preempt(curthread);
 245                 disable_intr();
 246                 goto stray;
 247 #endif
 248         } else {
 249                 ints[PCPU_GET(cpuid)]++;
 250                 atomic_add_int(&td->td_intr_nesting_level, 1);
 251                 ia64_dispatch_intr(tf, vector);
 252                 atomic_subtract_int(&td->td_intr_nesting_level, 1);
 253         }
 254
 255         __asm __volatile("mov cr.eoi = r0;; srlz.d");
 256         vector = ia64_get_ivr();
 257         if (vector != 15)
 258                 goto next;
 259
 260 stray:
 261         if (TRAPF_USERMODE(tf)) {
 262                 enable_intr();
 263                 userret(td, tf);
 264                 mtx_assert(&Giant, MA_NOTOWNED);
 265                 do_ast(tf);
 266         }
 267 }
 268
 269 /*
 270  * Hardware irqs have vectors starting at this offset.
 271  */
 272 #define IA64_HARDWARE_IRQ_BASE  0x20
 273
 274 struct ia64_intr {
 275         struct intr_event *event;       /* interrupt event */
 276         volatile long *cntp;            /* interrupt counter */
 277         struct sapic *sapic;
 278         u_int   irq;
 279 };
 280
 281 static struct ia64_intr *ia64_intrs[256];
 282
 283 static void
 284 ia64_intr_eoi(void *arg)
 285 {
 286         u_int vector = (uintptr_t)arg;
 287         struct ia64_intr *i;
 288
 289         i = ia64_intrs[vector];
 290         if (i != NULL)
 291                 sapic_eoi(i->sapic, vector);
 292 }
 293
 294 static void
 295 ia64_intr_mask(void *arg)
 296 {
 297         u_int vector = (uintptr_t)arg;
 298         struct ia64_intr *i;
 299
 300         i = ia64_intrs[vector];
 301         if (i != NULL) {
 302                 sapic_mask(i->sapic, i->irq);
 303                 sapic_eoi(i->sapic, vector);
 304         }
 305 }
 306
 307 static void
 308 ia64_intr_unmask(void *arg)
 309 {
 310         u_int vector = (uintptr_t)arg;
 311         struct ia64_intr *i;
 312
 313         i = ia64_intrs[vector];
 314         if (i != NULL)
 315                 sapic_unmask(i->sapic, i->irq);
 316 }
 317
 318 int
 319 ia64_setup_intr(const char *name, int irq, driver_filter_t filter,
 320     driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep)
 321 {
 322         struct ia64_intr *i;
 323         struct sapic *sa;
 324         char *intrname;
 325         u_int vector;
 326         int error;
 327
 328         /* Get the I/O SAPIC that corresponds to the IRQ. */
 329         sa = sapic_lookup(irq);
 330         if (sa == NULL)
 331                 return (EINVAL);
 332
 333         /*
 334          * XXX - There's a priority implied by the choice of vector.
 335          * We should therefore relate the vector to the interrupt type.
 336          */
 337         vector = irq + IA64_HARDWARE_IRQ_BASE;
 338
 339         i = ia64_intrs[vector];
 340         if (i == NULL) {
 341                 i = malloc(sizeof(struct ia64_intr), M_DEVBUF, M_NOWAIT);
 342                 if (i == NULL)
 343                         return (ENOMEM);
 344
 345                 error = intr_event_create(&i->event, (void *)(uintptr_t)vector,
 346                     0, irq, ia64_intr_mask, ia64_intr_unmask, ia64_intr_eoi,
 347                     NULL, "irq%u:", irq);
 348                 if (error) {
 349                         free(i, M_DEVBUF);
 350                         return (error);
 351                 }
 352
 353                 if (!atomic_cmpset_ptr(&ia64_intrs[vector], NULL, i)) {
 354                         intr_event_destroy(i->event);
 355                         free(i, M_DEVBUF);
 356                         i = ia64_intrs[vector];
 357                 } else {
 358                         i->sapic = sa;
 359                         i->irq = irq;
 360
 361                         i->cntp = intrcnt + irq + INTRCNT_ISA_IRQ;
 362                         if (name != NULL && *name != '\0') {
 363                                 /* XXX needs abstraction. Too error prone. */
 364                                 intrname = intrnames +
 365                                     (irq + INTRCNT_ISA_IRQ) * INTRNAME_LEN;
 366                                 memset(intrname, ' ', INTRNAME_LEN - 1);
 367                                 bcopy(name, intrname, strlen(name));
 368                         }
 369
 370                         sapic_enable(i->sapic, irq, vector);
 371                 }
 372         }
 373
 374         error = intr_event_add_handler(i->event, name, filter, handler, arg,
 375             intr_priority(flags), flags, cookiep);
 376         return (error);
 377 }
 378
 379 int
 380 ia64_teardown_intr(void *cookie)
 381 {
 382
 383         return (intr_event_remove_handler(cookie));
 384 }
 385
 386 static void
 387 ia64_dispatch_intr(void *frame, u_int vector)
 388 {
 389         struct ia64_intr *i;
 390         struct intr_event *ie;                  /* our interrupt event */
 391
 392         /*
 393          * Find the interrupt thread for this vector.
 394          */
 395         i = ia64_intrs[vector];
 396         KASSERT(i != NULL, ("%s: unassigned vector", __func__));
 397
 398         (*i->cntp)++;
 399
 400         ie = i->event;
 401         KASSERT(ie != NULL, ("%s: interrupt without event", __func__));
 402
 403         if (intr_event_handle(ie, frame) != 0) {
 404                 /*
 405                  * XXX: The pre-INTR_FILTER code didn't mask stray
 406                  * interrupts.
 407                  */
 408                 ia64_intr_mask((void *)(uintptr_t)vector);
 409                 log(LOG_ERR, "stray irq%u\n", i->irq);
 410         }
 411 }
 412
 413 #ifdef DDB
 414
 415 static void
 416 db_print_vector(u_int vector, int always)
 417 {
 418         struct ia64_intr *i;
 419
 420         i = ia64_intrs[vector];
 421         if (i != NULL) {
 422                 db_printf("vector %u (%p): ", vector, i);
 423                 sapic_print(i->sapic, i->irq);
 424         } else if (always)
 425                 db_printf("vector %u: unassigned\n", vector);
 426 }
 427
 428 DB_SHOW_COMMAND(vector, db_show_vector)
 429 {
 430         u_int vector;
 431
 432         if (have_addr) {
 433                 vector = ((addr >> 4) % 16) * 10 + (addr % 16);
 434                 if (vector >= 256)
 435                         db_printf("error: vector %u not in range [0..255]\n",
 436                             vector);
 437                 else
 438                         db_print_vector(vector, 1);
 439         } else {
 440                 for (vector = 0; vector < 256; vector++)
 441                         db_print_vector(vector, 0);
 442         }
 443 }
 444
 445 #endif