sys/ia64/ia64/interrupt.c

   1 /* $FreeBSD$ */
   2 /* $NetBSD: interrupt.c,v 1.23 1998/02/24 07:38:01 thorpej Exp $ */
   3
   4 /*-
   5  * Copyright (c) 1994, 1995, 1996 Carnegie-Mellon University.
   6  * All rights reserved.
   7  *
   8  * Authors: Keith Bostic, Chris G. Demetriou
   9  *
  10  * Permission to use, copy, modify and distribute this software and
  11  * its documentation is hereby granted, provided that both the copyright
  12  * notice and this permission notice appear in all copies of the
  13  * software, derivative works or modified versions, and any portions
  14  * thereof, and that both notices appear in supporting documentation.
  15  *
  16  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  17  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
  18  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  19  *
  20  * Carnegie Mellon requests users of this software to return to
  21  *
  22  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  23  *  School of Computer Science
  24  *  Carnegie Mellon University
  25  *  Pittsburgh PA 15213-3890
  26  *
  27  * any improvements or extensions that they make and grant Carnegie the
  28  * rights to redistribute these changes.
  29  */
  30 /*-
  31  * Additional Copyright (c) 1997 by Matthew Jacob for NASA/Ames Research Center.
  32  * Redistribute and modify at will, leaving only this additional copyright
  33  * notice.
  34  */
  35
  36 #include "opt_ddb.h"
  37
  38 #include <sys/cdefs.h>                  /* RCS ID & Copyright macro defns */
  39
  40 #include <sys/param.h>
  41 #include <sys/systm.h>
  42 #include <sys/kernel.h>
  43 #include <sys/proc.h>
  44 #include <sys/vmmeter.h>
  45 #include <sys/bus.h>
  46 #include <sys/malloc.h>
  47 #include <sys/ktr.h>
  48 #include <sys/lock.h>
  49 #include <sys/mutex.h>
  50 #include <sys/sched.h>
  51 #include <sys/smp.h>
  52 #include <sys/sysctl.h>
  53 #include <sys/syslog.h>
  54
  55 #include <machine/clock.h>
  56 #include <machine/cpu.h>
  57 #include <machine/fpu.h>
  58 #include <machine/frame.h>
  59 #include <machine/intr.h>
  60 #include <machine/md_var.h>
  61 #include <machine/pcb.h>
  62 #include <machine/reg.h>
  63 #include <machine/sapicvar.h>
  64 #include <machine/smp.h>
  65
  66 #ifdef EVCNT_COUNTERS
  67 struct evcnt clock_intr_evcnt;  /* event counter for clock intrs. */
  68 #else
  69 #include <sys/interrupt.h>
  70 #include <machine/intrcnt.h>
  71 #endif
  72
  73 #ifdef DDB
  74 #include <ddb/ddb.h>
  75 #endif
  76
  77 static void ia64_dispatch_intr(void *, u_int);
  78
  79 static void
  80 dummy_perf(unsigned long vector, struct trapframe *tf)
  81 {
  82         printf("performance interrupt!\n");
  83 }
  84
  85 void (*perf_irq)(unsigned long, struct trapframe *) = dummy_perf;
  86
  87 static unsigned int ints[MAXCPU];
  88 SYSCTL_OPAQUE(_debug, OID_AUTO, ints, CTLFLAG_RW, &ints, sizeof(ints), "IU",
  89     "");
  90
  91 static unsigned int clks[MAXCPU];
  92 #ifdef SMP
  93 SYSCTL_OPAQUE(_debug, OID_AUTO, clks, CTLFLAG_RW, &clks, sizeof(clks), "IU",
  94     "");
  95 #else
  96 SYSCTL_INT(_debug, OID_AUTO, clks, CTLFLAG_RW, clks, 0, "");
  97 #endif
  98
  99 #ifdef SMP
 100 static unsigned int asts[MAXCPU];
 101 SYSCTL_OPAQUE(_debug, OID_AUTO, asts, CTLFLAG_RW, &asts, sizeof(asts), "IU",
 102     "");
 103
 104 static unsigned int rdvs[MAXCPU];
 105 SYSCTL_OPAQUE(_debug, OID_AUTO, rdvs, CTLFLAG_RW, &rdvs, sizeof(rdvs), "IU",
 106     "");
 107 #endif
 108
 109 SYSCTL_NODE(_debug, OID_AUTO, clock, CTLFLAG_RW, 0, "clock statistics");
 110
 111 static int adjust_edges = 0;
 112 SYSCTL_INT(_debug_clock, OID_AUTO, adjust_edges, CTLFLAG_RD,
 113     &adjust_edges, 0, "Number of times ITC got more than 12.5% behind");
 114
 115 static int adjust_excess = 0;
 116 SYSCTL_INT(_debug_clock, OID_AUTO, adjust_excess, CTLFLAG_RD,
 117     &adjust_excess, 0, "Total number of ignored ITC interrupts");
 118
 119 static int adjust_lost = 0;
 120 SYSCTL_INT(_debug_clock, OID_AUTO, adjust_lost, CTLFLAG_RD,
 121     &adjust_lost, 0, "Total number of lost ITC interrupts");
 122
 123 static int adjust_ticks = 0;
 124 SYSCTL_INT(_debug_clock, OID_AUTO, adjust_ticks, CTLFLAG_RD,
 125     &adjust_ticks, 0, "Total number of ITC interrupts with adjustment");
 126
 127 void
 128 interrupt(struct trapframe *tf)
 129 {
 130         struct thread *td;
 131         volatile struct ia64_interrupt_block *ib = IA64_INTERRUPT_BLOCK;
 132         uint64_t adj, clk, itc;
 133         int64_t delta;
 134         u_int vector;
 135         int count;
 136         uint8_t inta;
 137
 138         ia64_set_fpsr(IA64_FPSR_DEFAULT);
 139
 140         td = curthread;
 141
 142         vector = tf->tf_special.ifa;
 143
 144  next:
 145         /*
 146          * Handle ExtINT interrupts by generating an INTA cycle to
 147          * read the vector.
 148          * IPI_STOP_HARD is mapped to IPI_STOP so it is not necessary
 149          * to add it to this switch-like construct.
 150          */
 151         if (vector == 0) {
 152                 inta = ib->ib_inta;
 153                 printf("ExtINT interrupt: vector=%u\n", (int)inta);
 154                 if (inta == 15) {
 155                         __asm __volatile("mov cr.eoi = r0;; srlz.d");
 156                         goto stray;
 157                 }
 158                 vector = (int)inta;
 159         } else if (vector == 15)
 160                 goto stray;
 161
 162         if (vector == CLOCK_VECTOR) {/* clock interrupt */
 163                 /* CTR0(KTR_INTR, "clock interrupt"); */
 164
 165                 itc = ia64_get_itc();
 166
 167                 PCPU_INC(cnt.v_intr);
 168 #ifdef EVCNT_COUNTERS
 169                 clock_intr_evcnt.ev_count++;
 170 #else
 171                 intrcnt[INTRCNT_CLOCK]++;
 172 #endif
 173                 clks[PCPU_GET(cpuid)]++;
 174
 175                 critical_enter();
 176
 177                 adj = PCPU_GET(clockadj);
 178                 clk = PCPU_GET(clock);
 179                 delta = itc - clk;
 180                 count = 0;
 181                 while (delta >= ia64_clock_reload) {
 182                         /* Only the BSP runs the real clock */
 183                         if (PCPU_GET(cpuid) == 0)
 184                                 hardclock(TRAPF_USERMODE(tf), TRAPF_PC(tf));
 185                         else
 186                                 hardclock_cpu(TRAPF_USERMODE(tf));
 187                         if (profprocs != 0)
 188                                 profclock(TRAPF_USERMODE(tf), TRAPF_PC(tf));
 189                         statclock(TRAPF_USERMODE(tf));
 190                         delta -= ia64_clock_reload;
 191                         clk += ia64_clock_reload;
 192                         if (adj != 0)
 193                                 adjust_ticks++;
 194                         count++;
 195                 }
 196                 ia64_set_itm(ia64_get_itc() + ia64_clock_reload - adj);
 197                 if (count > 0) {
 198                         adjust_lost += count - 1;
 199                         if (delta > (ia64_clock_reload >> 3)) {
 200                                 if (adj == 0)
 201                                         adjust_edges++;
 202                                 adj = ia64_clock_reload >> 4;
 203                         } else
 204                                 adj = 0;
 205                 } else {
 206                         adj = 0;
 207                         adjust_excess++;
 208                 }
 209                 PCPU_SET(clock, clk);
 210                 PCPU_SET(clockadj, adj);
 211                 critical_exit();
 212                 ia64_srlz_d();
 213
 214 #ifdef SMP
 215         } else if (vector == ipi_vector[IPI_AST]) {
 216                 asts[PCPU_GET(cpuid)]++;
 217                 CTR1(KTR_SMP, "IPI_AST, cpuid=%d", PCPU_GET(cpuid));
 218         } else if (vector == ipi_vector[IPI_HIGH_FP]) {
 219                 struct thread *thr = PCPU_GET(fpcurthread);
 220                 if (thr != NULL) {
 221                         mtx_lock_spin(&thr->td_md.md_highfp_mtx);
 222                         save_high_fp(&thr->td_pcb->pcb_high_fp);
 223                         thr->td_pcb->pcb_fpcpu = NULL;
 224                         PCPU_SET(fpcurthread, NULL);
 225                         mtx_unlock_spin(&thr->td_md.md_highfp_mtx);
 226                 }
 227         } else if (vector == ipi_vector[IPI_RENDEZVOUS]) {
 228                 rdvs[PCPU_GET(cpuid)]++;
 229                 CTR1(KTR_SMP, "IPI_RENDEZVOUS, cpuid=%d", PCPU_GET(cpuid));
 230                 enable_intr();
 231                 smp_rendezvous_action();
 232                 disable_intr();
 233         } else if (vector == ipi_vector[IPI_STOP]) {
 234                 cpumask_t mybit = PCPU_GET(cpumask);
 235
 236                 savectx(PCPU_PTR(pcb));
 237                 atomic_set_int(&stopped_cpus, mybit);
 238                 while ((started_cpus & mybit) == 0)
 239                         cpu_spinwait();
 240                 atomic_clear_int(&started_cpus, mybit);
 241                 atomic_clear_int(&stopped_cpus, mybit);
 242         } else if (vector == ipi_vector[IPI_PREEMPT]) {
 243                 CTR1(KTR_SMP, "IPI_PREEMPT, cpuid=%d", PCPU_GET(cpuid));
 244                 __asm __volatile("mov cr.eoi = r0;; srlz.d");
 245                 enable_intr();
 246                 sched_preempt(curthread);
 247                 disable_intr();
 248                 goto stray;
 249 #endif
 250         } else {
 251                 ints[PCPU_GET(cpuid)]++;
 252                 atomic_add_int(&td->td_intr_nesting_level, 1);
 253                 ia64_dispatch_intr(tf, vector);
 254                 atomic_subtract_int(&td->td_intr_nesting_level, 1);
 255         }
 256
 257         __asm __volatile("mov cr.eoi = r0;; srlz.d");
 258         vector = ia64_get_ivr();
 259         if (vector != 15)
 260                 goto next;
 261
 262 stray:
 263         if (TRAPF_USERMODE(tf)) {
 264                 enable_intr();
 265                 userret(td, tf);
 266                 mtx_assert(&Giant, MA_NOTOWNED);
 267                 do_ast(tf);
 268         }
 269 }
 270
 271 /*
 272  * Hardware irqs have vectors starting at this offset.
 273  */
 274 #define IA64_HARDWARE_IRQ_BASE  0x20
 275
 276 struct ia64_intr {
 277         struct intr_event *event;       /* interrupt event */
 278         volatile long *cntp;            /* interrupt counter */
 279         struct sapic *sapic;
 280         u_int   irq;
 281 };
 282
 283 static struct ia64_intr *ia64_intrs[256];
 284
 285 static void
 286 ia64_intr_eoi(void *arg)
 287 {
 288         u_int vector = (uintptr_t)arg;
 289         struct ia64_intr *i;
 290
 291         i = ia64_intrs[vector];
 292         if (i != NULL)
 293                 sapic_eoi(i->sapic, vector);
 294 }
 295
 296 static void
 297 ia64_intr_mask(void *arg)
 298 {
 299         u_int vector = (uintptr_t)arg;
 300         struct ia64_intr *i;
 301
 302         i = ia64_intrs[vector];
 303         if (i != NULL) {
 304                 sapic_mask(i->sapic, i->irq);
 305                 sapic_eoi(i->sapic, vector);
 306         }
 307 }
 308
 309 static void
 310 ia64_intr_unmask(void *arg)
 311 {
 312         u_int vector = (uintptr_t)arg;
 313         struct ia64_intr *i;
 314
 315         i = ia64_intrs[vector];
 316         if (i != NULL)
 317                 sapic_unmask(i->sapic, i->irq);
 318 }
 319
 320 int
 321 ia64_setup_intr(const char *name, int irq, driver_filter_t filter,
 322     driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep)
 323 {
 324         struct ia64_intr *i;
 325         struct sapic *sa;
 326         char *intrname;
 327         u_int vector;
 328         int error;
 329
 330         /* Get the I/O SAPIC that corresponds to the IRQ. */
 331         sa = sapic_lookup(irq);
 332         if (sa == NULL)
 333                 return (EINVAL);
 334
 335         /*
 336          * XXX - There's a priority implied by the choice of vector.
 337          * We should therefore relate the vector to the interrupt type.
 338          */
 339         vector = irq + IA64_HARDWARE_IRQ_BASE;
 340
 341         i = ia64_intrs[vector];
 342         if (i == NULL) {
 343                 i = malloc(sizeof(struct ia64_intr), M_DEVBUF, M_NOWAIT);
 344                 if (i == NULL)
 345                         return (ENOMEM);
 346
 347                 error = intr_event_create(&i->event, (void *)(uintptr_t)vector,
 348                     0, irq, ia64_intr_mask, ia64_intr_unmask, ia64_intr_eoi,
 349                     NULL, "irq%u:", irq);
 350                 if (error) {
 351                         free(i, M_DEVBUF);
 352                         return (error);
 353                 }
 354
 355                 if (!atomic_cmpset_ptr(&ia64_intrs[vector], NULL, i)) {
 356                         intr_event_destroy(i->event);
 357                         free(i, M_DEVBUF);
 358                         i = ia64_intrs[vector];
 359                 } else {
 360                         i->sapic = sa;
 361                         i->irq = irq;
 362
 363                         i->cntp = intrcnt + irq + INTRCNT_ISA_IRQ;
 364                         if (name != NULL && *name != '\0') {
 365                                 /* XXX needs abstraction. Too error prone. */
 366                                 intrname = intrnames +
 367                                     (irq + INTRCNT_ISA_IRQ) * INTRNAME_LEN;
 368                                 memset(intrname, ' ', INTRNAME_LEN - 1);
 369                                 bcopy(name, intrname, strlen(name));
 370                         }
 371
 372                         sapic_enable(i->sapic, irq, vector);
 373                 }
 374         }
 375
 376         error = intr_event_add_handler(i->event, name, filter, handler, arg,
 377             intr_priority(flags), flags, cookiep);
 378         return (error);
 379 }
 380
 381 int
 382 ia64_teardown_intr(void *cookie)
 383 {
 384
 385         return (intr_event_remove_handler(cookie));
 386 }
 387
 388 static void
 389 ia64_dispatch_intr(void *frame, u_int vector)
 390 {
 391         struct ia64_intr *i;
 392         struct intr_event *ie;                  /* our interrupt event */
 393
 394         /*
 395          * Find the interrupt thread for this vector.
 396          */
 397         i = ia64_intrs[vector];
 398         KASSERT(i != NULL, ("%s: unassigned vector", __func__));
 399
 400         (*i->cntp)++;
 401
 402         ie = i->event;
 403         KASSERT(ie != NULL, ("%s: interrupt without event", __func__));
 404
 405         if (intr_event_handle(ie, frame) != 0) {
 406                 /*
 407                  * XXX: The pre-INTR_FILTER code didn't mask stray
 408                  * interrupts.
 409                  */
 410                 ia64_intr_mask((void *)(uintptr_t)vector);
 411                 log(LOG_ERR, "stray irq%u\n", i->irq);
 412         }
 413 }
 414
 415 #ifdef DDB
 416
 417 static void
 418 db_print_vector(u_int vector, int always)
 419 {
 420         struct ia64_intr *i;
 421
 422         i = ia64_intrs[vector];
 423         if (i != NULL) {
 424                 db_printf("vector %u (%p): ", vector, i);
 425                 sapic_print(i->sapic, i->irq);
 426         } else if (always)
 427                 db_printf("vector %u: unassigned\n", vector);
 428 }
 429
 430 DB_SHOW_COMMAND(vector, db_show_vector)
 431 {
 432         u_int vector;
 433
 434         if (have_addr) {
 435                 vector = ((addr >> 4) % 16) * 10 + (addr % 16);
 436                 if (vector >= 256)
 437                         db_printf("error: vector %u not in range [0..255]\n",
 438                             vector);
 439                 else
 440                         db_print_vector(vector, 1);
 441         } else {
 442                 for (vector = 0; vector < 256; vector++)
 443                         db_print_vector(vector, 0);
 444         }
 445 }
 446
 447 #endif