sys/ia64/ia64/machdep.c

   1 /*-
   2  * Copyright (c) 2003,2004 Marcel Moolenaar
   3  * Copyright (c) 2000,2001 Doug Rabson
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  */
  27
  28 #include <sys/cdefs.h>
  29 __FBSDID("$FreeBSD$");
  30
  31 #include "opt_compat.h"
  32 #include "opt_ddb.h"
  33 #include "opt_kstack_pages.h"
  34 #include "opt_msgbuf.h"
  35 #include "opt_sched.h"
  36
  37 #include <sys/param.h>
  38 #include <sys/proc.h>
  39 #include <sys/systm.h>
  40 #include <sys/bio.h>
  41 #include <sys/buf.h>
  42 #include <sys/bus.h>
  43 #include <sys/cons.h>
  44 #include <sys/cpu.h>
  45 #include <sys/eventhandler.h>
  46 #include <sys/exec.h>
  47 #include <sys/imgact.h>
  48 #include <sys/kdb.h>
  49 #include <sys/kernel.h>
  50 #include <sys/linker.h>
  51 #include <sys/lock.h>
  52 #include <sys/malloc.h>
  53 #include <sys/mbuf.h>
  54 #include <sys/msgbuf.h>
  55 #include <sys/pcpu.h>
  56 #include <sys/ptrace.h>
  57 #include <sys/random.h>
  58 #include <sys/reboot.h>
  59 #include <sys/sched.h>
  60 #include <sys/signalvar.h>
  61 #include <sys/syscall.h>
  62 #include <sys/syscallsubr.h>
  63 #include <sys/sysctl.h>
  64 #include <sys/sysproto.h>
  65 #include <sys/ucontext.h>
  66 #include <sys/uio.h>
  67 #include <sys/uuid.h>
  68 #include <sys/vmmeter.h>
  69 #include <sys/vnode.h>
  70
  71 #include <ddb/ddb.h>
  72
  73 #include <net/netisr.h>
  74
  75 #include <vm/vm.h>
  76 #include <vm/vm_extern.h>
  77 #include <vm/vm_kern.h>
  78 #include <vm/vm_page.h>
  79 #include <vm/vm_map.h>
  80 #include <vm/vm_object.h>
  81 #include <vm/vm_pager.h>
  82
  83 #include <machine/bootinfo.h>
  84 #include <machine/cpu.h>
  85 #include <machine/efi.h>
  86 #include <machine/elf.h>
  87 #include <machine/fpu.h>
  88 #include <machine/intr.h>
  89 #include <machine/mca.h>
  90 #include <machine/md_var.h>
  91 #include <machine/mutex.h>
  92 #include <machine/pal.h>
  93 #include <machine/pcb.h>
  94 #include <machine/reg.h>
  95 #include <machine/sal.h>
  96 #include <machine/sigframe.h>
  97 #ifdef SMP
  98 #include <machine/smp.h>
  99 #endif
 100 #include <machine/unwind.h>
 101 #include <machine/vmparam.h>
 102
 103 SYSCTL_NODE(_hw, OID_AUTO, freq, CTLFLAG_RD, 0, "");
 104 SYSCTL_NODE(_machdep, OID_AUTO, cpu, CTLFLAG_RD, 0, "");
 105
 106 static u_int bus_freq;
 107 SYSCTL_UINT(_hw_freq, OID_AUTO, bus, CTLFLAG_RD, &bus_freq, 0,
 108     "Bus clock frequency");
 109
 110 static u_int cpu_freq;
 111 SYSCTL_UINT(_hw_freq, OID_AUTO, cpu, CTLFLAG_RD, &cpu_freq, 0,
 112     "CPU clock frequency");
 113
 114 static u_int itc_freq;
 115 SYSCTL_UINT(_hw_freq, OID_AUTO, itc, CTLFLAG_RD, &itc_freq, 0,
 116     "ITC frequency");
 117
 118 int cold = 1;
 119
 120 u_int64_t pa_bootinfo;
 121 struct bootinfo bootinfo;
 122
 123 struct pcpu pcpu0;
 124
 125 extern u_int64_t kernel_text[], _end[];
 126
 127 extern u_int64_t ia64_gateway_page[];
 128 extern u_int64_t break_sigtramp[];
 129 extern u_int64_t epc_sigtramp[];
 130
 131 struct fpswa_iface *fpswa_iface;
 132
 133 u_int64_t ia64_pal_base;
 134 u_int64_t ia64_port_base;
 135
 136 u_int64_t ia64_lapic_addr = PAL_PIB_DEFAULT_ADDR;
 137
 138 struct ia64_pib *ia64_pib;
 139
 140 static int ia64_sync_icache_needed;
 141
 142 char machine[] = MACHINE;
 143 SYSCTL_STRING(_hw, HW_MACHINE, machine, CTLFLAG_RD, machine, 0, "");
 144
 145 static char cpu_model[64];
 146 SYSCTL_STRING(_hw, HW_MODEL, model, CTLFLAG_RD, cpu_model, 0,
 147     "The CPU model name");
 148
 149 static char cpu_family[64];
 150 SYSCTL_STRING(_hw, OID_AUTO, family, CTLFLAG_RD, cpu_family, 0,
 151     "The CPU family name");
 152
 153 #ifdef DDB
 154 extern vm_offset_t ksym_start, ksym_end;
 155 #endif
 156
 157
 158 struct msgbuf *msgbufp = NULL;
 159
 160 /* Other subsystems (e.g., ACPI) can hook this later. */
 161 void (*cpu_idle_hook)(void) = NULL;
 162
 163 long Maxmem = 0;
 164 long realmem = 0;
 165
 166 #define PHYSMAP_SIZE    (2 * VM_PHYSSEG_MAX)
 167
 168 vm_paddr_t phys_avail[PHYSMAP_SIZE + 2];
 169
 170 /* must be 2 less so 0 0 can signal end of chunks */
 171 #define PHYS_AVAIL_ARRAY_END ((sizeof(phys_avail) / sizeof(vm_offset_t)) - 2)
 172
 173 struct kva_md_info kmi;
 174
 175 #define Mhz     1000000L
 176 #define Ghz     (1000L*Mhz)
 177
 178 #define SN_SAL_SET_OS_FEATURE_SET       0x02000066
 179
 180 #define OSF_ACPI_ENABLE         2
 181 #define OSF_PCISEGMENT_ENABLE   3
 182
 183 #include <contrib/dev/acpica/include/acpi.h>
 184 #include <contrib/dev/acpica/include/actables.h>
 185 #include <dev/acpica/acpivar.h>
 186
 187 static void
 188 srat_dump_entry(ACPI_SUBTABLE_HEADER *entry, void *arg)
 189 {
 190         ACPI_SRAT_CPU_AFFINITY *cpu;
 191         ACPI_SRAT_MEM_AFFINITY *mem;
 192         uint32_t domain;
 193         uint16_t sapicid;
 194
 195         switch (entry->Type) {
 196         case ACPI_SRAT_TYPE_CPU_AFFINITY:
 197                 cpu = (ACPI_SRAT_CPU_AFFINITY *)entry;
 198                 domain = cpu->ProximityDomainLo |
 199                     cpu->ProximityDomainHi[0] << 8 |
 200                     cpu->ProximityDomainHi[1] << 16 |
 201                     cpu->ProximityDomainHi[2] << 24;
 202                 sapicid = (cpu->ApicId << 8) | cpu->LocalSapicEid;
 203                 printf("SRAT: Sapic ID %u domain %d: %s\n", sapicid, domain,
 204                     (cpu->Flags & ACPI_SRAT_CPU_ENABLED) ? "enabled" :
 205                     "disabled");
 206                 break;
 207         case ACPI_SRAT_TYPE_MEMORY_AFFINITY:
 208                 mem = (ACPI_SRAT_MEM_AFFINITY *)entry;
 209                 printf("SRAT: memory domain %d addr %lx len %lx: %s\n",
 210                     mem->ProximityDomain, mem->BaseAddress, mem->Length,
 211                     (mem->Flags & ACPI_SRAT_MEM_ENABLED) ? "enabled" :
 212                     "disabled");
 213                 break;
 214         default:
 215                 printf("SRAT: unknown type (%u)\n", entry->Type);
 216                 break;
 217         }
 218 }
 219
 220 static void
 221 check_sn_sal(void)
 222 {
 223         struct ia64_sal_result r;
 224         ACPI_TABLE_HEADER *tbl;
 225         void *ptr;
 226
 227         r = ia64_sal_entry(SAL_SGISN_SN_INFO, 0, 0, 0, 0, 0, 0, 0);
 228         printf("XXX: %s: stat=%ld, res0=%#lx, res1=%#lx, res2=%#lx\n",
 229             __func__, r.sal_status, r.sal_result[0], r.sal_result[1],
 230             r.sal_result[2]);
 231         if (r.sal_status != 0)
 232                 return;
 233
 234         tbl = ptr = acpi_find_table(ACPI_SIG_SRAT);
 235         printf("XXX: %s: SRAT table at %p\n", __func__, ptr);
 236         acpi_walk_subtables((char *)ptr + sizeof(ACPI_TABLE_SRAT),
 237             (char *)ptr + tbl->Length, srat_dump_entry, ptr);
 238         tbl = acpi_find_table(ACPI_SIG_SLIT);
 239         printf("XXX: %s: SLIT table at %p\n", __func__, tbl);
 240 }
 241
 242 static void
 243 identifycpu(void)
 244 {
 245         char vendor[17];
 246         char *family_name, *model_name;
 247         u_int64_t features, tmp;
 248         int number, revision, model, family, archrev;
 249
 250         /*
 251          * Assumes little-endian.
 252          */
 253         *(u_int64_t *) &vendor[0] = ia64_get_cpuid(0);
 254         *(u_int64_t *) &vendor[8] = ia64_get_cpuid(1);
 255         vendor[16] = '\0';
 256
 257         tmp = ia64_get_cpuid(3);
 258         number = (tmp >> 0) & 0xff;
 259         revision = (tmp >> 8) & 0xff;
 260         model = (tmp >> 16) & 0xff;
 261         family = (tmp >> 24) & 0xff;
 262         archrev = (tmp >> 32) & 0xff;
 263
 264         family_name = model_name = "unknown";
 265         switch (family) {
 266         case 0x07:
 267                 family_name = "Itanium";
 268                 model_name = "Merced";
 269                 break;
 270         case 0x1f:
 271                 family_name = "Itanium 2";
 272                 switch (model) {
 273                 case 0x00:
 274                         model_name = "McKinley";
 275                         break;
 276                 case 0x01:
 277                         /*
 278                          * Deerfield is a low-voltage variant based on the
 279                          * Madison core. We need circumstantial evidence
 280                          * (i.e. the clock frequency) to identify those.
 281                          * Allow for roughly 1% error margin.
 282                          */
 283                         if (cpu_freq > 990 && cpu_freq < 1010)
 284                                 model_name = "Deerfield";
 285                         else
 286                                 model_name = "Madison";
 287                         break;
 288                 case 0x02:
 289                         model_name = "Madison II";
 290                         break;
 291                 }
 292                 break;
 293         case 0x20:
 294                 ia64_sync_icache_needed = 1;
 295
 296                 family_name = "Itanium 2";
 297                 switch (model) {
 298                 case 0x00:
 299                         model_name = "Montecito";
 300                         break;
 301                 }
 302                 break;
 303         }
 304         snprintf(cpu_family, sizeof(cpu_family), "%s", family_name);
 305         snprintf(cpu_model, sizeof(cpu_model), "%s", model_name);
 306
 307         features = ia64_get_cpuid(4);
 308
 309         printf("CPU: %s (", model_name);
 310         if (cpu_freq)
 311                 printf("%u Mhz ", cpu_freq);
 312         printf("%s)\n", family_name);
 313         printf("  Origin = \"%s\"  Revision = %d\n", vendor, revision);
 314         printf("  Features = 0x%b\n", (u_int32_t) features,
 315             "\020"
 316             "\001LB"    /* long branch (brl) instruction. */
 317             "\002SD"    /* Spontaneous deferral. */
 318             "\003AO"    /* 16-byte atomic operations (ld, st, cmpxchg). */ );
 319 }
 320
 321 static void
 322 cpu_startup(void *dummy)
 323 {
 324         char nodename[16];
 325         struct pcpu *pc;
 326         struct pcpu_stats *pcs;
 327
 328         /*
 329          * Good {morning,afternoon,evening,night}.
 330          */
 331         identifycpu();
 332
 333 #ifdef PERFMON
 334         perfmon_init();
 335 #endif
 336         printf("real memory  = %ld (%ld MB)\n", ia64_ptob(Maxmem),
 337             ia64_ptob(Maxmem) / 1048576);
 338         realmem = Maxmem;
 339
 340         /*
 341          * Display any holes after the first chunk of extended memory.
 342          */
 343         if (bootverbose) {
 344                 int indx;
 345
 346                 printf("Physical memory chunk(s):\n");
 347                 for (indx = 0; phys_avail[indx + 1] != 0; indx += 2) {
 348                         long size1 = phys_avail[indx + 1] - phys_avail[indx];
 349
 350                         printf("0x%08lx - 0x%08lx, %ld bytes (%ld pages)\n",
 351                             phys_avail[indx], phys_avail[indx + 1] - 1, size1,
 352                             size1 >> PAGE_SHIFT);
 353                 }
 354         }
 355
 356         vm_ksubmap_init(&kmi);
 357
 358         printf("avail memory = %ld (%ld MB)\n", ptoa(cnt.v_free_count),
 359             ptoa(cnt.v_free_count) / 1048576);
 360
 361         if (fpswa_iface == NULL)
 362                 printf("Warning: no FPSWA package supplied\n");
 363         else
 364                 printf("FPSWA Revision = 0x%lx, Entry = %p\n",
 365                     (long)fpswa_iface->if_rev, (void *)fpswa_iface->if_fpswa);
 366
 367         /*
 368          * Set up buffers, so they can be used to read disk labels.
 369          */
 370         bufinit();
 371         vm_pager_bufferinit();
 372
 373         /*
 374          * Traverse the MADT to discover IOSAPIC and Local SAPIC
 375          * information.
 376          */
 377         ia64_probe_sapics();
 378         ia64_pib = pmap_mapdev(ia64_lapic_addr, sizeof(*ia64_pib));
 379
 380         ia64_mca_init();
 381
 382         /*
 383          * Create sysctl tree for per-CPU information.
 384          */
 385         SLIST_FOREACH(pc, &cpuhead, pc_allcpu) {
 386                 snprintf(nodename, sizeof(nodename), "%u", pc->pc_cpuid);
 387                 sysctl_ctx_init(&pc->pc_md.sysctl_ctx);
 388                 pc->pc_md.sysctl_tree = SYSCTL_ADD_NODE(&pc->pc_md.sysctl_ctx,
 389                     SYSCTL_STATIC_CHILDREN(_machdep_cpu), OID_AUTO, nodename,
 390                     CTLFLAG_RD, NULL, "");
 391                 if (pc->pc_md.sysctl_tree == NULL)
 392                         continue;
 393
 394                 pcs = &pc->pc_md.stats;
 395
 396                 SYSCTL_ADD_ULONG(&pc->pc_md.sysctl_ctx,
 397                     SYSCTL_CHILDREN(pc->pc_md.sysctl_tree), OID_AUTO,
 398                     "nasts", CTLFLAG_RD, &pcs->pcs_nasts,
 399                     "Number of IPI_AST interrupts");
 400
 401                 SYSCTL_ADD_ULONG(&pc->pc_md.sysctl_ctx,
 402                     SYSCTL_CHILDREN(pc->pc_md.sysctl_tree), OID_AUTO,
 403                     "nclks", CTLFLAG_RD, &pcs->pcs_nclks,
 404                     "Number of clock interrupts");
 405
 406                 SYSCTL_ADD_ULONG(&pc->pc_md.sysctl_ctx,
 407                     SYSCTL_CHILDREN(pc->pc_md.sysctl_tree), OID_AUTO,
 408                     "nextints", CTLFLAG_RD, &pcs->pcs_nextints,
 409                     "Number of ExtINT interrupts");
 410
 411                 SYSCTL_ADD_ULONG(&pc->pc_md.sysctl_ctx,
 412                     SYSCTL_CHILDREN(pc->pc_md.sysctl_tree), OID_AUTO,
 413                     "nhighfps", CTLFLAG_RD, &pcs->pcs_nhighfps,
 414                     "Number of IPI_HIGH_FP interrupts");
 415
 416                 SYSCTL_ADD_ULONG(&pc->pc_md.sysctl_ctx,
 417                     SYSCTL_CHILDREN(pc->pc_md.sysctl_tree), OID_AUTO,
 418                     "nhwints", CTLFLAG_RD, &pcs->pcs_nhwints,
 419                     "Number of hardware (device) interrupts");
 420
 421                 SYSCTL_ADD_ULONG(&pc->pc_md.sysctl_ctx,
 422                     SYSCTL_CHILDREN(pc->pc_md.sysctl_tree), OID_AUTO,
 423                     "npreempts", CTLFLAG_RD, &pcs->pcs_npreempts,
 424                     "Number of IPI_PREEMPT interrupts");
 425
 426                 SYSCTL_ADD_ULONG(&pc->pc_md.sysctl_ctx,
 427                     SYSCTL_CHILDREN(pc->pc_md.sysctl_tree), OID_AUTO,
 428                     "nrdvs", CTLFLAG_RD, &pcs->pcs_nrdvs,
 429                     "Number of IPI_RENDEZVOUS interrupts");
 430
 431                 SYSCTL_ADD_ULONG(&pc->pc_md.sysctl_ctx,
 432                     SYSCTL_CHILDREN(pc->pc_md.sysctl_tree), OID_AUTO,
 433                     "nstops", CTLFLAG_RD, &pcs->pcs_nstops,
 434                     "Number of IPI_STOP interrupts");
 435
 436                 SYSCTL_ADD_ULONG(&pc->pc_md.sysctl_ctx,
 437                     SYSCTL_CHILDREN(pc->pc_md.sysctl_tree), OID_AUTO,
 438                     "nstrays", CTLFLAG_RD, &pcs->pcs_nstrays,
 439                     "Number of stray interrupts");
 440         }
 441 }
 442 SYSINIT(cpu_startup, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL);
 443
 444 void
 445 cpu_flush_dcache(void *ptr, size_t len)
 446 {
 447         vm_offset_t lim, va;
 448
 449         va = (uintptr_t)ptr & ~31;
 450         lim = (uintptr_t)ptr + len;
 451         while (va < lim) {
 452                 ia64_fc(va);
 453                 va += 32;
 454         }
 455
 456         ia64_srlz_d();
 457 }
 458
 459 /* Get current clock frequency for the given cpu id. */
 460 int
 461 cpu_est_clockrate(int cpu_id, uint64_t *rate)
 462 {
 463
 464         if (pcpu_find(cpu_id) == NULL || rate == NULL)
 465                 return (EINVAL);
 466         *rate = (u_long)cpu_freq * 1000000ul;
 467         return (0);
 468 }
 469
 470 void
 471 cpu_halt()
 472 {
 473
 474         efi_reset_system();
 475 }
 476
 477 void
 478 cpu_idle(int busy)
 479 {
 480         struct ia64_pal_result res;
 481
 482         if (cpu_idle_hook != NULL)
 483                 (*cpu_idle_hook)();
 484         else
 485                 res = ia64_call_pal_static(PAL_HALT_LIGHT, 0, 0, 0);
 486 }
 487
 488 int
 489 cpu_idle_wakeup(int cpu)
 490 {
 491
 492         return (0);
 493 }
 494
 495 void
 496 cpu_reset()
 497 {
 498
 499         efi_reset_system();
 500 }
 501
 502 void
 503 cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx)
 504 {
 505         struct pcb *oldpcb, *newpcb;
 506
 507         oldpcb = old->td_pcb;
 508 #ifdef COMPAT_FREEBSD32
 509         ia32_savectx(oldpcb);
 510 #endif
 511         if (PCPU_GET(fpcurthread) == old)
 512                 old->td_frame->tf_special.psr |= IA64_PSR_DFH;
 513         if (!savectx(oldpcb)) {
 514                 atomic_store_rel_ptr(&old->td_lock, mtx);
 515
 516                 newpcb = new->td_pcb;
 517                 oldpcb->pcb_current_pmap =
 518                     pmap_switch(newpcb->pcb_current_pmap);
 519
 520 #if defined(SCHED_ULE) && defined(SMP)
 521                 while (atomic_load_acq_ptr(&new->td_lock) == &blocked_lock)
 522                         cpu_spinwait();
 523 #endif
 524
 525                 PCPU_SET(curthread, new);
 526
 527 #ifdef COMPAT_FREEBSD32
 528                 ia32_restorectx(newpcb);
 529 #endif
 530
 531                 if (PCPU_GET(fpcurthread) == new)
 532                         new->td_frame->tf_special.psr &= ~IA64_PSR_DFH;
 533                 restorectx(newpcb);
 534                 /* We should not get here. */
 535                 panic("cpu_switch: restorectx() returned");
 536                 /* NOTREACHED */
 537         }
 538 }
 539
 540 void
 541 cpu_throw(struct thread *old __unused, struct thread *new)
 542 {
 543         struct pcb *newpcb;
 544
 545         newpcb = new->td_pcb;
 546         (void)pmap_switch(newpcb->pcb_current_pmap);
 547
 548 #if defined(SCHED_ULE) && defined(SMP)
 549         while (atomic_load_acq_ptr(&new->td_lock) == &blocked_lock)
 550                 cpu_spinwait();
 551 #endif
 552
 553         PCPU_SET(curthread, new);
 554
 555 #ifdef COMPAT_FREEBSD32
 556         ia32_restorectx(newpcb);
 557 #endif
 558
 559         restorectx(newpcb);
 560         /* We should not get here. */
 561         panic("cpu_throw: restorectx() returned");
 562         /* NOTREACHED */
 563 }
 564
 565 void
 566 cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size)
 567 {
 568
 569         /*
 570          * Set pc_acpi_id to "uninitialized".
 571          * See sys/dev/acpica/acpi_cpu.c
 572          */
 573         pcpu->pc_acpi_id = 0xffffffff;
 574 }
 575
 576 void
 577 cpu_pcpu_setup(struct pcpu *pc, u_int acpi_id, u_int sapic_id)
 578 {
 579         struct ia64_sal_result r;
 580
 581         pc->pc_acpi_id = acpi_id;
 582         pc->pc_md.lid = IA64_LID_SET_SAPIC_ID(sapic_id);
 583
 584         r = ia64_sal_entry(SAL_SGISN_SAPIC_INFO, sapic_id, 0, 0, 0, 0, 0, 0);
 585         if (r.sal_status == 0) {
 586                 pc->pc_md.sgisn_nasid = r.sal_result[0];
 587                 pc->pc_md.sgisn_subnode = r.sal_result[1];
 588                 pc->pc_md.sgisn_slice = r.sal_result[2];
 589         }
 590 }
 591
 592 void
 593 spinlock_enter(void)
 594 {
 595         struct thread *td;
 596
 597         td = curthread;
 598         if (td->td_md.md_spinlock_count == 0)
 599                 td->td_md.md_saved_intr = intr_disable();
 600         td->td_md.md_spinlock_count++;
 601         critical_enter();
 602 }
 603
 604 void
 605 spinlock_exit(void)
 606 {
 607         struct thread *td;
 608
 609         td = curthread;
 610         critical_exit();
 611         td->td_md.md_spinlock_count--;
 612         if (td->td_md.md_spinlock_count == 0)
 613                 intr_restore(td->td_md.md_saved_intr);
 614 }
 615
 616 void
 617 map_vhpt(uintptr_t vhpt)
 618 {
 619         pt_entry_t pte;
 620         uint64_t psr;
 621
 622         pte = PTE_PRESENT | PTE_MA_WB | PTE_ACCESSED | PTE_DIRTY |
 623             PTE_PL_KERN | PTE_AR_RW;
 624         pte |= vhpt & PTE_PPN_MASK;
 625
 626         __asm __volatile("ptr.d %0,%1" :: "r"(vhpt),
 627             "r"(IA64_ID_PAGE_SHIFT<<2));
 628
 629         __asm __volatile("mov   %0=psr" : "=r"(psr));
 630         __asm __volatile("rsm   psr.ic|psr.i");
 631         ia64_srlz_i();
 632         ia64_set_ifa(vhpt);
 633         ia64_set_itir(IA64_ID_PAGE_SHIFT << 2);
 634         ia64_srlz_d();
 635         __asm __volatile("itr.d dtr[%0]=%1" :: "r"(2), "r"(pte));
 636         __asm __volatile("mov   psr.l=%0" :: "r" (psr));
 637         ia64_srlz_i();
 638 }
 639
 640 void
 641 map_pal_code(void)
 642 {
 643         pt_entry_t pte;
 644         uint64_t psr;
 645
 646         if (ia64_pal_base == 0)
 647                 return;
 648
 649         pte = PTE_PRESENT | PTE_MA_WB | PTE_ACCESSED | PTE_DIRTY |
 650             PTE_PL_KERN | PTE_AR_RWX;
 651         pte |= ia64_pal_base & PTE_PPN_MASK;
 652
 653         __asm __volatile("ptr.d %0,%1; ptr.i %0,%1" ::
 654             "r"(IA64_PHYS_TO_RR7(ia64_pal_base)), "r"(IA64_ID_PAGE_SHIFT<<2));
 655
 656         __asm __volatile("mov   %0=psr" : "=r"(psr));
 657         __asm __volatile("rsm   psr.ic|psr.i");
 658         ia64_srlz_i();
 659         ia64_set_ifa(IA64_PHYS_TO_RR7(ia64_pal_base));
 660         ia64_set_itir(IA64_ID_PAGE_SHIFT << 2);
 661         ia64_srlz_d();
 662         __asm __volatile("itr.d dtr[%0]=%1" :: "r"(1), "r"(pte));
 663         ia64_srlz_d();
 664         __asm __volatile("itr.i itr[%0]=%1" :: "r"(1), "r"(pte));
 665         __asm __volatile("mov   psr.l=%0" :: "r" (psr));
 666         ia64_srlz_i();
 667 }
 668
 669 void
 670 map_gateway_page(void)
 671 {
 672         pt_entry_t pte;
 673         uint64_t psr;
 674
 675         pte = PTE_PRESENT | PTE_MA_WB | PTE_ACCESSED | PTE_DIRTY |
 676             PTE_PL_KERN | PTE_AR_X_RX;
 677         pte |= (uint64_t)ia64_gateway_page & PTE_PPN_MASK;
 678
 679         __asm __volatile("ptr.d %0,%1; ptr.i %0,%1" ::
 680             "r"(VM_MAX_ADDRESS), "r"(PAGE_SHIFT << 2));
 681
 682         __asm __volatile("mov   %0=psr" : "=r"(psr));
 683         __asm __volatile("rsm   psr.ic|psr.i");
 684         ia64_srlz_i();
 685         ia64_set_ifa(VM_MAX_ADDRESS);
 686         ia64_set_itir(PAGE_SHIFT << 2);
 687         ia64_srlz_d();
 688         __asm __volatile("itr.d dtr[%0]=%1" :: "r"(3), "r"(pte));
 689         ia64_srlz_d();
 690         __asm __volatile("itr.i itr[%0]=%1" :: "r"(3), "r"(pte));
 691         __asm __volatile("mov   psr.l=%0" :: "r" (psr));
 692         ia64_srlz_i();
 693
 694         /* Expose the mapping to userland in ar.k5 */
 695         ia64_set_k5(VM_MAX_ADDRESS);
 696 }
 697
 698 static u_int
 699 freq_ratio(u_long base, u_long ratio)
 700 {
 701         u_long f;
 702
 703         f = (base * (ratio >> 32)) / (ratio & 0xfffffffful);
 704         return ((f + 500000) / 1000000);
 705 }
 706
 707 static void
 708 calculate_frequencies(void)
 709 {
 710         struct ia64_sal_result sal;
 711         struct ia64_pal_result pal;
 712
 713         sal = ia64_sal_entry(SAL_FREQ_BASE, 0, 0, 0, 0, 0, 0, 0);
 714         pal = ia64_call_pal_static(PAL_FREQ_RATIOS, 0, 0, 0);
 715
 716         if (sal.sal_status == 0 && pal.pal_status == 0) {
 717                 if (bootverbose) {
 718                         printf("Platform clock frequency %ld Hz\n",
 719                                sal.sal_result[0]);
 720                         printf("Processor ratio %ld/%ld, Bus ratio %ld/%ld, "
 721                                "ITC ratio %ld/%ld\n",
 722                                pal.pal_result[0] >> 32,
 723                                pal.pal_result[0] & ((1L << 32) - 1),
 724                                pal.pal_result[1] >> 32,
 725                                pal.pal_result[1] & ((1L << 32) - 1),
 726                                pal.pal_result[2] >> 32,
 727                                pal.pal_result[2] & ((1L << 32) - 1));
 728                 }
 729                 cpu_freq = freq_ratio(sal.sal_result[0], pal.pal_result[0]);
 730                 bus_freq = freq_ratio(sal.sal_result[0], pal.pal_result[1]);
 731                 itc_freq = freq_ratio(sal.sal_result[0], pal.pal_result[2]);
 732         }
 733 }
 734
 735 struct ia64_init_return
 736 ia64_init(void)
 737 {
 738         struct ia64_init_return ret;
 739         int phys_avail_cnt;
 740         vm_offset_t kernstart, kernend;
 741         vm_offset_t kernstartpfn, kernendpfn, pfn0, pfn1;
 742         char *p;
 743         struct efi_md *md;
 744         int metadata_missing;
 745
 746         /* NO OUTPUT ALLOWED UNTIL FURTHER NOTICE */
 747
 748         /*
 749          * TODO: Disable interrupts, floating point etc.
 750          * Maybe flush cache and tlb
 751          */
 752         ia64_set_fpsr(IA64_FPSR_DEFAULT);
 753
 754         /*
 755          * TODO: Get critical system information (if possible, from the
 756          * information provided by the boot program).
 757          */
 758
 759         /*
 760          * pa_bootinfo is the physical address of the bootinfo block as
 761          * passed to us by the loader and set in locore.s.
 762          */
 763         bootinfo = *(struct bootinfo *)(IA64_PHYS_TO_RR7(pa_bootinfo));
 764
 765         if (bootinfo.bi_magic != BOOTINFO_MAGIC || bootinfo.bi_version != 1) {
 766                 bzero(&bootinfo, sizeof(bootinfo));
 767                 bootinfo.bi_kernend = (vm_offset_t) round_page(_end);
 768         }
 769
 770         /*
 771          * Look for the I/O ports first - we need them for console
 772          * probing.
 773          */
 774         for (md = efi_md_first(); md != NULL; md = efi_md_next(md)) {
 775                 switch (md->md_type) {
 776                 case EFI_MD_TYPE_IOPORT:
 777                         ia64_port_base = (uintptr_t)pmap_mapdev(md->md_phys,
 778                             md->md_pages * EFI_PAGE_SIZE);
 779                         break;
 780                 case EFI_MD_TYPE_PALCODE:
 781                         ia64_pal_base = md->md_phys;
 782                         break;
 783                 }
 784         }
 785
 786         metadata_missing = 0;
 787         if (bootinfo.bi_modulep)
 788                 preload_metadata = (caddr_t)bootinfo.bi_modulep;
 789         else
 790                 metadata_missing = 1;
 791
 792         if (envmode == 0 && bootinfo.bi_envp)
 793                 kern_envp = (caddr_t)bootinfo.bi_envp;
 794         else
 795                 kern_envp = static_env;
 796
 797         /*
 798          * Look at arguments passed to us and compute boothowto.
 799          */
 800         boothowto = bootinfo.bi_boothowto;
 801
 802         if (boothowto & RB_VERBOSE)
 803                 bootverbose = 1;
 804
 805         /*
 806          * Find the beginning and end of the kernel.
 807          */
 808         kernstart = trunc_page(kernel_text);
 809 #ifdef DDB
 810         ksym_start = bootinfo.bi_symtab;
 811         ksym_end = bootinfo.bi_esymtab;
 812         kernend = (vm_offset_t)round_page(ksym_end);
 813 #else
 814         kernend = (vm_offset_t)round_page(_end);
 815 #endif
 816         /* But if the bootstrap tells us otherwise, believe it! */
 817         if (bootinfo.bi_kernend)
 818                 kernend = round_page(bootinfo.bi_kernend);
 819
 820         /*
 821          * Setup the PCPU data for the bootstrap processor. It is needed
 822          * by printf(). Also, since printf() has critical sections, we
 823          * need to initialize at least pc_curthread.
 824          */
 825         pcpup = &pcpu0;
 826         ia64_set_k4((u_int64_t)pcpup);
 827         pcpu_init(pcpup, 0, sizeof(pcpu0));
 828         dpcpu_init((void *)kernend, 0);
 829         cpu_pcpu_setup(pcpup, ~0U, ia64_get_lid());
 830         kernend += DPCPU_SIZE;
 831         PCPU_SET(curthread, &thread0);
 832
 833 #if 0
 834         if (ia64_pal_base != 0) {
 835                 ia64_pal_base &= ~IA64_ID_PAGE_MASK;
 836                 /*
 837                  * We use a TR to map the first 256M of memory - this might
 838                  * cover the palcode too.
 839                  */
 840                 if (ia64_pal_base == 0)
 841                         printf("PAL code mapped by the kernel's TR\n");
 842         } else
 843                 printf("PAL code not found\n");
 844 #endif
 845
 846         /*
 847          * Wire things up so we can call the firmware.
 848          */
 849         map_pal_code();
 850         efi_boot_minimal(bootinfo.bi_systab);
 851         ia64_xiv_init();
 852         ia64_sal_init();
 853         calculate_frequencies();
 854
 855         /*
 856          * Initialize the console before we print anything out.
 857          */
 858         cninit();
 859
 860         /* OUTPUT NOW ALLOWED */
 861
 862         if (metadata_missing)
 863                 printf("WARNING: loader(8) metadata is missing!\n");
 864
 865         check_sn_sal();
 866
 867         /* Get FPSWA interface */
 868         fpswa_iface = (bootinfo.bi_fpswa == 0) ? NULL :
 869             (struct fpswa_iface *)IA64_PHYS_TO_RR7(bootinfo.bi_fpswa);
 870
 871         /* Init basic tunables, including hz */
 872         init_param1();
 873
 874         p = getenv("kernelname");
 875         if (p != NULL) {
 876                 strncpy(kernelname, p, sizeof(kernelname) - 1);
 877                 freeenv(p);
 878         }
 879
 880         kernstartpfn = atop(IA64_RR_MASK(kernstart));
 881         kernendpfn = atop(IA64_RR_MASK(kernend));
 882
 883         /*
 884          * Size the memory regions and load phys_avail[] with the results.
 885          */
 886
 887         /*
 888          * Find out how much memory is available, by looking at
 889          * the memory descriptors.
 890          */
 891
 892 #ifdef DEBUG_MD
 893         printf("Memory descriptor count: %d\n", mdcount);
 894 #endif
 895
 896         phys_avail_cnt = 0;
 897         for (md = efi_md_first(); md != NULL; md = efi_md_next(md)) {
 898 #ifdef DEBUG_MD
 899                 printf("MD %p: type %d pa 0x%lx cnt 0x%lx\n", md,
 900                     md->md_type, md->md_phys, md->md_pages);
 901 #endif
 902
 903                 pfn0 = ia64_btop(round_page(md->md_phys));
 904                 pfn1 = ia64_btop(trunc_page(md->md_phys + md->md_pages * 4096));
 905                 if (pfn1 <= pfn0)
 906                         continue;
 907
 908                 if (md->md_type != EFI_MD_TYPE_FREE)
 909                         continue;
 910
 911                 /*
 912                  * We have a memory descriptor that describes conventional
 913                  * memory that is for general use. We must determine if the
 914                  * loader has put the kernel in this region.
 915                  */
 916                 physmem += (pfn1 - pfn0);
 917                 if (pfn0 <= kernendpfn && kernstartpfn <= pfn1) {
 918                         /*
 919                          * Must compute the location of the kernel
 920                          * within the segment.
 921                          */
 922 #ifdef DEBUG_MD
 923                         printf("Descriptor %p contains kernel\n", mp);
 924 #endif
 925                         if (pfn0 < kernstartpfn) {
 926                                 /*
 927                                  * There is a chunk before the kernel.
 928                                  */
 929 #ifdef DEBUG_MD
 930                                 printf("Loading chunk before kernel: "
 931                                        "0x%lx / 0x%lx\n", pfn0, kernstartpfn);
 932 #endif
 933                                 phys_avail[phys_avail_cnt] = ia64_ptob(pfn0);
 934                                 phys_avail[phys_avail_cnt+1] = ia64_ptob(kernstartpfn);
 935                                 phys_avail_cnt += 2;
 936                         }
 937                         if (kernendpfn < pfn1) {
 938                                 /*
 939                                  * There is a chunk after the kernel.
 940                                  */
 941 #ifdef DEBUG_MD
 942                                 printf("Loading chunk after kernel: "
 943                                        "0x%lx / 0x%lx\n", kernendpfn, pfn1);
 944 #endif
 945                                 phys_avail[phys_avail_cnt] = ia64_ptob(kernendpfn);
 946                                 phys_avail[phys_avail_cnt+1] = ia64_ptob(pfn1);
 947                                 phys_avail_cnt += 2;
 948                         }
 949                 } else {
 950                         /*
 951                          * Just load this cluster as one chunk.
 952                          */
 953 #ifdef DEBUG_MD
 954                         printf("Loading descriptor %d: 0x%lx / 0x%lx\n", i,
 955                                pfn0, pfn1);
 956 #endif
 957                         phys_avail[phys_avail_cnt] = ia64_ptob(pfn0);
 958                         phys_avail[phys_avail_cnt+1] = ia64_ptob(pfn1);
 959                         phys_avail_cnt += 2;
 960
 961                 }
 962         }
 963         phys_avail[phys_avail_cnt] = 0;
 964
 965         Maxmem = physmem;
 966         init_param2(physmem);
 967
 968         /*
 969          * Initialize error message buffer (at end of core).
 970          */
 971         msgbufp = (struct msgbuf *)pmap_steal_memory(MSGBUF_SIZE);
 972         msgbufinit(msgbufp, MSGBUF_SIZE);
 973
 974         proc_linkup0(&proc0, &thread0);
 975         /*
 976          * Init mapping for kernel stack for proc 0
 977          */
 978         thread0.td_kstack = pmap_steal_memory(KSTACK_PAGES * PAGE_SIZE);
 979         thread0.td_kstack_pages = KSTACK_PAGES;
 980
 981         mutex_init();
 982
 983         /*
 984          * Initialize the rest of proc 0's PCB.
 985          *
 986          * Set the kernel sp, reserving space for an (empty) trapframe,
 987          * and make proc0's trapframe pointer point to it for sanity.
 988          * Initialise proc0's backing store to start after u area.
 989          */
 990         cpu_thread_alloc(&thread0);
 991         thread0.td_frame->tf_flags = FRAME_SYSCALL;
 992         thread0.td_pcb->pcb_special.sp =
 993             (u_int64_t)thread0.td_frame - 16;
 994         thread0.td_pcb->pcb_special.bspstore = thread0.td_kstack;
 995
 996         /*
 997          * Initialize the virtual memory system.
 998          */
 999         pmap_bootstrap();
1000
1001         /*
1002          * Initialize debuggers, and break into them if appropriate.
1003          */
1004         kdb_init();
1005
1006 #ifdef KDB
1007         if (boothowto & RB_KDB)
1008                 kdb_enter(KDB_WHY_BOOTFLAGS,
1009                     "Boot flags requested debugger\n");
1010 #endif
1011
1012         ia64_set_tpr(0);
1013         ia64_srlz_d();
1014
1015         ret.bspstore = thread0.td_pcb->pcb_special.bspstore;
1016         ret.sp = thread0.td_pcb->pcb_special.sp;
1017         return (ret);
1018 }
1019
1020 uint64_t
1021 ia64_get_hcdp(void)
1022 {
1023
1024         return (bootinfo.bi_hcdp);
1025 }
1026
1027 void
1028 bzero(void *buf, size_t len)
1029 {
1030         caddr_t p = buf;
1031
1032         while (((vm_offset_t) p & (sizeof(u_long) - 1)) && len) {
1033                 *p++ = 0;
1034                 len--;
1035         }
1036         while (len >= sizeof(u_long) * 8) {
1037                 *(u_long*) p = 0;
1038                 *((u_long*) p + 1) = 0;
1039                 *((u_long*) p + 2) = 0;
1040                 *((u_long*) p + 3) = 0;
1041                 len -= sizeof(u_long) * 8;
1042                 *((u_long*) p + 4) = 0;
1043                 *((u_long*) p + 5) = 0;
1044                 *((u_long*) p + 6) = 0;
1045                 *((u_long*) p + 7) = 0;
1046                 p += sizeof(u_long) * 8;
1047         }
1048         while (len >= sizeof(u_long)) {
1049                 *(u_long*) p = 0;
1050                 len -= sizeof(u_long);
1051                 p += sizeof(u_long);
1052         }
1053         while (len) {
1054                 *p++ = 0;
1055                 len--;
1056         }
1057 }
1058
1059 u_int
1060 ia64_itc_freq(void)
1061 {
1062
1063         return (itc_freq);
1064 }
1065
1066 void
1067 DELAY(int n)
1068 {
1069         u_int64_t start, end, now;
1070
1071         sched_pin();
1072
1073         start = ia64_get_itc();
1074         end = start + itc_freq * n;
1075         /* printf("DELAY from 0x%lx to 0x%lx\n", start, end); */
1076         do {
1077                 now = ia64_get_itc();
1078         } while (now < end || (now > start && end < start));
1079
1080         sched_unpin();
1081 }
1082
1083 /*
1084  * Send an interrupt (signal) to a process.
1085  */
1086 void
1087 sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask)
1088 {
1089         struct proc *p;
1090         struct thread *td;
1091         struct trapframe *tf;
1092         struct sigacts *psp;
1093         struct sigframe sf, *sfp;
1094         u_int64_t sbs, sp;
1095         int oonstack;
1096         int sig;
1097         u_long code;
1098
1099         td = curthread;
1100         p = td->td_proc;
1101         PROC_LOCK_ASSERT(p, MA_OWNED);
1102         sig = ksi->ksi_signo;
1103         code = ksi->ksi_code;
1104         psp = p->p_sigacts;
1105         mtx_assert(&psp->ps_mtx, MA_OWNED);
1106         tf = td->td_frame;
1107         sp = tf->tf_special.sp;
1108         oonstack = sigonstack(sp);
1109         sbs = 0;
1110
1111         /* save user context */
1112         bzero(&sf, sizeof(struct sigframe));
1113         sf.sf_uc.uc_sigmask = *mask;
1114         sf.sf_uc.uc_stack = td->td_sigstk;
1115         sf.sf_uc.uc_stack.ss_flags = (td->td_pflags & TDP_ALTSTACK)
1116             ? ((oonstack) ? SS_ONSTACK : 0) : SS_DISABLE;
1117
1118         /*
1119          * Allocate and validate space for the signal handler
1120          * context. Note that if the stack is in P0 space, the
1121          * call to grow() is a nop, and the useracc() check
1122          * will fail if the process has not already allocated
1123          * the space with a `brk'.
1124          */
1125         if ((td->td_pflags & TDP_ALTSTACK) != 0 && !oonstack &&
1126             SIGISMEMBER(psp->ps_sigonstack, sig)) {
1127                 sbs = (u_int64_t)td->td_sigstk.ss_sp;
1128                 sbs = (sbs + 15) & ~15;
1129                 sfp = (struct sigframe *)(sbs + td->td_sigstk.ss_size);
1130 #if defined(COMPAT_43)
1131                 td->td_sigstk.ss_flags |= SS_ONSTACK;
1132 #endif
1133         } else
1134                 sfp = (struct sigframe *)sp;
1135         sfp = (struct sigframe *)((u_int64_t)(sfp - 1) & ~15);
1136
1137         /* Fill in the siginfo structure for POSIX handlers. */
1138         if (SIGISMEMBER(psp->ps_siginfo, sig)) {
1139                 sf.sf_si = ksi->ksi_info;
1140                 sf.sf_si.si_signo = sig;
1141                 /*
1142                  * XXX this shouldn't be here after code in trap.c
1143                  * is fixed
1144                  */
1145                 sf.sf_si.si_addr = (void*)tf->tf_special.ifa;
1146                 code = (u_int64_t)&sfp->sf_si;
1147         }
1148
1149         mtx_unlock(&psp->ps_mtx);
1150         PROC_UNLOCK(p);
1151
1152         get_mcontext(td, &sf.sf_uc.uc_mcontext, 0);
1153
1154         /* Copy the frame out to userland. */
1155         if (copyout(&sf, sfp, sizeof(sf)) != 0) {
1156                 /*
1157                  * Process has trashed its stack; give it an illegal
1158                  * instruction to halt it in its tracks.
1159                  */
1160                 PROC_LOCK(p);
1161                 sigexit(td, SIGILL);
1162                 return;
1163         }
1164
1165         if ((tf->tf_flags & FRAME_SYSCALL) == 0) {
1166                 tf->tf_special.psr &= ~IA64_PSR_RI;
1167                 tf->tf_special.iip = ia64_get_k5() +
1168                     ((uint64_t)break_sigtramp - (uint64_t)ia64_gateway_page);
1169         } else
1170                 tf->tf_special.iip = ia64_get_k5() +
1171                     ((uint64_t)epc_sigtramp - (uint64_t)ia64_gateway_page);
1172
1173         /*
1174          * Setup the trapframe to return to the signal trampoline. We pass
1175          * information to the trampoline in the following registers:
1176          *
1177          *      gp      new backing store or NULL
1178          *      r8      signal number
1179          *      r9      signal code or siginfo pointer
1180          *      r10     signal handler (function descriptor)
1181          */
1182         tf->tf_special.sp = (u_int64_t)sfp - 16;
1183         tf->tf_special.gp = sbs;
1184         tf->tf_special.bspstore = sf.sf_uc.uc_mcontext.mc_special.bspstore;
1185         tf->tf_special.ndirty = 0;
1186         tf->tf_special.rnat = sf.sf_uc.uc_mcontext.mc_special.rnat;
1187         tf->tf_scratch.gr8 = sig;
1188         tf->tf_scratch.gr9 = code;
1189         tf->tf_scratch.gr10 = (u_int64_t)catcher;
1190
1191         PROC_LOCK(p);
1192         mtx_lock(&psp->ps_mtx);
1193 }
1194
1195 /*
1196  * System call to cleanup state after a signal
1197  * has been taken.  Reset signal mask and
1198  * stack state from context left by sendsig (above).
1199  * Return to previous pc and psl as specified by
1200  * context left by sendsig. Check carefully to
1201  * make sure that the user has not modified the
1202  * state to gain improper privileges.
1203  *
1204  * MPSAFE
1205  */
1206 int
1207 sigreturn(struct thread *td,
1208         struct sigreturn_args /* {
1209                 ucontext_t *sigcntxp;
1210         } */ *uap)
1211 {
1212         ucontext_t uc;
1213         struct trapframe *tf;
1214         struct pcb *pcb;
1215
1216         tf = td->td_frame;
1217         pcb = td->td_pcb;
1218
1219         /*
1220          * Fetch the entire context structure at once for speed.
1221          * We don't use a normal argument to simplify RSE handling.
1222          */
1223         if (copyin(uap->sigcntxp, (caddr_t)&uc, sizeof(uc)))
1224                 return (EFAULT);
1225
1226         set_mcontext(td, &uc.uc_mcontext);
1227
1228 #if defined(COMPAT_43)
1229         if (sigonstack(tf->tf_special.sp))
1230                 td->td_sigstk.ss_flags |= SS_ONSTACK;
1231         else
1232                 td->td_sigstk.ss_flags &= ~SS_ONSTACK;
1233 #endif
1234         kern_sigprocmask(td, SIG_SETMASK, &uc.uc_sigmask, NULL, 0);
1235
1236         return (EJUSTRETURN);
1237 }
1238
1239 #ifdef COMPAT_FREEBSD4
1240 int
1241 freebsd4_sigreturn(struct thread *td, struct freebsd4_sigreturn_args *uap)
1242 {
1243
1244         return sigreturn(td, (struct sigreturn_args *)uap);
1245 }
1246 #endif
1247
1248 /*
1249  * Construct a PCB from a trapframe. This is called from kdb_trap() where
1250  * we want to start a backtrace from the function that caused us to enter
1251  * the debugger. We have the context in the trapframe, but base the trace
1252  * on the PCB. The PCB doesn't have to be perfect, as long as it contains
1253  * enough for a backtrace.
1254  */
1255 void
1256 makectx(struct trapframe *tf, struct pcb *pcb)
1257 {
1258
1259         pcb->pcb_special = tf->tf_special;
1260         pcb->pcb_special.__spare = ~0UL;        /* XXX see unwind.c */
1261         save_callee_saved(&pcb->pcb_preserved);
1262         save_callee_saved_fp(&pcb->pcb_preserved_fp);
1263 }
1264
1265 int
1266 ia64_flush_dirty(struct thread *td, struct _special *r)
1267 {
1268         struct iovec iov;
1269         struct uio uio;
1270         uint64_t bspst, kstk, rnat;
1271         int error, locked;
1272
1273         if (r->ndirty == 0)
1274                 return (0);
1275
1276         kstk = td->td_kstack + (r->bspstore & 0x1ffUL);
1277         if (td == curthread) {
1278                 __asm __volatile("mov   ar.rsc=0;;");
1279                 __asm __volatile("mov   %0=ar.bspstore" : "=r"(bspst));
1280                 /* Make sure we have all the user registers written out. */
1281                 if (bspst - kstk < r->ndirty) {
1282                         __asm __volatile("flushrs;;");
1283                         __asm __volatile("mov   %0=ar.bspstore" : "=r"(bspst));
1284                 }
1285                 __asm __volatile("mov   %0=ar.rnat;;" : "=r"(rnat));
1286                 __asm __volatile("mov   ar.rsc=3");
1287                 error = copyout((void*)kstk, (void*)r->bspstore, r->ndirty);
1288                 kstk += r->ndirty;
1289                 r->rnat = (bspst > kstk && (bspst & 0x1ffL) < (kstk & 0x1ffL))
1290                     ? *(uint64_t*)(kstk | 0x1f8L) : rnat;
1291         } else {
1292                 locked = PROC_LOCKED(td->td_proc);
1293                 if (!locked)
1294                         PHOLD(td->td_proc);
1295                 iov.iov_base = (void*)(uintptr_t)kstk;
1296                 iov.iov_len = r->ndirty;
1297                 uio.uio_iov = &iov;
1298                 uio.uio_iovcnt = 1;
1299                 uio.uio_offset = r->bspstore;
1300                 uio.uio_resid = r->ndirty;
1301                 uio.uio_segflg = UIO_SYSSPACE;
1302                 uio.uio_rw = UIO_WRITE;
1303                 uio.uio_td = td;
1304                 error = proc_rwmem(td->td_proc, &uio);
1305                 /*
1306                  * XXX proc_rwmem() doesn't currently return ENOSPC,
1307                  * so I think it can bogusly return 0. Neither do
1308                  * we allow short writes.
1309                  */
1310                 if (uio.uio_resid != 0 && error == 0)
1311                         error = ENOSPC;
1312                 if (!locked)
1313                         PRELE(td->td_proc);
1314         }
1315
1316         r->bspstore += r->ndirty;
1317         r->ndirty = 0;
1318         return (error);
1319 }
1320
1321 int
1322 get_mcontext(struct thread *td, mcontext_t *mc, int flags)
1323 {
1324         struct trapframe *tf;
1325         int error;
1326
1327         tf = td->td_frame;
1328         bzero(mc, sizeof(*mc));
1329         mc->mc_special = tf->tf_special;
1330         error = ia64_flush_dirty(td, &mc->mc_special);
1331         if (tf->tf_flags & FRAME_SYSCALL) {
1332                 mc->mc_flags |= _MC_FLAGS_SYSCALL_CONTEXT;
1333                 mc->mc_scratch = tf->tf_scratch;
1334                 if (flags & GET_MC_CLEAR_RET) {
1335                         mc->mc_scratch.gr8 = 0;
1336                         mc->mc_scratch.gr9 = 0;
1337                         mc->mc_scratch.gr10 = 0;
1338                         mc->mc_scratch.gr11 = 0;
1339                 }
1340         } else {
1341                 mc->mc_flags |= _MC_FLAGS_ASYNC_CONTEXT;
1342                 mc->mc_scratch = tf->tf_scratch;
1343                 mc->mc_scratch_fp = tf->tf_scratch_fp;
1344                 /*
1345                  * XXX If the thread never used the high FP registers, we
1346                  * probably shouldn't waste time saving them.
1347                  */
1348                 ia64_highfp_save(td);
1349                 mc->mc_flags |= _MC_FLAGS_HIGHFP_VALID;
1350                 mc->mc_high_fp = td->td_pcb->pcb_high_fp;
1351         }
1352         save_callee_saved(&mc->mc_preserved);
1353         save_callee_saved_fp(&mc->mc_preserved_fp);
1354         return (error);
1355 }
1356
1357 int
1358 set_mcontext(struct thread *td, const mcontext_t *mc)
1359 {
1360         struct _special s;
1361         struct trapframe *tf;
1362         uint64_t psrmask;
1363
1364         tf = td->td_frame;
1365
1366         KASSERT((tf->tf_special.ndirty & ~PAGE_MASK) == 0,
1367             ("Whoa there! We have more than 8KB of dirty registers!"));
1368
1369         s = mc->mc_special;
1370         /*
1371          * Only copy the user mask and the restart instruction bit from
1372          * the new context.
1373          */
1374         psrmask = IA64_PSR_BE | IA64_PSR_UP | IA64_PSR_AC | IA64_PSR_MFL |
1375             IA64_PSR_MFH | IA64_PSR_RI;
1376         s.psr = (tf->tf_special.psr & ~psrmask) | (s.psr & psrmask);
1377         /* We don't have any dirty registers of the new context. */
1378         s.ndirty = 0;
1379         if (mc->mc_flags & _MC_FLAGS_ASYNC_CONTEXT) {
1380                 /*
1381                  * We can get an async context passed to us while we
1382                  * entered the kernel through a syscall: sigreturn(2)
1383                  * takes contexts that could previously be the result of
1384                  * a trap or interrupt.
1385                  * Hence, we cannot assert that the trapframe is not
1386                  * a syscall frame, but we can assert that it's at
1387                  * least an expected syscall.
1388                  */
1389                 if (tf->tf_flags & FRAME_SYSCALL) {
1390                         KASSERT(tf->tf_scratch.gr15 == SYS_sigreturn, ("foo"));
1391                         tf->tf_flags &= ~FRAME_SYSCALL;
1392                 }
1393                 tf->tf_scratch = mc->mc_scratch;
1394                 tf->tf_scratch_fp = mc->mc_scratch_fp;
1395                 if (mc->mc_flags & _MC_FLAGS_HIGHFP_VALID)
1396                         td->td_pcb->pcb_high_fp = mc->mc_high_fp;
1397         } else {
1398                 KASSERT((tf->tf_flags & FRAME_SYSCALL) != 0, ("foo"));
1399                 if ((mc->mc_flags & _MC_FLAGS_SYSCALL_CONTEXT) == 0) {
1400                         s.cfm = tf->tf_special.cfm;
1401                         s.iip = tf->tf_special.iip;
1402                         tf->tf_scratch.gr15 = 0;        /* Clear syscall nr. */
1403                 } else
1404                         tf->tf_scratch = mc->mc_scratch;
1405         }
1406         tf->tf_special = s;
1407         restore_callee_saved(&mc->mc_preserved);
1408         restore_callee_saved_fp(&mc->mc_preserved_fp);
1409
1410         return (0);
1411 }
1412
1413 /*
1414  * Clear registers on exec.
1415  */
1416 void
1417 exec_setregs(struct thread *td, struct image_params *imgp, u_long stack)
1418 {
1419         struct trapframe *tf;
1420         uint64_t *ksttop, *kst;
1421
1422         tf = td->td_frame;
1423         ksttop = (uint64_t*)(td->td_kstack + tf->tf_special.ndirty +
1424             (tf->tf_special.bspstore & 0x1ffUL));
1425
1426         /*
1427          * We can ignore up to 8KB of dirty registers by masking off the
1428          * lower 13 bits in exception_restore() or epc_syscall(). This
1429          * should be enough for a couple of years, but if there are more
1430          * than 8KB of dirty registers, we lose track of the bottom of
1431          * the kernel stack. The solution is to copy the active part of
1432          * the kernel stack down 1 page (or 2, but not more than that)
1433          * so that we always have less than 8KB of dirty registers.
1434          */
1435         KASSERT((tf->tf_special.ndirty & ~PAGE_MASK) == 0,
1436             ("Whoa there! We have more than 8KB of dirty registers!"));
1437
1438         bzero(&tf->tf_special, sizeof(tf->tf_special));
1439         if ((tf->tf_flags & FRAME_SYSCALL) == 0) {      /* break syscalls. */
1440                 bzero(&tf->tf_scratch, sizeof(tf->tf_scratch));
1441                 bzero(&tf->tf_scratch_fp, sizeof(tf->tf_scratch_fp));
1442                 tf->tf_special.cfm = (1UL<<63) | (3UL<<7) | 3UL;
1443                 tf->tf_special.bspstore = IA64_BACKINGSTORE;
1444                 /*
1445                  * Copy the arguments onto the kernel register stack so that
1446                  * they get loaded by the loadrs instruction. Skip over the
1447                  * NaT collection points.
1448                  */
1449                 kst = ksttop - 1;
1450                 if (((uintptr_t)kst & 0x1ff) == 0x1f8)
1451                         *kst-- = 0;
1452                 *kst-- = 0;
1453                 if (((uintptr_t)kst & 0x1ff) == 0x1f8)
1454                         *kst-- = 0;
1455                 *kst-- = imgp->ps_strings;
1456                 if (((uintptr_t)kst & 0x1ff) == 0x1f8)
1457                         *kst-- = 0;
1458                 *kst = stack;
1459                 tf->tf_special.ndirty = (ksttop - kst) << 3;
1460         } else {                                /* epc syscalls (default). */
1461                 tf->tf_special.cfm = (3UL<<62) | (3UL<<7) | 3UL;
1462                 tf->tf_special.bspstore = IA64_BACKINGSTORE + 24;
1463                 /*
1464                  * Write values for out0, out1 and out2 to the user's backing
1465                  * store and arrange for them to be restored into the user's
1466                  * initial register frame.
1467                  * Assumes that (bspstore & 0x1f8) < 0x1e0.
1468                  */
1469                 suword((caddr_t)tf->tf_special.bspstore - 24, stack);
1470                 suword((caddr_t)tf->tf_special.bspstore - 16, imgp->ps_strings);
1471                 suword((caddr_t)tf->tf_special.bspstore -  8, 0);
1472         }
1473
1474         tf->tf_special.iip = imgp->entry_addr;
1475         tf->tf_special.sp = (stack & ~15) - 16;
1476         tf->tf_special.rsc = 0xf;
1477         tf->tf_special.fpsr = IA64_FPSR_DEFAULT;
1478         tf->tf_special.psr = IA64_PSR_IC | IA64_PSR_I | IA64_PSR_IT |
1479             IA64_PSR_DT | IA64_PSR_RT | IA64_PSR_DFH | IA64_PSR_BN |
1480             IA64_PSR_CPL_USER;
1481 }
1482
1483 int
1484 ptrace_set_pc(struct thread *td, unsigned long addr)
1485 {
1486         uint64_t slot;
1487
1488         switch (addr & 0xFUL) {
1489         case 0:
1490                 slot = IA64_PSR_RI_0;
1491                 break;
1492         case 1:
1493                 /* XXX we need to deal with MLX bundles here */
1494                 slot = IA64_PSR_RI_1;
1495                 break;
1496         case 2:
1497                 slot = IA64_PSR_RI_2;
1498                 break;
1499         default:
1500                 return (EINVAL);
1501         }
1502
1503         td->td_frame->tf_special.iip = addr & ~0x0FULL;
1504         td->td_frame->tf_special.psr =
1505             (td->td_frame->tf_special.psr & ~IA64_PSR_RI) | slot;
1506         return (0);
1507 }
1508
1509 int
1510 ptrace_single_step(struct thread *td)
1511 {
1512         struct trapframe *tf;
1513
1514         /*
1515          * There's no way to set single stepping when we're leaving the
1516          * kernel through the EPC syscall path. The way we solve this is
1517          * by enabling the lower-privilege trap so that we re-enter the
1518          * kernel as soon as the privilege level changes. See trap.c for
1519          * how we proceed from there.
1520          */
1521         tf = td->td_frame;
1522         if (tf->tf_flags & FRAME_SYSCALL)
1523                 tf->tf_special.psr |= IA64_PSR_LP;
1524         else
1525                 tf->tf_special.psr |= IA64_PSR_SS;
1526         return (0);
1527 }
1528
1529 int
1530 ptrace_clear_single_step(struct thread *td)
1531 {
1532         struct trapframe *tf;
1533
1534         /*
1535          * Clear any and all status bits we may use to implement single
1536          * stepping.
1537          */
1538         tf = td->td_frame;
1539         tf->tf_special.psr &= ~IA64_PSR_SS;
1540         tf->tf_special.psr &= ~IA64_PSR_LP;
1541         tf->tf_special.psr &= ~IA64_PSR_TB;
1542         return (0);
1543 }
1544
1545 int
1546 fill_regs(struct thread *td, struct reg *regs)
1547 {
1548         struct trapframe *tf;
1549
1550         tf = td->td_frame;
1551         regs->r_special = tf->tf_special;
1552         regs->r_scratch = tf->tf_scratch;
1553         save_callee_saved(&regs->r_preserved);
1554         return (0);
1555 }
1556
1557 int
1558 set_regs(struct thread *td, struct reg *regs)
1559 {
1560         struct trapframe *tf;
1561         int error;
1562
1563         tf = td->td_frame;
1564         error = ia64_flush_dirty(td, &tf->tf_special);
1565         if (!error) {
1566                 tf->tf_special = regs->r_special;
1567                 tf->tf_special.bspstore += tf->tf_special.ndirty;
1568                 tf->tf_special.ndirty = 0;
1569                 tf->tf_scratch = regs->r_scratch;
1570                 restore_callee_saved(&regs->r_preserved);
1571         }
1572         return (error);
1573 }
1574
1575 int
1576 fill_dbregs(struct thread *td, struct dbreg *dbregs)
1577 {
1578
1579         return (ENOSYS);
1580 }
1581
1582 int
1583 set_dbregs(struct thread *td, struct dbreg *dbregs)
1584 {
1585
1586         return (ENOSYS);
1587 }
1588
1589 int
1590 fill_fpregs(struct thread *td, struct fpreg *fpregs)
1591 {
1592         struct trapframe *frame = td->td_frame;
1593         struct pcb *pcb = td->td_pcb;
1594
1595         /* Save the high FP registers. */
1596         ia64_highfp_save(td);
1597
1598         fpregs->fpr_scratch = frame->tf_scratch_fp;
1599         save_callee_saved_fp(&fpregs->fpr_preserved);
1600         fpregs->fpr_high = pcb->pcb_high_fp;
1601         return (0);
1602 }
1603
1604 int
1605 set_fpregs(struct thread *td, struct fpreg *fpregs)
1606 {
1607         struct trapframe *frame = td->td_frame;
1608         struct pcb *pcb = td->td_pcb;
1609
1610         /* Throw away the high FP registers (should be redundant). */
1611         ia64_highfp_drop(td);
1612
1613         frame->tf_scratch_fp = fpregs->fpr_scratch;
1614         restore_callee_saved_fp(&fpregs->fpr_preserved);
1615         pcb->pcb_high_fp = fpregs->fpr_high;
1616         return (0);
1617 }
1618
1619 void
1620 ia64_sync_icache(vm_offset_t va, vm_offset_t sz)
1621 {
1622         vm_offset_t lim;
1623
1624         if (!ia64_sync_icache_needed)
1625                 return;
1626
1627         lim = va + sz;
1628         while (va < lim) {
1629                 ia64_fc_i(va);
1630                 va += 32;       /* XXX */
1631         }
1632
1633         ia64_sync_i();
1634         ia64_srlz_i();
1635 }