sys/i386/xen/xen_machdep.c

   1 /*
   2  *
   3  * Copyright (c) 2004 Christian Limpach.
   4  * Copyright (c) 2004-2006,2008 Kip Macy
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer.
  12  * 2. Redistributions in binary form must reproduce the above copyright
  13  *    notice, this list of conditions and the following disclaimer in the
  14  *    documentation and/or other materials provided with the distribution.
  15  * 3. All advertising materials mentioning features or use of this software
  16  *    must display the following acknowledgement:
  17  *      This product includes software developed by Christian Limpach.
  18  * 4. The name of the author may not be used to endorse or promote products
  19  *    derived from this software without specific prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  22  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  23  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  24  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  25  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  26  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  30  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <sys/cdefs.h>
  34 __FBSDID("$FreeBSD$");
  35
  36 #include <sys/param.h>
  37 #include <sys/systm.h>
  38 #include <sys/bus.h>
  39 #include <sys/ktr.h>
  40 #include <sys/lock.h>
  41 #include <sys/mount.h>
  42 #include <sys/malloc.h>
  43 #include <sys/mutex.h>
  44 #include <sys/kernel.h>
  45 #include <sys/proc.h>
  46 #include <sys/reboot.h>
  47 #include <sys/rwlock.h>
  48 #include <sys/sysproto.h>
  49 #include <sys/boot.h>
  50
  51 #include <xen/xen-os.h>
  52
  53 #include <vm/vm.h>
  54 #include <vm/pmap.h>
  55 #include <machine/segments.h>
  56 #include <machine/pcb.h>
  57 #include <machine/stdarg.h>
  58 #include <machine/vmparam.h>
  59 #include <machine/cpu.h>
  60 #include <machine/intr_machdep.h>
  61 #include <machine/md_var.h>
  62 #include <machine/asmacros.h>
  63
  64
  65
  66 #include <xen/hypervisor.h>
  67 #include <xen/xenstore/xenstorevar.h>
  68 #include <machine/xen/xenvar.h>
  69 #include <machine/xen/xenfunc.h>
  70 #include <machine/xen/xenpmap.h>
  71 #include <machine/xen/xenfunc.h>
  72 #include <xen/interface/memory.h>
  73 #include <machine/xen/features.h>
  74 #ifdef SMP
  75 #include <machine/privatespace.h>
  76 #endif
  77
  78
  79 #include <vm/vm_page.h>
  80
  81
  82 #define IDTVEC(name)    __CONCAT(X,name)
  83
  84 extern inthand_t
  85 IDTVEC(div), IDTVEC(dbg), IDTVEC(nmi), IDTVEC(bpt), IDTVEC(ofl),
  86         IDTVEC(bnd), IDTVEC(ill), IDTVEC(dna), IDTVEC(fpusegm),
  87         IDTVEC(tss), IDTVEC(missing), IDTVEC(stk), IDTVEC(prot),
  88         IDTVEC(page), IDTVEC(mchk), IDTVEC(rsvd), IDTVEC(fpu), IDTVEC(align),
  89         IDTVEC(xmm), IDTVEC(lcall_syscall), IDTVEC(int0x80_syscall);
  90
  91
  92 int xendebug_flags;
  93 start_info_t *xen_start_info;
  94 start_info_t *HYPERVISOR_start_info;
  95 shared_info_t *HYPERVISOR_shared_info;
  96 xen_pfn_t *xen_machine_phys = machine_to_phys_mapping;
  97 xen_pfn_t *xen_phys_machine;
  98 xen_pfn_t *xen_pfn_to_mfn_frame_list[16];
  99 xen_pfn_t *xen_pfn_to_mfn_frame_list_list;
 100 int preemptable, init_first;
 101 extern unsigned int avail_space;
 102 int xen_vector_callback_enabled = 0;
 103 enum xen_domain_type xen_domain_type = XEN_PV_DOMAIN;
 104
 105 void ni_cli(void);
 106 void ni_sti(void);
 107
 108
 109 void
 110 ni_cli(void)
 111 {
 112         CTR0(KTR_SPARE2, "ni_cli disabling interrupts");
 113         __asm__("pushl %edx;"
 114                 "pushl %eax;"
 115                 );
 116         __cli();
 117         __asm__("popl %eax;"
 118                 "popl %edx;"
 119                 );
 120 }
 121
 122
 123 void
 124 ni_sti(void)
 125 {
 126         __asm__("pushl %edx;"
 127                 "pushl %esi;"
 128                 "pushl %eax;"
 129                 );
 130         __sti();
 131         __asm__("popl %eax;"
 132                 "popl %esi;"
 133                 "popl %edx;"
 134                 );
 135 }
 136
 137 void
 138 force_evtchn_callback(void)
 139 {
 140     (void)HYPERVISOR_xen_version(0, NULL);
 141 }
 142
 143 /*
 144  * Modify the cmd_line by converting ',' to NULLs so that it is in a  format
 145  * suitable for the static env vars.
 146  */
 147 char *
 148 xen_setbootenv(char *cmd_line)
 149 {
 150         char *cmd_line_next;
 151
 152         /* Skip leading spaces */
 153         for (; *cmd_line == ' '; cmd_line++);
 154
 155         xc_printf("xen_setbootenv(): cmd_line='%s'\n", cmd_line);
 156
 157         for (cmd_line_next = cmd_line; strsep(&cmd_line_next, ",") != NULL;);
 158         return cmd_line;
 159 }
 160
 161 int
 162 xen_boothowto(char *envp)
 163 {
 164         int i, howto = 0;
 165
 166         /* get equivalents from the environment */
 167         for (i = 0; howto_names[i].ev != NULL; i++)
 168                 if (getenv(howto_names[i].ev) != NULL)
 169                         howto |= howto_names[i].mask;
 170         return howto;
 171 }
 172
 173
 174 #define XPQUEUE_SIZE 128
 175
 176 struct mmu_log {
 177         char *file;
 178         int line;
 179 };
 180
 181 #ifdef SMP
 182 /* per-cpu queues and indices */
 183 #ifdef INVARIANTS
 184 static struct mmu_log xpq_queue_log[XEN_LEGACY_MAX_VCPUS][XPQUEUE_SIZE];
 185 #endif
 186
 187 static int xpq_idx[XEN_LEGACY_MAX_VCPUS];
 188 static mmu_update_t xpq_queue[XEN_LEGACY_MAX_VCPUS][XPQUEUE_SIZE];
 189
 190 #define XPQ_QUEUE_LOG xpq_queue_log[vcpu]
 191 #define XPQ_QUEUE xpq_queue[vcpu]
 192 #define XPQ_IDX xpq_idx[vcpu]
 193 #define SET_VCPU() int vcpu = smp_processor_id()
 194 #else
 195
 196 static mmu_update_t xpq_queue[XPQUEUE_SIZE];
 197 #ifdef INVARIANTS
 198 static struct mmu_log xpq_queue_log[XPQUEUE_SIZE];
 199 #endif
 200 static int xpq_idx = 0;
 201
 202 #define XPQ_QUEUE_LOG xpq_queue_log
 203 #define XPQ_QUEUE xpq_queue
 204 #define XPQ_IDX xpq_idx
 205 #define SET_VCPU()
 206 #endif /* !SMP */
 207
 208 #define XPQ_IDX_INC atomic_add_int(&XPQ_IDX, 1);
 209
 210 #if 0
 211 static void
 212 xen_dump_queue(void)
 213 {
 214         int _xpq_idx = XPQ_IDX;
 215         int i;
 216
 217         if (_xpq_idx <= 1)
 218                 return;
 219
 220         xc_printf("xen_dump_queue(): %u entries\n", _xpq_idx);
 221         for (i = 0; i < _xpq_idx; i++) {
 222                 xc_printf(" val: %llx ptr: %llx\n", XPQ_QUEUE[i].val,
 223                     XPQ_QUEUE[i].ptr);
 224         }
 225 }
 226 #endif
 227
 228
 229 static __inline void
 230 _xen_flush_queue(void)
 231 {
 232         SET_VCPU();
 233         int _xpq_idx = XPQ_IDX;
 234         int error, i;
 235
 236 #ifdef INVARIANTS
 237         if (__predict_true(gdtset))
 238                 CRITICAL_ASSERT(curthread);
 239 #endif
 240
 241         XPQ_IDX = 0;
 242         /* Make sure index is cleared first to avoid double updates. */
 243         error = HYPERVISOR_mmu_update((mmu_update_t *)&XPQ_QUEUE,
 244                                       _xpq_idx, NULL, DOMID_SELF);
 245
 246 #if 0
 247         if (__predict_true(gdtset))
 248         for (i = _xpq_idx; i > 0;) {
 249                 if (i >= 3) {
 250                         CTR6(KTR_PMAP, "mmu:val: %lx ptr: %lx val: %lx "
 251                             "ptr: %lx val: %lx ptr: %lx",
 252                             (XPQ_QUEUE[i-1].val & 0xffffffff),
 253                             (XPQ_QUEUE[i-1].ptr & 0xffffffff),
 254                             (XPQ_QUEUE[i-2].val & 0xffffffff),
 255                             (XPQ_QUEUE[i-2].ptr & 0xffffffff),
 256                             (XPQ_QUEUE[i-3].val & 0xffffffff),
 257                             (XPQ_QUEUE[i-3].ptr & 0xffffffff));
 258                             i -= 3;
 259                 } else if (i == 2) {
 260                         CTR4(KTR_PMAP, "mmu: val: %lx ptr: %lx val: %lx ptr: %lx",
 261                             (XPQ_QUEUE[i-1].val & 0xffffffff),
 262                             (XPQ_QUEUE[i-1].ptr & 0xffffffff),
 263                             (XPQ_QUEUE[i-2].val & 0xffffffff),
 264                             (XPQ_QUEUE[i-2].ptr & 0xffffffff));
 265                         i = 0;
 266                 } else {
 267                         CTR2(KTR_PMAP, "mmu: val: %lx ptr: %lx",
 268                             (XPQ_QUEUE[i-1].val & 0xffffffff),
 269                             (XPQ_QUEUE[i-1].ptr & 0xffffffff));
 270                         i = 0;
 271                 }
 272         }
 273 #endif
 274         if (__predict_false(error < 0)) {
 275                 for (i = 0; i < _xpq_idx; i++)
 276                         printf("val: %llx ptr: %llx\n",
 277                             XPQ_QUEUE[i].val, XPQ_QUEUE[i].ptr);
 278                 panic("Failed to execute MMU updates: %d", error);
 279         }
 280
 281 }
 282
 283 void
 284 xen_flush_queue(void)
 285 {
 286         SET_VCPU();
 287
 288         if (__predict_true(gdtset))
 289                 critical_enter();
 290         if (XPQ_IDX != 0) _xen_flush_queue();
 291         if (__predict_true(gdtset))
 292                 critical_exit();
 293 }
 294
 295 static __inline void
 296 xen_increment_idx(void)
 297 {
 298         SET_VCPU();
 299
 300         XPQ_IDX++;
 301         if (__predict_false(XPQ_IDX == XPQUEUE_SIZE))
 302                 xen_flush_queue();
 303 }
 304
 305 void
 306 xen_check_queue(void)
 307 {
 308 #ifdef INVARIANTS
 309         SET_VCPU();
 310
 311         KASSERT(XPQ_IDX == 0, ("pending operations XPQ_IDX=%d", XPQ_IDX));
 312 #endif
 313 }
 314
 315 void
 316 xen_invlpg(vm_offset_t va)
 317 {
 318         struct mmuext_op op;
 319         op.cmd = MMUEXT_INVLPG_ALL;
 320         op.arg1.linear_addr = va & ~PAGE_MASK;
 321         PANIC_IF(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
 322 }
 323
 324 void
 325 xen_load_cr3(u_int val)
 326 {
 327         struct mmuext_op op;
 328 #ifdef INVARIANTS
 329         SET_VCPU();
 330
 331         KASSERT(XPQ_IDX == 0, ("pending operations XPQ_IDX=%d", XPQ_IDX));
 332 #endif
 333         op.cmd = MMUEXT_NEW_BASEPTR;
 334         op.arg1.mfn = xpmap_ptom(val) >> PAGE_SHIFT;
 335         PANIC_IF(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
 336 }
 337
 338 #ifdef KTR
 339 static __inline u_int
 340 rebp(void)
 341 {
 342         u_int   data;
 343
 344         __asm __volatile("movl 4(%%ebp),%0" : "=r" (data));
 345         return (data);
 346 }
 347 #endif
 348
 349 u_int
 350 read_eflags(void)
 351 {
 352         vcpu_info_t *_vcpu;
 353         u_int eflags;
 354
 355         eflags = _read_eflags();
 356         _vcpu = &HYPERVISOR_shared_info->vcpu_info[smp_processor_id()];
 357         if (_vcpu->evtchn_upcall_mask)
 358                 eflags &= ~PSL_I;
 359
 360         return (eflags);
 361 }
 362
 363 void
 364 write_eflags(u_int eflags)
 365 {
 366         u_int intr;
 367
 368         CTR2(KTR_SPARE2, "%x xen_restore_flags eflags %x", rebp(), eflags);
 369         intr = ((eflags & PSL_I) == 0);
 370         __restore_flags(intr);
 371         _write_eflags(eflags);
 372 }
 373
 374 void
 375 xen_cli(void)
 376 {
 377         CTR1(KTR_SPARE2, "%x xen_cli disabling interrupts", rebp());
 378         __cli();
 379 }
 380
 381 void
 382 xen_sti(void)
 383 {
 384         CTR1(KTR_SPARE2, "%x xen_sti enabling interrupts", rebp());
 385         __sti();
 386 }
 387
 388 u_int
 389 xen_rcr2(void)
 390 {
 391
 392         return (HYPERVISOR_shared_info->vcpu_info[curcpu].arch.cr2);
 393 }
 394
 395 void
 396 _xen_machphys_update(vm_paddr_t mfn, vm_paddr_t pfn, char *file, int line)
 397 {
 398         SET_VCPU();
 399
 400         if (__predict_true(gdtset))
 401                 critical_enter();
 402         XPQ_QUEUE[XPQ_IDX].ptr = (mfn << PAGE_SHIFT) | MMU_MACHPHYS_UPDATE;
 403         XPQ_QUEUE[XPQ_IDX].val = pfn;
 404 #ifdef INVARIANTS
 405         XPQ_QUEUE_LOG[XPQ_IDX].file = file;
 406         XPQ_QUEUE_LOG[XPQ_IDX].line = line;
 407 #endif
 408         xen_increment_idx();
 409         if (__predict_true(gdtset))
 410                 critical_exit();
 411 }
 412
 413 extern struct rwlock pvh_global_lock;
 414
 415 void
 416 _xen_queue_pt_update(vm_paddr_t ptr, vm_paddr_t val, char *file, int line)
 417 {
 418         SET_VCPU();
 419
 420         if (__predict_true(gdtset))
 421                 rw_assert(&pvh_global_lock, RA_WLOCKED);
 422
 423         KASSERT((ptr & 7) == 0, ("misaligned update"));
 424
 425         if (__predict_true(gdtset))
 426                 critical_enter();
 427
 428         XPQ_QUEUE[XPQ_IDX].ptr = ((uint64_t)ptr) | MMU_NORMAL_PT_UPDATE;
 429         XPQ_QUEUE[XPQ_IDX].val = (uint64_t)val;
 430 #ifdef INVARIANTS
 431         XPQ_QUEUE_LOG[XPQ_IDX].file = file;
 432         XPQ_QUEUE_LOG[XPQ_IDX].line = line;
 433 #endif
 434         xen_increment_idx();
 435         if (__predict_true(gdtset))
 436                 critical_exit();
 437 }
 438
 439 void
 440 xen_pgdpt_pin(vm_paddr_t ma)
 441 {
 442         struct mmuext_op op;
 443         op.cmd = MMUEXT_PIN_L3_TABLE;
 444         op.arg1.mfn = ma >> PAGE_SHIFT;
 445         xen_flush_queue();
 446         PANIC_IF(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
 447 }
 448
 449 void
 450 xen_pgd_pin(vm_paddr_t ma)
 451 {
 452         struct mmuext_op op;
 453         op.cmd = MMUEXT_PIN_L2_TABLE;
 454         op.arg1.mfn = ma >> PAGE_SHIFT;
 455         xen_flush_queue();
 456         PANIC_IF(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
 457 }
 458
 459 void
 460 xen_pgd_unpin(vm_paddr_t ma)
 461 {
 462         struct mmuext_op op;
 463         op.cmd = MMUEXT_UNPIN_TABLE;
 464         op.arg1.mfn = ma >> PAGE_SHIFT;
 465         xen_flush_queue();
 466         PANIC_IF(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
 467 }
 468
 469 void
 470 xen_pt_pin(vm_paddr_t ma)
 471 {
 472         struct mmuext_op op;
 473         op.cmd = MMUEXT_PIN_L1_TABLE;
 474         op.arg1.mfn = ma >> PAGE_SHIFT;
 475         xen_flush_queue();
 476         PANIC_IF(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
 477 }
 478
 479 void
 480 xen_pt_unpin(vm_paddr_t ma)
 481 {
 482         struct mmuext_op op;
 483         op.cmd = MMUEXT_UNPIN_TABLE;
 484         op.arg1.mfn = ma >> PAGE_SHIFT;
 485         xen_flush_queue();
 486         PANIC_IF(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
 487 }
 488
 489 void
 490 xen_set_ldt(vm_paddr_t ptr, unsigned long len)
 491 {
 492         struct mmuext_op op;
 493         op.cmd = MMUEXT_SET_LDT;
 494         op.arg1.linear_addr = ptr;
 495         op.arg2.nr_ents = len;
 496         xen_flush_queue();
 497         PANIC_IF(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
 498 }
 499
 500 void xen_tlb_flush(void)
 501 {
 502         struct mmuext_op op;
 503         op.cmd = MMUEXT_TLB_FLUSH_LOCAL;
 504         xen_flush_queue();
 505         PANIC_IF(HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0);
 506 }
 507
 508 void
 509 xen_update_descriptor(union descriptor *table, union descriptor *entry)
 510 {
 511         vm_paddr_t pa;
 512         pt_entry_t *ptp;
 513
 514         ptp = vtopte((vm_offset_t)table);
 515         pa = (*ptp & PG_FRAME) | ((vm_offset_t)table & PAGE_MASK);
 516         if (HYPERVISOR_update_descriptor(pa, *(uint64_t *)entry))
 517                 panic("HYPERVISOR_update_descriptor failed\n");
 518 }
 519
 520
 521 #if 0
 522 /*
 523  * Bitmap is indexed by page number. If bit is set, the page is part of a
 524  * xen_create_contiguous_region() area of memory.
 525  */
 526 unsigned long *contiguous_bitmap;
 527
 528 static void
 529 contiguous_bitmap_set(unsigned long first_page, unsigned long nr_pages)
 530 {
 531         unsigned long start_off, end_off, curr_idx, end_idx;
 532
 533         curr_idx  = first_page / BITS_PER_LONG;
 534         start_off = first_page & (BITS_PER_LONG-1);
 535         end_idx   = (first_page + nr_pages) / BITS_PER_LONG;
 536         end_off   = (first_page + nr_pages) & (BITS_PER_LONG-1);
 537
 538         if (curr_idx == end_idx) {
 539                 contiguous_bitmap[curr_idx] |=
 540                         ((1UL<<end_off)-1) & -(1UL<<start_off);
 541         } else {
 542                 contiguous_bitmap[curr_idx] |= -(1UL<<start_off);
 543                 while ( ++curr_idx < end_idx )
 544                         contiguous_bitmap[curr_idx] = ~0UL;
 545                 contiguous_bitmap[curr_idx] |= (1UL<<end_off)-1;
 546         }
 547 }
 548
 549 static void
 550 contiguous_bitmap_clear(unsigned long first_page, unsigned long nr_pages)
 551 {
 552         unsigned long start_off, end_off, curr_idx, end_idx;
 553
 554         curr_idx  = first_page / BITS_PER_LONG;
 555         start_off = first_page & (BITS_PER_LONG-1);
 556         end_idx   = (first_page + nr_pages) / BITS_PER_LONG;
 557         end_off   = (first_page + nr_pages) & (BITS_PER_LONG-1);
 558
 559         if (curr_idx == end_idx) {
 560                 contiguous_bitmap[curr_idx] &=
 561                         -(1UL<<end_off) | ((1UL<<start_off)-1);
 562         } else {
 563                 contiguous_bitmap[curr_idx] &= (1UL<<start_off)-1;
 564                 while ( ++curr_idx != end_idx )
 565                         contiguous_bitmap[curr_idx] = 0;
 566                 contiguous_bitmap[curr_idx] &= -(1UL<<end_off);
 567         }
 568 }
 569 #endif
 570
 571 /* Ensure multi-page extents are contiguous in machine memory. */
 572 int
 573 xen_create_contiguous_region(vm_page_t pages, int npages)
 574 {
 575         unsigned long  mfn, i, flags;
 576         int order;
 577         struct xen_memory_reservation reservation = {
 578                 .nr_extents   = 1,
 579                 .extent_order = 0,
 580                 .domid        = DOMID_SELF
 581         };
 582         set_xen_guest_handle(reservation.extent_start, &mfn);
 583
 584         balloon_lock(flags);
 585
 586         /* can currently only handle power of two allocation */
 587         PANIC_IF(ffs(npages) != fls(npages));
 588
 589         /* 0. determine order */
 590         order = (ffs(npages) == fls(npages)) ? fls(npages) - 1 : fls(npages);
 591
 592         /* 1. give away machine pages. */
 593         for (i = 0; i < (1 << order); i++) {
 594                 int pfn;
 595                 pfn = VM_PAGE_TO_PHYS(&pages[i]) >> PAGE_SHIFT;
 596                 mfn = PFNTOMFN(pfn);
 597                 PFNTOMFN(pfn) = INVALID_P2M_ENTRY;
 598                 PANIC_IF(HYPERVISOR_memory_op(XENMEM_decrease_reservation, &reservation) != 1);
 599         }
 600
 601
 602         /* 2. Get a new contiguous memory extent. */
 603         reservation.extent_order = order;
 604         /* xenlinux hardcodes this because of aacraid - maybe set to 0 if we're not
 605          * running with a broxen driver XXXEN
 606          */
 607         reservation.address_bits = 31;
 608         if (HYPERVISOR_memory_op(XENMEM_increase_reservation, &reservation) != 1)
 609                 goto fail;
 610
 611         /* 3. Map the new extent in place of old pages. */
 612         for (i = 0; i < (1 << order); i++) {
 613                 int pfn;
 614                 pfn = VM_PAGE_TO_PHYS(&pages[i]) >> PAGE_SHIFT;
 615                 xen_machphys_update(mfn+i, pfn);
 616                 PFNTOMFN(pfn) = mfn+i;
 617         }
 618
 619         xen_tlb_flush();
 620
 621 #if 0
 622         contiguous_bitmap_set(VM_PAGE_TO_PHYS(&pages[0]) >> PAGE_SHIFT, 1UL << order);
 623 #endif
 624
 625         balloon_unlock(flags);
 626
 627         return 0;
 628
 629  fail:
 630         reservation.extent_order = 0;
 631         reservation.address_bits = 0;
 632
 633         for (i = 0; i < (1 << order); i++) {
 634                 int pfn;
 635                 pfn = VM_PAGE_TO_PHYS(&pages[i]) >> PAGE_SHIFT;
 636                 PANIC_IF(HYPERVISOR_memory_op(
 637                         XENMEM_increase_reservation, &reservation) != 1);
 638                 xen_machphys_update(mfn, pfn);
 639                 PFNTOMFN(pfn) = mfn;
 640         }
 641
 642         xen_tlb_flush();
 643
 644         balloon_unlock(flags);
 645
 646         return ENOMEM;
 647 }
 648
 649 void
 650 xen_destroy_contiguous_region(void *addr, int npages)
 651 {
 652         unsigned long  mfn, i, flags, order, pfn0;
 653         struct xen_memory_reservation reservation = {
 654                 .nr_extents   = 1,
 655                 .extent_order = 0,
 656                 .domid        = DOMID_SELF
 657         };
 658         set_xen_guest_handle(reservation.extent_start, &mfn);
 659
 660         pfn0 = vtophys(addr) >> PAGE_SHIFT;
 661 #if 0
 662         scrub_pages(vstart, 1 << order);
 663 #endif
 664         /* can currently only handle power of two allocation */
 665         PANIC_IF(ffs(npages) != fls(npages));
 666
 667         /* 0. determine order */
 668         order = (ffs(npages) == fls(npages)) ? fls(npages) - 1 : fls(npages);
 669
 670         balloon_lock(flags);
 671
 672 #if 0
 673         contiguous_bitmap_clear(vtophys(addr) >> PAGE_SHIFT, 1UL << order);
 674 #endif
 675
 676         /* 1. Zap current PTEs, giving away the underlying pages. */
 677         for (i = 0; i < (1 << order); i++) {
 678                 int pfn;
 679                 uint64_t new_val = 0;
 680                 pfn = vtomach((char *)addr + i*PAGE_SIZE) >> PAGE_SHIFT;
 681
 682                 PANIC_IF(HYPERVISOR_update_va_mapping((vm_offset_t)((char *)addr + (i * PAGE_SIZE)), new_val, 0));
 683                 PFNTOMFN(pfn) = INVALID_P2M_ENTRY;
 684                 PANIC_IF(HYPERVISOR_memory_op(
 685                         XENMEM_decrease_reservation, &reservation) != 1);
 686         }
 687
 688         /* 2. Map new pages in place of old pages. */
 689         for (i = 0; i < (1 << order); i++) {
 690                 int pfn;
 691                 uint64_t new_val;
 692                 pfn = pfn0 + i;
 693                 PANIC_IF(HYPERVISOR_memory_op(XENMEM_increase_reservation, &reservation) != 1);
 694
 695                 new_val = mfn << PAGE_SHIFT;
 696                 PANIC_IF(HYPERVISOR_update_va_mapping((vm_offset_t)addr + (i * PAGE_SIZE),
 697                                                       new_val, PG_KERNEL));
 698                 xen_machphys_update(mfn, pfn);
 699                 PFNTOMFN(pfn) = mfn;
 700         }
 701
 702         xen_tlb_flush();
 703
 704         balloon_unlock(flags);
 705 }
 706
 707 extern  vm_offset_t     proc0kstack;
 708 extern int vm86paddr, vm86phystk;
 709 char *bootmem_start, *bootmem_current, *bootmem_end;
 710
 711 pteinfo_t *pteinfo_list;
 712 void initvalues(start_info_t *startinfo);
 713
 714 void *
 715 bootmem_alloc(unsigned int size)
 716 {
 717         char *retptr;
 718
 719         retptr = bootmem_current;
 720         PANIC_IF(retptr + size > bootmem_end);
 721         bootmem_current += size;
 722
 723         return retptr;
 724 }
 725
 726 void
 727 bootmem_free(void *ptr, unsigned int size)
 728 {
 729         char *tptr;
 730
 731         tptr = ptr;
 732         PANIC_IF(tptr != bootmem_current - size ||
 733                 bootmem_current - size < bootmem_start);
 734
 735         bootmem_current -= size;
 736 }
 737
 738 #if 0
 739 static vm_paddr_t
 740 xpmap_mtop2(vm_paddr_t mpa)
 741 {
 742         return ((machine_to_phys_mapping[mpa >> PAGE_SHIFT] << PAGE_SHIFT)
 743             ) | (mpa & ~PG_FRAME);
 744 }
 745
 746 static pd_entry_t
 747 xpmap_get_bootpde(vm_paddr_t va)
 748 {
 749
 750         return ((pd_entry_t *)xen_start_info->pt_base)[va >> 22];
 751 }
 752
 753 static pd_entry_t
 754 xpmap_get_vbootpde(vm_paddr_t va)
 755 {
 756         pd_entry_t pde;
 757
 758         pde = xpmap_get_bootpde(va);
 759         if ((pde & PG_V) == 0)
 760                 return (pde & ~PG_FRAME);
 761         return (pde & ~PG_FRAME) |
 762                 (xpmap_mtop2(pde & PG_FRAME) + KERNBASE);
 763 }
 764
 765 static pt_entry_t 8*
 766 xpmap_get_bootptep(vm_paddr_t va)
 767 {
 768         pd_entry_t pde;
 769
 770         pde = xpmap_get_vbootpde(va);
 771         if ((pde & PG_V) == 0)
 772                 return (void *)-1;
 773 #define PT_MASK         0x003ff000      /* page table address bits */
 774         return &(((pt_entry_t *)(pde & PG_FRAME))[(va & PT_MASK) >> PAGE_SHIFT]);
 775 }
 776
 777 static pt_entry_t
 778 xpmap_get_bootpte(vm_paddr_t va)
 779 {
 780
 781         return xpmap_get_bootptep(va)[0];
 782 }
 783 #endif
 784
 785
 786 #ifdef ADD_ISA_HOLE
 787 static void
 788 shift_phys_machine(unsigned long *phys_machine, int nr_pages)
 789 {
 790
 791         unsigned long *tmp_page, *current_page, *next_page;
 792         int i;
 793
 794         tmp_page = bootmem_alloc(PAGE_SIZE);
 795         current_page = phys_machine + nr_pages - (PAGE_SIZE/sizeof(unsigned long));
 796         next_page = current_page - (PAGE_SIZE/sizeof(unsigned long));
 797         bcopy(phys_machine, tmp_page, PAGE_SIZE);
 798
 799         while (current_page > phys_machine) {
 800                 /*  save next page */
 801                 bcopy(next_page, tmp_page, PAGE_SIZE);
 802                 /* shift down page */
 803                 bcopy(current_page, next_page, PAGE_SIZE);
 804                 /*  finish swap */
 805                 bcopy(tmp_page, current_page, PAGE_SIZE);
 806
 807                 current_page -= (PAGE_SIZE/sizeof(unsigned long));
 808                 next_page -= (PAGE_SIZE/sizeof(unsigned long));
 809         }
 810         bootmem_free(tmp_page, PAGE_SIZE);
 811
 812         for (i = 0; i < nr_pages; i++) {
 813                 xen_machphys_update(phys_machine[i], i);
 814         }
 815         memset(phys_machine, INVALID_P2M_ENTRY, PAGE_SIZE);
 816
 817 }
 818 #endif /* ADD_ISA_HOLE */
 819
 820 /*
 821  * Build a directory of the pages that make up our Physical to Machine
 822  * mapping table. The Xen suspend/restore code uses this to find our
 823  * mapping table.
 824  */
 825 static void
 826 init_frame_list_list(void *arg)
 827 {
 828         unsigned long nr_pages = xen_start_info->nr_pages;
 829 #define FPP     (PAGE_SIZE/sizeof(xen_pfn_t))
 830         int i, j, k;
 831
 832         xen_pfn_to_mfn_frame_list_list = malloc(PAGE_SIZE, M_DEVBUF, M_WAITOK);
 833         for (i = 0, j = 0, k = -1; i < nr_pages;
 834              i += FPP, j++) {
 835                 if ((j & (FPP - 1)) == 0) {
 836                         k++;
 837                         xen_pfn_to_mfn_frame_list[k] =
 838                                 malloc(PAGE_SIZE, M_DEVBUF, M_WAITOK);
 839                         xen_pfn_to_mfn_frame_list_list[k] =
 840                                 VTOMFN(xen_pfn_to_mfn_frame_list[k]);
 841                         j = 0;
 842                 }
 843                 xen_pfn_to_mfn_frame_list[k][j] =
 844                         VTOMFN(&xen_phys_machine[i]);
 845         }
 846
 847         HYPERVISOR_shared_info->arch.max_pfn = nr_pages;
 848         HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list_list
 849                 = VTOMFN(xen_pfn_to_mfn_frame_list_list);
 850 }
 851 SYSINIT(init_fll, SI_SUB_DEVFS, SI_ORDER_ANY, init_frame_list_list, NULL);
 852
 853 extern unsigned long physfree;
 854
 855 int pdir, curoffset;
 856 extern int nkpt;
 857
 858 extern uint32_t kernbase;
 859
 860 void
 861 initvalues(start_info_t *startinfo)
 862 {
 863         vm_offset_t cur_space, cur_space_pt;
 864         struct physdev_set_iopl set_iopl;
 865
 866         int l3_pages, l2_pages, l1_pages, offset;
 867         vm_paddr_t console_page_ma, xen_store_ma;
 868         vm_offset_t tmpva;
 869         vm_paddr_t shinfo;
 870 #ifdef PAE
 871         vm_paddr_t IdlePDPTma, IdlePDPTnewma;
 872         vm_paddr_t IdlePTDnewma[4];
 873         pd_entry_t *IdlePDPTnew, *IdlePTDnew;
 874         vm_paddr_t IdlePTDma[4];
 875 #else
 876         vm_paddr_t IdlePTDma[1];
 877 #endif
 878         unsigned long i;
 879         int ncpus = MAXCPU;
 880
 881         nkpt = min(
 882                 min(
 883                         max((startinfo->nr_pages >> NPGPTD_SHIFT), nkpt),
 884                     NPGPTD*NPDEPG - KPTDI),
 885                     (HYPERVISOR_VIRT_START - KERNBASE) >> PDRSHIFT);
 886
 887         HYPERVISOR_vm_assist(VMASST_CMD_enable, VMASST_TYPE_4gb_segments);
 888 #ifdef notyet
 889         /*
 890          * need to install handler
 891          */
 892         HYPERVISOR_vm_assist(VMASST_CMD_enable, VMASST_TYPE_4gb_segments_notify);
 893 #endif
 894         xen_start_info = startinfo;
 895         HYPERVISOR_start_info = startinfo;
 896         xen_phys_machine = (xen_pfn_t *)startinfo->mfn_list;
 897
 898         IdlePTD = (pd_entry_t *)((uint8_t *)startinfo->pt_base + PAGE_SIZE);
 899         l1_pages = 0;
 900
 901 #ifdef PAE
 902         l3_pages = 1;
 903         l2_pages = 0;
 904         IdlePDPT = (pd_entry_t *)startinfo->pt_base;
 905         IdlePDPTma = VTOM(startinfo->pt_base);
 906         for (i = (KERNBASE >> 30);
 907              (i < 4) && (IdlePDPT[i] != 0); i++)
 908                         l2_pages++;
 909         /*
 910          * Note that only one page directory has been allocated at this point.
 911          * Thus, if KERNBASE
 912          */
 913         for (i = 0; i < l2_pages; i++)
 914                 IdlePTDma[i] = VTOM(IdlePTD + i*PAGE_SIZE);
 915
 916         l2_pages = (l2_pages == 0) ? 1 : l2_pages;
 917 #else
 918         l3_pages = 0;
 919         l2_pages = 1;
 920 #endif
 921         for (i = (((KERNBASE>>18) & PAGE_MASK)>>PAGE_SHIFT);
 922              (i<l2_pages*NPDEPG) && (i<(VM_MAX_KERNEL_ADDRESS>>PDRSHIFT)); i++) {
 923
 924                 if (IdlePTD[i] == 0)
 925                         break;
 926                 l1_pages++;
 927         }
 928
 929         /* number of pages allocated after the pts + 1*/;
 930         cur_space = xen_start_info->pt_base +
 931             (l3_pages + l2_pages + l1_pages + 1)*PAGE_SIZE;
 932
 933         xc_printf("initvalues(): wooh - availmem=%x,%x\n", avail_space,
 934             cur_space);
 935
 936         xc_printf("KERNBASE=%x,pt_base=%lx, VTOPFN(base)=%x, nr_pt_frames=%lx\n",
 937             KERNBASE,xen_start_info->pt_base, VTOPFN(xen_start_info->pt_base),
 938             xen_start_info->nr_pt_frames);
 939         xendebug_flags = 0; /* 0xffffffff; */
 940
 941 #ifdef ADD_ISA_HOLE
 942         shift_phys_machine(xen_phys_machine, xen_start_info->nr_pages);
 943 #endif
 944         XENPRINTF("IdlePTD %p\n", IdlePTD);
 945         XENPRINTF("nr_pages: %ld shared_info: 0x%lx flags: 0x%x pt_base: 0x%lx "
 946                   "mod_start: 0x%lx mod_len: 0x%lx\n",
 947                   xen_start_info->nr_pages, xen_start_info->shared_info,
 948                   xen_start_info->flags, xen_start_info->pt_base,
 949                   xen_start_info->mod_start, xen_start_info->mod_len);
 950
 951 #ifdef PAE
 952         IdlePDPTnew = (pd_entry_t *)cur_space; cur_space += PAGE_SIZE;
 953         bzero(IdlePDPTnew, PAGE_SIZE);
 954
 955         IdlePDPTnewma =  VTOM(IdlePDPTnew);
 956         IdlePTDnew = (pd_entry_t *)cur_space; cur_space += 4*PAGE_SIZE;
 957         bzero(IdlePTDnew, 4*PAGE_SIZE);
 958
 959         for (i = 0; i < 4; i++)
 960                 IdlePTDnewma[i] = VTOM((uint8_t *)IdlePTDnew + i*PAGE_SIZE);
 961         /*
 962          * L3
 963          *
 964          * Copy the 4 machine addresses of the new PTDs in to the PDPT
 965          *
 966          */
 967         for (i = 0; i < 4; i++)
 968                 IdlePDPTnew[i] = IdlePTDnewma[i] | PG_V;
 969
 970         __asm__("nop;");
 971         /*
 972          *
 973          * re-map the new PDPT read-only
 974          */
 975         PT_SET_MA(IdlePDPTnew, IdlePDPTnewma | PG_V);
 976         /*
 977          *
 978          * Unpin the current PDPT
 979          */
 980         xen_pt_unpin(IdlePDPTma);
 981
 982 #endif  /* PAE */
 983
 984         /* Map proc0's KSTACK */
 985         proc0kstack = cur_space; cur_space += (KSTACK_PAGES * PAGE_SIZE);
 986         xc_printf("proc0kstack=%u\n", proc0kstack);
 987
 988         /* vm86/bios stack */
 989         cur_space += PAGE_SIZE;
 990
 991         /* Map space for the vm86 region */
 992         vm86paddr = (vm_offset_t)cur_space;
 993         cur_space += (PAGE_SIZE * 3);
 994
 995         /* allocate 4 pages for bootmem allocator */
 996         bootmem_start = bootmem_current = (char *)cur_space;
 997         cur_space += (4 * PAGE_SIZE);
 998         bootmem_end = (char *)cur_space;
 999
1000         /* allocate pages for gdt */
1001         gdt = (union descriptor *)cur_space;
1002         cur_space += PAGE_SIZE*ncpus;
1003
1004         /* allocate page for ldt */
1005         ldt = (union descriptor *)cur_space; cur_space += PAGE_SIZE;
1006         cur_space += PAGE_SIZE;
1007
1008         /* unmap remaining pages from initial chunk
1009          *
1010          */
1011         for (tmpva = cur_space; tmpva < (((uint32_t)&kernbase) + (l1_pages<<PDRSHIFT));
1012              tmpva += PAGE_SIZE) {
1013                 bzero((char *)tmpva, PAGE_SIZE);
1014                 PT_SET_MA(tmpva, (vm_paddr_t)0);
1015         }
1016
1017         PT_UPDATES_FLUSH();
1018
1019         memcpy(((uint8_t *)IdlePTDnew) + ((unsigned int)(KERNBASE >> 18)),
1020             ((uint8_t *)IdlePTD) + ((KERNBASE >> 18) & PAGE_MASK),
1021             l1_pages*sizeof(pt_entry_t));
1022
1023         for (i = 0; i < 4; i++) {
1024                 PT_SET_MA((uint8_t *)IdlePTDnew + i*PAGE_SIZE,
1025                     IdlePTDnewma[i] | PG_V);
1026         }
1027         xen_load_cr3(VTOP(IdlePDPTnew));
1028         xen_pgdpt_pin(VTOM(IdlePDPTnew));
1029
1030         /* allocate remainder of nkpt pages */
1031         cur_space_pt = cur_space;
1032         for (offset = (KERNBASE >> PDRSHIFT), i = l1_pages; i < nkpt;
1033              i++, cur_space += PAGE_SIZE) {
1034                 pdir = (offset + i) / NPDEPG;
1035                 curoffset = ((offset + i) % NPDEPG);
1036                 if (((offset + i) << PDRSHIFT) == VM_MAX_KERNEL_ADDRESS)
1037                         break;
1038
1039                 /*
1040                  * make sure that all the initial page table pages
1041                  * have been zeroed
1042                  */
1043                 PT_SET_MA(cur_space, VTOM(cur_space) | PG_V | PG_RW);
1044                 bzero((char *)cur_space, PAGE_SIZE);
1045                 PT_SET_MA(cur_space, (vm_paddr_t)0);
1046                 xen_pt_pin(VTOM(cur_space));
1047                 xen_queue_pt_update((vm_paddr_t)(IdlePTDnewma[pdir] +
1048                         curoffset*sizeof(vm_paddr_t)),
1049                     VTOM(cur_space) | PG_KERNEL);
1050                 PT_UPDATES_FLUSH();
1051         }
1052
1053         for (i = 0; i < 4; i++) {
1054                 pdir = (PTDPTDI + i) / NPDEPG;
1055                 curoffset = (PTDPTDI + i) % NPDEPG;
1056
1057                 xen_queue_pt_update((vm_paddr_t)(IdlePTDnewma[pdir] +
1058                         curoffset*sizeof(vm_paddr_t)),
1059                     IdlePTDnewma[i] | PG_V);
1060         }
1061
1062         PT_UPDATES_FLUSH();
1063
1064         IdlePTD = IdlePTDnew;
1065         IdlePDPT = IdlePDPTnew;
1066         IdlePDPTma = IdlePDPTnewma;
1067
1068         HYPERVISOR_shared_info = (shared_info_t *)cur_space;
1069         cur_space += PAGE_SIZE;
1070
1071         xen_store = (struct xenstore_domain_interface *)cur_space;
1072         cur_space += PAGE_SIZE;
1073
1074         console_page = (char *)cur_space;
1075         cur_space += PAGE_SIZE;
1076
1077         /*
1078          * shared_info is an unsigned long so this will randomly break if
1079          * it is allocated above 4GB - I guess people are used to that
1080          * sort of thing with Xen ... sigh
1081          */
1082         shinfo = xen_start_info->shared_info;
1083         PT_SET_MA(HYPERVISOR_shared_info, shinfo | PG_KERNEL);
1084
1085         xc_printf("#4\n");
1086
1087         xen_store_ma = (((vm_paddr_t)xen_start_info->store_mfn) << PAGE_SHIFT);
1088         PT_SET_MA(xen_store, xen_store_ma | PG_KERNEL);
1089         console_page_ma = (((vm_paddr_t)xen_start_info->console.domU.mfn) << PAGE_SHIFT);
1090         PT_SET_MA(console_page, console_page_ma | PG_KERNEL);
1091
1092         xc_printf("#5\n");
1093
1094         set_iopl.iopl = 1;
1095         PANIC_IF(HYPERVISOR_physdev_op(PHYSDEVOP_SET_IOPL, &set_iopl));
1096         xc_printf("#6\n");
1097 #if 0
1098         /* add page table for KERNBASE */
1099         xen_queue_pt_update(IdlePTDma + KPTDI*sizeof(vm_paddr_t),
1100                             VTOM(cur_space) | PG_KERNEL);
1101         xen_flush_queue();
1102 #ifdef PAE
1103         xen_queue_pt_update(pdir_shadow_ma[3] + KPTDI*sizeof(vm_paddr_t),
1104                             VTOM(cur_space) | PG_V | PG_A);
1105 #else
1106         xen_queue_pt_update(pdir_shadow_ma + KPTDI*sizeof(vm_paddr_t),
1107                             VTOM(cur_space) | PG_V | PG_A);
1108 #endif
1109         xen_flush_queue();
1110         cur_space += PAGE_SIZE;
1111         xc_printf("#6\n");
1112 #endif /* 0 */
1113 #ifdef notyet
1114         if (xen_start_info->flags & SIF_INITDOMAIN) {
1115                 /* Map first megabyte */
1116                 for (i = 0; i < (256 << PAGE_SHIFT); i += PAGE_SIZE)
1117                         PT_SET_MA(KERNBASE + i, i | PG_KERNEL | PG_NC_PCD);
1118                 xen_flush_queue();
1119         }
1120 #endif
1121         /*
1122          * re-map kernel text read-only
1123          *
1124          */
1125         for (i = (((vm_offset_t)&btext) & ~PAGE_MASK);
1126              i < (((vm_offset_t)&etext) & ~PAGE_MASK); i += PAGE_SIZE)
1127                 PT_SET_MA(i, VTOM(i) | PG_V | PG_A);
1128
1129         xc_printf("#7\n");
1130         physfree = VTOP(cur_space);
1131         init_first = physfree >> PAGE_SHIFT;
1132         IdlePTD = (pd_entry_t *)VTOP(IdlePTD);
1133         IdlePDPT = (pd_entry_t *)VTOP(IdlePDPT);
1134         setup_xen_features();
1135         xc_printf("#8, proc0kstack=%u\n", proc0kstack);
1136 }
1137
1138
1139 trap_info_t trap_table[] = {
1140         { 0,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(div)},
1141         { 1,   0|4, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(dbg)},
1142         { 3,   3|4, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(bpt)},
1143         { 4,   3, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(ofl)},
1144         /* This is UPL on Linux and KPL on BSD */
1145         { 5,   3, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(bnd)},
1146         { 6,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(ill)},
1147         { 7,   0|4, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(dna)},
1148         /*
1149          * { 8,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(XXX)},
1150          *   no handler for double fault
1151          */
1152         { 9,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(fpusegm)},
1153         {10,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(tss)},
1154         {11,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(missing)},
1155         {12,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(stk)},
1156         {13,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(prot)},
1157         {14,   0|4, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(page)},
1158         {15,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(rsvd)},
1159         {16,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(fpu)},
1160         {17,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(align)},
1161         {18,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(mchk)},
1162         {19,   0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(xmm)},
1163         {0x80, 3, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(int0x80_syscall)},
1164         {  0, 0,           0, 0 }
1165 };
1166
1167 /* Perform a multicall and check that individual calls succeeded. */
1168 int
1169 HYPERVISOR_multicall(struct multicall_entry * call_list, int nr_calls)
1170 {
1171         int ret = 0;
1172         int i;
1173
1174         /* Perform the multicall. */
1175         PANIC_IF(_HYPERVISOR_multicall(call_list, nr_calls));
1176
1177         /* Check the results of individual hypercalls. */
1178         for (i = 0; i < nr_calls; i++)
1179                 if (__predict_false(call_list[i].result < 0))
1180                         ret++;
1181         if (__predict_false(ret > 0))
1182                 panic("%d multicall(s) failed: cpu %d\n",
1183                     ret, smp_processor_id());
1184
1185         /* If we didn't panic already, everything succeeded. */
1186         return (0);
1187 }
1188
1189 /********** CODE WORTH KEEPING ABOVE HERE *****************/
1190
1191 void xen_failsafe_handler(void);
1192
1193 void
1194 xen_failsafe_handler(void)
1195 {
1196
1197         panic("xen_failsafe_handler called!\n");
1198 }
1199
1200 void xen_handle_thread_switch(struct pcb *pcb);
1201
1202 /* This is called by cpu_switch() when switching threads. */
1203 /* The pcb arg refers to the process control block of the */
1204 /* next thread which is to run */
1205 void
1206 xen_handle_thread_switch(struct pcb *pcb)
1207 {
1208     uint32_t *a = (uint32_t *)&PCPU_GET(fsgs_gdt)[0];
1209     uint32_t *b = (uint32_t *)&pcb->pcb_fsd;
1210     multicall_entry_t mcl[3];
1211     int i = 0;
1212
1213     /* Notify Xen of task switch */
1214     mcl[i].op = __HYPERVISOR_stack_switch;
1215     mcl[i].args[0] = GSEL(GDATA_SEL, SEL_KPL);
1216     mcl[i++].args[1] = (unsigned long)pcb;
1217
1218     /* Check for update of fsd */
1219     if (*a != *b || *(a+1) != *(b+1)) {
1220         mcl[i].op = __HYPERVISOR_update_descriptor;
1221         *(uint64_t *)&mcl[i].args[0] = vtomach((vm_offset_t)a);
1222         *(uint64_t *)&mcl[i++].args[2] = *(uint64_t *)b;
1223     }
1224
1225     a += 2;
1226     b += 2;
1227
1228     /* Check for update of gsd */
1229     if (*a != *b || *(a+1) != *(b+1)) {
1230         mcl[i].op = __HYPERVISOR_update_descriptor;
1231         *(uint64_t *)&mcl[i].args[0] = vtomach((vm_offset_t)a);
1232         *(uint64_t *)&mcl[i++].args[2] = *(uint64_t *)b;
1233     }
1234
1235     (void)HYPERVISOR_multicall(mcl, i);
1236 }