sys/x86/xen/xen_apic.c

   1 /*
   2  * Copyright (c) 2014 Roger Pau Monné <roger.pau@citrix.com>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions and the following disclaimer.
  10  * 2. Redistributions in binary form must reproduce the above copyright
  11  *    notice, this list of conditions and the following disclaimer in the
  12  *    documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS AS IS'' AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 #include <sys/cdefs.h>
  28 __FBSDID("$FreeBSD$");
  29
  30 #include <sys/param.h>
  31 #include <sys/bus.h>
  32 #include <sys/kernel.h>
  33 #include <sys/malloc.h>
  34 #include <sys/proc.h>
  35 #include <sys/smp.h>
  36 #include <sys/systm.h>
  37
  38 #include <vm/vm.h>
  39 #include <vm/pmap.h>
  40
  41 #include <machine/cpufunc.h>
  42 #include <machine/cpu.h>
  43 #include <machine/intr_machdep.h>
  44 #include <machine/md_var.h>
  45 #include <machine/smp.h>
  46
  47 #include <x86/apicreg.h>
  48 #include <x86/apicvar.h>
  49
  50 #include <xen/xen-os.h>
  51 #include <xen/features.h>
  52 #include <xen/gnttab.h>
  53 #include <xen/hypervisor.h>
  54 #include <xen/hvm.h>
  55 #include <xen/xen_intr.h>
  56
  57 #include <xen/interface/vcpu.h>
  58
  59 /*--------------------------------- Macros -----------------------------------*/
  60
  61 #define XEN_APIC_UNSUPPORTED \
  62         panic("%s: not available in Xen PV port.", __func__)
  63
  64
  65 /*--------------------------- Forward Declarations ---------------------------*/
  66 #ifdef SMP
  67 static driver_filter_t xen_smp_rendezvous_action;
  68 static driver_filter_t xen_invltlb;
  69 static driver_filter_t xen_invlpg;
  70 static driver_filter_t xen_invlrng;
  71 static driver_filter_t xen_invlcache;
  72 static driver_filter_t xen_ipi_bitmap_handler;
  73 static driver_filter_t xen_cpustop_handler;
  74 static driver_filter_t xen_cpususpend_handler;
  75 #endif
  76
  77 /*---------------------------------- Macros ----------------------------------*/
  78 #define IPI_TO_IDX(ipi) ((ipi) - APIC_IPI_INTS)
  79
  80 /*--------------------------------- Xen IPIs ---------------------------------*/
  81 #ifdef SMP
  82 struct xen_ipi_handler
  83 {
  84         driver_filter_t *filter;
  85         const char      *description;
  86 };
  87
  88 static struct xen_ipi_handler xen_ipis[] =
  89 {
  90         [IPI_TO_IDX(IPI_RENDEZVOUS)]    = { xen_smp_rendezvous_action,  "r"   },
  91         [IPI_TO_IDX(IPI_INVLTLB)]       = { xen_invltlb,                "itlb"},
  92         [IPI_TO_IDX(IPI_INVLPG)]        = { xen_invlpg,                 "ipg" },
  93         [IPI_TO_IDX(IPI_INVLRNG)]       = { xen_invlrng,                "irg" },
  94         [IPI_TO_IDX(IPI_INVLCACHE)]     = { xen_invlcache,              "ic"  },
  95         [IPI_TO_IDX(IPI_BITMAP_VECTOR)] = { xen_ipi_bitmap_handler,     "b"   },
  96         [IPI_TO_IDX(IPI_STOP)]          = { xen_cpustop_handler,        "st"  },
  97         [IPI_TO_IDX(IPI_SUSPEND)]       = { xen_cpususpend_handler,     "sp"  },
  98 };
  99 #endif
 100
 101 /*------------------------------- Per-CPU Data -------------------------------*/
 102 #ifdef SMP
 103 DPCPU_DEFINE(xen_intr_handle_t, ipi_handle[nitems(xen_ipis)]);
 104 #endif
 105
 106 /*------------------------------- Xen PV APIC --------------------------------*/
 107
 108 static void
 109 xen_pv_lapic_create(u_int apic_id, int boot_cpu)
 110 {
 111 #ifdef SMP
 112         cpu_add(apic_id, boot_cpu);
 113 #endif
 114 }
 115
 116 static void
 117 xen_pv_lapic_init(vm_paddr_t addr)
 118 {
 119
 120 }
 121
 122 static void
 123 xen_pv_lapic_setup(int boot)
 124 {
 125
 126 }
 127
 128 static void
 129 xen_pv_lapic_dump(const char *str)
 130 {
 131
 132         printf("cpu%d %s XEN PV LAPIC\n", PCPU_GET(cpuid), str);
 133 }
 134
 135 static void
 136 xen_pv_lapic_disable(void)
 137 {
 138
 139 }
 140
 141 static bool
 142 xen_pv_lapic_is_x2apic(void)
 143 {
 144
 145         return (false);
 146 }
 147
 148 static void
 149 xen_pv_lapic_eoi(void)
 150 {
 151
 152         XEN_APIC_UNSUPPORTED;
 153 }
 154
 155 static int
 156 xen_pv_lapic_id(void)
 157 {
 158
 159         return (PCPU_GET(apic_id));
 160 }
 161
 162 static int
 163 xen_pv_lapic_intr_pending(u_int vector)
 164 {
 165
 166         XEN_APIC_UNSUPPORTED;
 167         return (0);
 168 }
 169
 170 static u_int
 171 xen_pv_apic_cpuid(u_int apic_id)
 172 {
 173 #ifdef SMP
 174         return (apic_cpuids[apic_id]);
 175 #else
 176         return (0);
 177 #endif
 178 }
 179
 180 static u_int
 181 xen_pv_apic_alloc_vector(u_int apic_id, u_int irq)
 182 {
 183
 184         XEN_APIC_UNSUPPORTED;
 185         return (0);
 186 }
 187
 188 static u_int
 189 xen_pv_apic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count, u_int align)
 190 {
 191
 192         XEN_APIC_UNSUPPORTED;
 193         return (0);
 194 }
 195
 196 static void
 197 xen_pv_apic_disable_vector(u_int apic_id, u_int vector)
 198 {
 199
 200         XEN_APIC_UNSUPPORTED;
 201 }
 202
 203 static void
 204 xen_pv_apic_enable_vector(u_int apic_id, u_int vector)
 205 {
 206
 207         XEN_APIC_UNSUPPORTED;
 208 }
 209
 210 static void
 211 xen_pv_apic_free_vector(u_int apic_id, u_int vector, u_int irq)
 212 {
 213
 214         XEN_APIC_UNSUPPORTED;
 215 }
 216
 217 static void
 218 xen_pv_lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id)
 219 {
 220
 221         XEN_APIC_UNSUPPORTED;
 222 }
 223
 224 static int
 225 xen_pv_lapic_enable_pmc(void)
 226 {
 227
 228         XEN_APIC_UNSUPPORTED;
 229         return (0);
 230 }
 231
 232 static void
 233 xen_pv_lapic_disable_pmc(void)
 234 {
 235
 236         XEN_APIC_UNSUPPORTED;
 237 }
 238
 239 static void
 240 xen_pv_lapic_reenable_pmc(void)
 241 {
 242
 243         XEN_APIC_UNSUPPORTED;
 244 }
 245
 246 static void
 247 xen_pv_lapic_enable_cmc(void)
 248 {
 249
 250 }
 251
 252 #ifdef SMP
 253 static void
 254 xen_pv_lapic_ipi_raw(register_t icrlo, u_int dest)
 255 {
 256
 257         XEN_APIC_UNSUPPORTED;
 258 }
 259
 260 #define PCPU_ID_GET(id, field) (pcpu_find(id)->pc_##field)
 261 static void
 262 send_nmi(int dest)
 263 {
 264         unsigned int cpu;
 265
 266         /*
 267          * NMIs are not routed over event channels, and instead delivered as on
 268          * native using the exception vector (#2). Triggering them can be done
 269          * using the local APIC, or an hypercall as a shortcut like it's done
 270          * below.
 271          */
 272         switch(dest) {
 273         case APIC_IPI_DEST_SELF:
 274                 HYPERVISOR_vcpu_op(VCPUOP_send_nmi, PCPU_GET(vcpu_id), NULL);
 275                 break;
 276         case APIC_IPI_DEST_ALL:
 277                 CPU_FOREACH(cpu)
 278                         HYPERVISOR_vcpu_op(VCPUOP_send_nmi,
 279                             PCPU_ID_GET(cpu, vcpu_id), NULL);
 280                 break;
 281         case APIC_IPI_DEST_OTHERS:
 282                 CPU_FOREACH(cpu)
 283                         if (cpu != PCPU_GET(cpuid))
 284                                 HYPERVISOR_vcpu_op(VCPUOP_send_nmi,
 285                                     PCPU_ID_GET(cpu, vcpu_id), NULL);
 286                 break;
 287         default:
 288                 HYPERVISOR_vcpu_op(VCPUOP_send_nmi,
 289                     PCPU_ID_GET(apic_cpuid(dest), vcpu_id), NULL);
 290                 break;
 291         }
 292 }
 293 #undef PCPU_ID_GET
 294
 295 static void
 296 xen_pv_lapic_ipi_vectored(u_int vector, int dest)
 297 {
 298         xen_intr_handle_t *ipi_handle;
 299         int ipi_idx, to_cpu, self;
 300
 301         if (vector >= IPI_NMI_FIRST) {
 302                 send_nmi(dest);
 303                 return;
 304         }
 305
 306         ipi_idx = IPI_TO_IDX(vector);
 307         if (ipi_idx >= nitems(xen_ipis))
 308                 panic("IPI out of range");
 309
 310         switch(dest) {
 311         case APIC_IPI_DEST_SELF:
 312                 ipi_handle = DPCPU_GET(ipi_handle);
 313                 xen_intr_signal(ipi_handle[ipi_idx]);
 314                 break;
 315         case APIC_IPI_DEST_ALL:
 316                 CPU_FOREACH(to_cpu) {
 317                         ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle);
 318                         xen_intr_signal(ipi_handle[ipi_idx]);
 319                 }
 320                 break;
 321         case APIC_IPI_DEST_OTHERS:
 322                 self = PCPU_GET(cpuid);
 323                 CPU_FOREACH(to_cpu) {
 324                         if (to_cpu != self) {
 325                                 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle);
 326                                 xen_intr_signal(ipi_handle[ipi_idx]);
 327                         }
 328                 }
 329                 break;
 330         default:
 331                 to_cpu = apic_cpuid(dest);
 332                 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle);
 333                 xen_intr_signal(ipi_handle[ipi_idx]);
 334                 break;
 335         }
 336 }
 337
 338 static int
 339 xen_pv_lapic_ipi_wait(int delay)
 340 {
 341
 342         XEN_APIC_UNSUPPORTED;
 343         return (0);
 344 }
 345 #endif  /* SMP */
 346
 347 static int
 348 xen_pv_lapic_ipi_alloc(inthand_t *ipifunc)
 349 {
 350
 351         XEN_APIC_UNSUPPORTED;
 352         return (-1);
 353 }
 354
 355 static void
 356 xen_pv_lapic_ipi_free(int vector)
 357 {
 358
 359         XEN_APIC_UNSUPPORTED;
 360 }
 361
 362 static int
 363 xen_pv_lapic_set_lvt_mask(u_int apic_id, u_int lvt, u_char masked)
 364 {
 365
 366         XEN_APIC_UNSUPPORTED;
 367         return (0);
 368 }
 369
 370 static int
 371 xen_pv_lapic_set_lvt_mode(u_int apic_id, u_int lvt, uint32_t mode)
 372 {
 373
 374         XEN_APIC_UNSUPPORTED;
 375         return (0);
 376 }
 377
 378 static int
 379 xen_pv_lapic_set_lvt_polarity(u_int apic_id, u_int lvt, enum intr_polarity pol)
 380 {
 381
 382         XEN_APIC_UNSUPPORTED;
 383         return (0);
 384 }
 385
 386 static int
 387 xen_pv_lapic_set_lvt_triggermode(u_int apic_id, u_int lvt,
 388     enum intr_trigger trigger)
 389 {
 390
 391         XEN_APIC_UNSUPPORTED;
 392         return (0);
 393 }
 394
 395 /* Xen apic_ops implementation */
 396 struct apic_ops xen_apic_ops = {
 397         .create                 = xen_pv_lapic_create,
 398         .init                   = xen_pv_lapic_init,
 399         .xapic_mode             = xen_pv_lapic_disable,
 400         .is_x2apic              = xen_pv_lapic_is_x2apic,
 401         .setup                  = xen_pv_lapic_setup,
 402         .dump                   = xen_pv_lapic_dump,
 403         .disable                = xen_pv_lapic_disable,
 404         .eoi                    = xen_pv_lapic_eoi,
 405         .id                     = xen_pv_lapic_id,
 406         .intr_pending           = xen_pv_lapic_intr_pending,
 407         .set_logical_id         = xen_pv_lapic_set_logical_id,
 408         .cpuid                  = xen_pv_apic_cpuid,
 409         .alloc_vector           = xen_pv_apic_alloc_vector,
 410         .alloc_vectors          = xen_pv_apic_alloc_vectors,
 411         .enable_vector          = xen_pv_apic_enable_vector,
 412         .disable_vector         = xen_pv_apic_disable_vector,
 413         .free_vector            = xen_pv_apic_free_vector,
 414         .enable_pmc             = xen_pv_lapic_enable_pmc,
 415         .disable_pmc            = xen_pv_lapic_disable_pmc,
 416         .reenable_pmc           = xen_pv_lapic_reenable_pmc,
 417         .enable_cmc             = xen_pv_lapic_enable_cmc,
 418 #ifdef SMP
 419         .ipi_raw                = xen_pv_lapic_ipi_raw,
 420         .ipi_vectored           = xen_pv_lapic_ipi_vectored,
 421         .ipi_wait               = xen_pv_lapic_ipi_wait,
 422 #endif
 423         .ipi_alloc              = xen_pv_lapic_ipi_alloc,
 424         .ipi_free               = xen_pv_lapic_ipi_free,
 425         .set_lvt_mask           = xen_pv_lapic_set_lvt_mask,
 426         .set_lvt_mode           = xen_pv_lapic_set_lvt_mode,
 427         .set_lvt_polarity       = xen_pv_lapic_set_lvt_polarity,
 428         .set_lvt_triggermode    = xen_pv_lapic_set_lvt_triggermode,
 429 };
 430
 431 #ifdef SMP
 432 /*---------------------------- XEN PV IPI Handlers ---------------------------*/
 433 /*
 434  * These are C clones of the ASM functions found in apic_vector.
 435  */
 436 static int
 437 xen_ipi_bitmap_handler(void *arg)
 438 {
 439         struct trapframe *frame;
 440
 441         frame = arg;
 442         ipi_bitmap_handler(*frame);
 443         return (FILTER_HANDLED);
 444 }
 445
 446 static int
 447 xen_smp_rendezvous_action(void *arg)
 448 {
 449 #ifdef COUNT_IPIS
 450         (*ipi_rendezvous_counts[PCPU_GET(cpuid)])++;
 451 #endif /* COUNT_IPIS */
 452
 453         smp_rendezvous_action();
 454         return (FILTER_HANDLED);
 455 }
 456
 457 static int
 458 xen_invltlb(void *arg)
 459 {
 460
 461         invltlb_handler();
 462         return (FILTER_HANDLED);
 463 }
 464
 465 #ifdef __amd64__
 466 static int
 467 xen_invltlb_invpcid(void *arg)
 468 {
 469
 470         invltlb_invpcid_handler();
 471         return (FILTER_HANDLED);
 472 }
 473
 474 static int
 475 xen_invltlb_pcid(void *arg)
 476 {
 477
 478         invltlb_pcid_handler();
 479         return (FILTER_HANDLED);
 480 }
 481
 482 static int
 483 xen_invltlb_invpcid_pti(void *arg)
 484 {
 485
 486         invltlb_invpcid_pti_handler();
 487         return (FILTER_HANDLED);
 488 }
 489
 490 static int
 491 xen_invlpg_invpcid_handler(void *arg)
 492 {
 493
 494         invlpg_invpcid_handler();
 495         return (FILTER_HANDLED);
 496 }
 497
 498 static int
 499 xen_invlpg_pcid_handler(void *arg)
 500 {
 501
 502         invlpg_pcid_handler();
 503         return (FILTER_HANDLED);
 504 }
 505
 506 static int
 507 xen_invlrng_invpcid_handler(void *arg)
 508 {
 509
 510         invlrng_invpcid_handler();
 511         return (FILTER_HANDLED);
 512 }
 513
 514 static int
 515 xen_invlrng_pcid_handler(void *arg)
 516 {
 517
 518         invlrng_pcid_handler();
 519         return (FILTER_HANDLED);
 520 }
 521 #endif
 522
 523 static int
 524 xen_invlpg(void *arg)
 525 {
 526
 527         invlpg_handler();
 528         return (FILTER_HANDLED);
 529 }
 530
 531 static int
 532 xen_invlrng(void *arg)
 533 {
 534
 535         invlrng_handler();
 536         return (FILTER_HANDLED);
 537 }
 538
 539 static int
 540 xen_invlcache(void *arg)
 541 {
 542
 543         invlcache_handler();
 544         return (FILTER_HANDLED);
 545 }
 546
 547 static int
 548 xen_cpustop_handler(void *arg)
 549 {
 550
 551         cpustop_handler();
 552         return (FILTER_HANDLED);
 553 }
 554
 555 static int
 556 xen_cpususpend_handler(void *arg)
 557 {
 558
 559         cpususpend_handler();
 560         return (FILTER_HANDLED);
 561 }
 562
 563 /*----------------------------- XEN PV IPI setup -----------------------------*/
 564 /*
 565  * Those functions are provided outside of the Xen PV APIC implementation
 566  * so PVHVM guests can also use PV IPIs without having an actual Xen PV APIC,
 567  * because on PVHVM there's an emulated LAPIC provided by Xen.
 568  */
 569 static void
 570 xen_cpu_ipi_init(int cpu)
 571 {
 572         xen_intr_handle_t *ipi_handle;
 573         const struct xen_ipi_handler *ipi;
 574         int idx, rc;
 575
 576         ipi_handle = DPCPU_ID_GET(cpu, ipi_handle);
 577
 578         for (ipi = xen_ipis, idx = 0; idx < nitems(xen_ipis); ipi++, idx++) {
 579
 580                 if (ipi->filter == NULL) {
 581                         ipi_handle[idx] = NULL;
 582                         continue;
 583                 }
 584
 585                 rc = xen_intr_alloc_and_bind_ipi(cpu, ipi->filter,
 586                     INTR_TYPE_TTY, &ipi_handle[idx]);
 587                 if (rc != 0)
 588                         panic("Unable to allocate a XEN IPI port");
 589                 xen_intr_describe(ipi_handle[idx], "%s", ipi->description);
 590         }
 591 }
 592
 593 static void
 594 xen_setup_cpus(void)
 595 {
 596         int i;
 597
 598         if (!xen_vector_callback_enabled)
 599                 return;
 600
 601 #ifdef __amd64__
 602         if (pmap_pcid_enabled) {
 603                 if (pti)
 604                         xen_ipis[IPI_TO_IDX(IPI_INVLTLB)].filter =
 605                             invpcid_works ? xen_invltlb_invpcid_pti :
 606                             xen_invltlb_pcid;
 607                 else
 608                         xen_ipis[IPI_TO_IDX(IPI_INVLTLB)].filter =
 609                             invpcid_works ? xen_invltlb_invpcid :
 610                             xen_invltlb_pcid;
 611                 xen_ipis[IPI_TO_IDX(IPI_INVLPG)].filter = invpcid_works ?
 612                     xen_invlpg_invpcid_handler : xen_invlpg_pcid_handler;
 613                 xen_ipis[IPI_TO_IDX(IPI_INVLRNG)].filter = invpcid_works ?
 614                     xen_invlrng_invpcid_handler : xen_invlrng_pcid_handler;
 615         }
 616 #endif
 617         CPU_FOREACH(i)
 618                 xen_cpu_ipi_init(i);
 619
 620         /* Set the xen pv ipi ops to replace the native ones */
 621         if (xen_hvm_domain())
 622                 apic_ops.ipi_vectored = xen_pv_lapic_ipi_vectored;
 623 }
 624
 625 /* Switch to using PV IPIs as soon as the vcpu_id is set. */
 626 SYSINIT(xen_setup_cpus, SI_SUB_SMP, SI_ORDER_SECOND, xen_setup_cpus, NULL);
 627 #endif /* SMP */