sys/dev/hyperv/vmbus/amd64/hyperv_machdep.c

   1 /*-
   2  * Copyright (c) 2016-2017 Microsoft Corp.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice unmodified, this list of conditions, and the following
  10  *    disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  */
  26
  27 #include <sys/cdefs.h>
  28 __FBSDID("$FreeBSD$");
  29
  30 #include <sys/param.h>
  31 #include <sys/conf.h>
  32 #include <sys/fcntl.h>
  33 #include <sys/kernel.h>
  34 #include <sys/systm.h>
  35 #include <sys/timetc.h>
  36 #include <sys/vdso.h>
  37
  38 #include <machine/cpufunc.h>
  39 #include <machine/cputypes.h>
  40 #include <machine/md_var.h>
  41 #include <machine/specialreg.h>
  42
  43 #include <vm/vm.h>
  44
  45 #include <dev/hyperv/include/hyperv.h>
  46 #include <dev/hyperv/include/hyperv_busdma.h>
  47 #include <dev/hyperv/vmbus/hyperv_machdep.h>
  48 #include <dev/hyperv/vmbus/hyperv_reg.h>
  49 #include <dev/hyperv/vmbus/hyperv_var.h>
  50
  51 struct hyperv_reftsc_ctx {
  52         struct hyperv_reftsc    *tsc_ref;
  53         struct hyperv_dma       tsc_ref_dma;
  54 };
  55
  56 static uint32_t                 hyperv_tsc_vdso_timehands(
  57                                     struct vdso_timehands *,
  58                                     struct timecounter *);
  59
  60 static d_open_t                 hyperv_tsc_open;
  61 static d_mmap_t                 hyperv_tsc_mmap;
  62
  63 static struct timecounter       hyperv_tsc_timecounter = {
  64         .tc_get_timecount       = NULL, /* based on CPU vendor. */
  65         .tc_counter_mask        = 0xffffffff,
  66         .tc_frequency           = HYPERV_TIMER_FREQ,
  67         .tc_name                = "Hyper-V-TSC",
  68         .tc_quality             = 3000,
  69         .tc_fill_vdso_timehands = hyperv_tsc_vdso_timehands,
  70 };
  71
  72 static struct cdevsw            hyperv_tsc_cdevsw = {
  73         .d_version              = D_VERSION,
  74         .d_open                 = hyperv_tsc_open,
  75         .d_mmap                 = hyperv_tsc_mmap,
  76         .d_name                 = HYPERV_REFTSC_DEVNAME
  77 };
  78
  79 static struct hyperv_reftsc_ctx hyperv_ref_tsc;
  80
  81 uint64_t
  82 hypercall_md(volatile void *hc_addr, uint64_t in_val,
  83     uint64_t in_paddr, uint64_t out_paddr)
  84 {
  85         uint64_t status;
  86
  87         __asm__ __volatile__ ("mov %0, %%r8" : : "r" (out_paddr): "r8");
  88         __asm__ __volatile__ ("call *%3" : "=a" (status) :
  89             "c" (in_val), "d" (in_paddr), "m" (hc_addr));
  90         return (status);
  91 }
  92
  93 static int
  94 hyperv_tsc_open(struct cdev *dev __unused, int oflags, int devtype __unused,
  95     struct thread *td __unused)
  96 {
  97
  98         if (oflags & FWRITE)
  99                 return (EPERM);
 100         return (0);
 101 }
 102
 103 static int
 104 hyperv_tsc_mmap(struct cdev *dev __unused, vm_ooffset_t offset,
 105     vm_paddr_t *paddr, int nprot __unused, vm_memattr_t *memattr __unused)
 106 {
 107
 108         KASSERT(hyperv_ref_tsc.tsc_ref != NULL, ("reftsc has not been setup"));
 109
 110         /*
 111          * NOTE:
 112          * 'nprot' does not contain information interested to us;
 113          * WR-open is blocked by d_open.
 114          */
 115
 116         if (offset != 0)
 117                 return (EOPNOTSUPP);
 118
 119         *paddr = hyperv_ref_tsc.tsc_ref_dma.hv_paddr;
 120         return (0);
 121 }
 122
 123 static uint32_t
 124 hyperv_tsc_vdso_timehands(struct vdso_timehands *vdso_th,
 125     struct timecounter *tc __unused)
 126 {
 127
 128         vdso_th->th_algo = VDSO_TH_ALGO_X86_HVTSC;
 129         vdso_th->th_x86_shift = 0;
 130         vdso_th->th_x86_hpet_idx = 0;
 131         vdso_th->th_x86_pvc_last_systime = 0;
 132         vdso_th->th_x86_pvc_stable_mask = 0;
 133         bzero(vdso_th->th_res, sizeof(vdso_th->th_res));
 134         return (1);
 135 }
 136
 137 #define HYPERV_TSC_TIMECOUNT(fence)                                     \
 138 static uint64_t                                                         \
 139 hyperv_tc64_tsc_##fence(void)                                           \
 140 {                                                                       \
 141         struct hyperv_reftsc *tsc_ref = hyperv_ref_tsc.tsc_ref;         \
 142         uint32_t seq;                                                   \
 143                                                                         \
 144         while ((seq = atomic_load_acq_int(&tsc_ref->tsc_seq)) != 0) {   \
 145                 uint64_t disc, ret, tsc;                                \
 146                 uint64_t scale = tsc_ref->tsc_scale;                    \
 147                 int64_t ofs = tsc_ref->tsc_ofs;                         \
 148                                                                         \
 149                 fence();                                                \
 150                 tsc = rdtsc();                                          \
 151                                                                         \
 152                 /* ret = ((tsc * scale) >> 64) + ofs */                 \
 153                 __asm__ __volatile__ ("mulq %3" :                       \
 154                     "=d" (ret), "=a" (disc) :                           \
 155                     "a" (tsc), "r" (scale));                            \
 156                 ret += ofs;                                             \
 157                                                                         \
 158                 atomic_thread_fence_acq();                              \
 159                 if (tsc_ref->tsc_seq == seq)                            \
 160                         return (ret);                                   \
 161                                                                         \
 162                 /* Sequence changed; re-sync. */                        \
 163         }                                                               \
 164         /* Fallback to the generic timecounter, i.e. rdmsr. */          \
 165         return (rdmsr(MSR_HV_TIME_REF_COUNT));                          \
 166 }                                                                       \
 167                                                                         \
 168 static u_int                                                            \
 169 hyperv_tsc_timecount_##fence(struct timecounter *tc __unused)           \
 170 {                                                                       \
 171                                                                         \
 172         return (hyperv_tc64_tsc_##fence());                             \
 173 }                                                                       \
 174 struct __hack
 175
 176 HYPERV_TSC_TIMECOUNT(lfence);
 177 HYPERV_TSC_TIMECOUNT(mfence);
 178
 179 static void
 180 hyperv_tsc_tcinit(void *dummy __unused)
 181 {
 182         hyperv_tc64_t tc64 = NULL;
 183         uint64_t val, orig;
 184
 185         if ((hyperv_features &
 186              (CPUID_HV_MSR_TIME_REFCNT | CPUID_HV_MSR_REFERENCE_TSC)) !=
 187             (CPUID_HV_MSR_TIME_REFCNT | CPUID_HV_MSR_REFERENCE_TSC) ||
 188             (cpu_feature & CPUID_SSE2) == 0)    /* SSE2 for mfence/lfence */
 189                 return;
 190
 191         switch (cpu_vendor_id) {
 192         case CPU_VENDOR_AMD:
 193         case CPU_VENDOR_HYGON:
 194                 hyperv_tsc_timecounter.tc_get_timecount =
 195                     hyperv_tsc_timecount_mfence;
 196                 tc64 = hyperv_tc64_tsc_mfence;
 197                 break;
 198
 199         case CPU_VENDOR_INTEL:
 200                 hyperv_tsc_timecounter.tc_get_timecount =
 201                     hyperv_tsc_timecount_lfence;
 202                 tc64 = hyperv_tc64_tsc_lfence;
 203                 break;
 204
 205         default:
 206                 /* Unsupport CPU vendors. */
 207                 return;
 208         }
 209
 210         hyperv_ref_tsc.tsc_ref = hyperv_dmamem_alloc(NULL, PAGE_SIZE, 0,
 211             sizeof(struct hyperv_reftsc), &hyperv_ref_tsc.tsc_ref_dma,
 212             BUS_DMA_WAITOK | BUS_DMA_ZERO);
 213         if (hyperv_ref_tsc.tsc_ref == NULL) {
 214                 printf("hyperv: reftsc page allocation failed\n");
 215                 return;
 216         }
 217
 218         orig = rdmsr(MSR_HV_REFERENCE_TSC);
 219         val = MSR_HV_REFTSC_ENABLE | (orig & MSR_HV_REFTSC_RSVD_MASK) |
 220             ((hyperv_ref_tsc.tsc_ref_dma.hv_paddr >> PAGE_SHIFT) <<
 221              MSR_HV_REFTSC_PGSHIFT);
 222         wrmsr(MSR_HV_REFERENCE_TSC, val);
 223
 224         /* Register "enlightened" timecounter. */
 225         tc_init(&hyperv_tsc_timecounter);
 226
 227         /* Install 64 bits timecounter method for other modules to use. */
 228         KASSERT(tc64 != NULL, ("tc64 is not set"));
 229         hyperv_tc64 = tc64;
 230
 231         /* Add device for mmap(2). */
 232         make_dev(&hyperv_tsc_cdevsw, 0, UID_ROOT, GID_WHEEL, 0444,
 233             HYPERV_REFTSC_DEVNAME);
 234 }
 235 SYSINIT(hyperv_tsc_init, SI_SUB_DRIVERS, SI_ORDER_FIRST, hyperv_tsc_tcinit,
 236     NULL);