2 * Copyright (c) 2016-2017 Microsoft Corp.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice unmodified, this list of conditions, and the following
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 #include <sys/cdefs.h>
28 __FBSDID("$FreeBSD$");
30 #include <sys/param.h>
32 #include <sys/fcntl.h>
33 #include <sys/kernel.h>
34 #include <sys/systm.h>
35 #include <sys/timetc.h>
38 #include <machine/cpufunc.h>
39 #include <machine/cputypes.h>
40 #include <machine/md_var.h>
41 #include <machine/specialreg.h>
45 #include <dev/hyperv/include/hyperv.h>
46 #include <dev/hyperv/include/hyperv_busdma.h>
47 #include <dev/hyperv/vmbus/hyperv_machdep.h>
48 #include <dev/hyperv/vmbus/hyperv_reg.h>
49 #include <dev/hyperv/vmbus/hyperv_var.h>
51 struct hyperv_reftsc_ctx {
52 struct hyperv_reftsc *tsc_ref;
53 struct hyperv_dma tsc_ref_dma;
56 static uint32_t hyperv_tsc_vdso_timehands(
57 struct vdso_timehands *,
58 struct timecounter *);
60 static d_open_t hyperv_tsc_open;
61 static d_mmap_t hyperv_tsc_mmap;
63 static struct timecounter hyperv_tsc_timecounter = {
64 .tc_get_timecount = NULL, /* based on CPU vendor. */
65 .tc_counter_mask = 0xffffffff,
66 .tc_frequency = HYPERV_TIMER_FREQ,
67 .tc_name = "Hyper-V-TSC",
69 .tc_fill_vdso_timehands = hyperv_tsc_vdso_timehands,
72 static struct cdevsw hyperv_tsc_cdevsw = {
73 .d_version = D_VERSION,
74 .d_open = hyperv_tsc_open,
75 .d_mmap = hyperv_tsc_mmap,
76 .d_name = HYPERV_REFTSC_DEVNAME
79 static struct hyperv_reftsc_ctx hyperv_ref_tsc;
82 hypercall_md(volatile void *hc_addr, uint64_t in_val,
83 uint64_t in_paddr, uint64_t out_paddr)
87 __asm__ __volatile__ ("mov %0, %%r8" : : "r" (out_paddr): "r8");
88 __asm__ __volatile__ ("call *%3" : "=a" (status) :
89 "c" (in_val), "d" (in_paddr), "m" (hc_addr));
94 hyperv_tsc_open(struct cdev *dev __unused, int oflags, int devtype __unused,
95 struct thread *td __unused)
104 hyperv_tsc_mmap(struct cdev *dev __unused, vm_ooffset_t offset,
105 vm_paddr_t *paddr, int nprot __unused, vm_memattr_t *memattr __unused)
108 KASSERT(hyperv_ref_tsc.tsc_ref != NULL, ("reftsc has not been setup"));
112 * 'nprot' does not contain information interested to us;
113 * WR-open is blocked by d_open.
119 *paddr = hyperv_ref_tsc.tsc_ref_dma.hv_paddr;
124 hyperv_tsc_vdso_timehands(struct vdso_timehands *vdso_th,
125 struct timecounter *tc __unused)
128 vdso_th->th_algo = VDSO_TH_ALGO_X86_HVTSC;
129 vdso_th->th_x86_shift = 0;
130 vdso_th->th_x86_hpet_idx = 0;
131 vdso_th->th_x86_pvc_last_systime = 0;
132 vdso_th->th_x86_pvc_stable_mask = 0;
133 bzero(vdso_th->th_res, sizeof(vdso_th->th_res));
137 #define HYPERV_TSC_TIMECOUNT(fence) \
139 hyperv_tc64_tsc_##fence(void) \
141 struct hyperv_reftsc *tsc_ref = hyperv_ref_tsc.tsc_ref; \
144 while ((seq = atomic_load_acq_int(&tsc_ref->tsc_seq)) != 0) { \
145 uint64_t disc, ret, tsc; \
146 uint64_t scale = tsc_ref->tsc_scale; \
147 int64_t ofs = tsc_ref->tsc_ofs; \
152 /* ret = ((tsc * scale) >> 64) + ofs */ \
153 __asm__ __volatile__ ("mulq %3" : \
154 "=d" (ret), "=a" (disc) : \
155 "a" (tsc), "r" (scale)); \
158 atomic_thread_fence_acq(); \
159 if (tsc_ref->tsc_seq == seq) \
162 /* Sequence changed; re-sync. */ \
164 /* Fallback to the generic timecounter, i.e. rdmsr. */ \
165 return (rdmsr(MSR_HV_TIME_REF_COUNT)); \
169 hyperv_tsc_timecount_##fence(struct timecounter *tc __unused) \
172 return (hyperv_tc64_tsc_##fence()); \
176 HYPERV_TSC_TIMECOUNT(lfence);
177 HYPERV_TSC_TIMECOUNT(mfence);
180 hyperv_tsc_tcinit(void *dummy __unused)
182 hyperv_tc64_t tc64 = NULL;
185 if ((hyperv_features &
186 (CPUID_HV_MSR_TIME_REFCNT | CPUID_HV_MSR_REFERENCE_TSC)) !=
187 (CPUID_HV_MSR_TIME_REFCNT | CPUID_HV_MSR_REFERENCE_TSC) ||
188 (cpu_feature & CPUID_SSE2) == 0) /* SSE2 for mfence/lfence */
191 switch (cpu_vendor_id) {
193 case CPU_VENDOR_HYGON:
194 hyperv_tsc_timecounter.tc_get_timecount =
195 hyperv_tsc_timecount_mfence;
196 tc64 = hyperv_tc64_tsc_mfence;
199 case CPU_VENDOR_INTEL:
200 hyperv_tsc_timecounter.tc_get_timecount =
201 hyperv_tsc_timecount_lfence;
202 tc64 = hyperv_tc64_tsc_lfence;
206 /* Unsupport CPU vendors. */
210 hyperv_ref_tsc.tsc_ref = hyperv_dmamem_alloc(NULL, PAGE_SIZE, 0,
211 sizeof(struct hyperv_reftsc), &hyperv_ref_tsc.tsc_ref_dma,
212 BUS_DMA_WAITOK | BUS_DMA_ZERO);
213 if (hyperv_ref_tsc.tsc_ref == NULL) {
214 printf("hyperv: reftsc page allocation failed\n");
218 orig = rdmsr(MSR_HV_REFERENCE_TSC);
219 val = MSR_HV_REFTSC_ENABLE | (orig & MSR_HV_REFTSC_RSVD_MASK) |
220 ((hyperv_ref_tsc.tsc_ref_dma.hv_paddr >> PAGE_SHIFT) <<
221 MSR_HV_REFTSC_PGSHIFT);
222 wrmsr(MSR_HV_REFERENCE_TSC, val);
224 /* Register "enlightened" timecounter. */
225 tc_init(&hyperv_tsc_timecounter);
227 /* Install 64 bits timecounter method for other modules to use. */
228 KASSERT(tc64 != NULL, ("tc64 is not set"));
231 /* Add device for mmap(2). */
232 make_dev(&hyperv_tsc_cdevsw, 0, UID_ROOT, GID_WHEEL, 0444,
233 HYPERV_REFTSC_DEVNAME);
235 SYSINIT(hyperv_tsc_init, SI_SUB_DRIVERS, SI_ORDER_FIRST, hyperv_tsc_tcinit,