2 * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org>
3 * Copyright (c) 2016, 2017, 2019 The FreeBSD Foundation
6 * Portions of this software were developed by Konstantin Belousov
7 * under sponsorship from the FreeBSD Foundation.
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
34 #include <sys/param.h>
35 #include "namespace.h"
36 #include <sys/capsicum.h>
38 #include <sys/fcntl.h>
45 #include "un-namespace.h"
46 #include <machine/atomic.h>
47 #include <machine/cpufunc.h>
48 #include <machine/specialreg.h>
49 #include <dev/acpica/acpi_hpet.h>
51 #include <dev/hyperv/hyperv.h>
53 #include <x86/ifunc.h>
54 #include "libc_private.h"
75 DEFINE_UIFUNC(static, void, rdtsc_mb, (void))
78 /* Not a typo, string matches our do_cpuid() registers use. */
79 static const char intel_id[] = "GenuntelineI";
81 if ((cpu_feature & CPUID_SSE2) == 0)
82 return (rdtsc_mb_none);
84 return (memcmp(p + 1, intel_id, sizeof(intel_id) - 1) == 0 ?
85 rdtsc_mb_lfence : rdtsc_mb_mfence);
89 __vdso_gettc_rdtsc_low(const struct vdso_timehands *th)
94 __asm __volatile("rdtsc; shrd %%cl, %%edx, %0"
95 : "=a" (rv) : "c" (th->th_x86_shift) : "edx");
107 #define HPET_DEV_MAP_MAX 10
108 static volatile char *hpet_dev_map[HPET_DEV_MAP_MAX];
111 __vdso_init_hpet(uint32_t u)
113 static const char devprefix[] = "/dev/hpet";
114 char devname[64], *c, *c1, t;
115 volatile char *new_map, *old_map;
120 c1 = c = stpcpy(devname, devprefix);
123 *c++ = u1 % 10 + '0';
127 for (c--; c1 != c; c1++, c--) {
133 old_map = hpet_dev_map[u];
138 * Explicitely check for the capability mode to avoid
139 * triggering trap_enocap on the device open by absolute path.
141 if ((cap_getmode(&mode) == 0 && mode != 0) ||
142 (fd = _open(devname, O_RDONLY)) == -1) {
143 /* Prevent the caller from re-entering. */
144 atomic_cmpset_rel_ptr((volatile uintptr_t *)&hpet_dev_map[u],
145 (uintptr_t)old_map, (uintptr_t)MAP_FAILED);
149 new_map = mmap(NULL, PAGE_SIZE, PROT_READ, MAP_SHARED, fd, 0);
151 if (atomic_cmpset_rel_ptr((volatile uintptr_t *)&hpet_dev_map[u],
152 (uintptr_t)old_map, (uintptr_t)new_map) == 0 &&
153 new_map != MAP_FAILED)
154 munmap((void *)new_map, PAGE_SIZE);
159 #define HYPERV_REFTSC_DEVPATH "/dev/" HYPERV_REFTSC_DEVNAME
163 * We use 'NULL' for this variable to indicate that initialization
164 * is required. And if this variable is 'MAP_FAILED', then Hyper-V
165 * reference TSC can not be used, e.g. in misconfigured jail.
167 static struct hyperv_reftsc *hyperv_ref_tsc;
170 __vdso_init_hyperv_tsc(void)
175 if (cap_getmode(&mode) == 0 && mode != 0)
178 fd = _open(HYPERV_REFTSC_DEVPATH, O_RDONLY);
181 hyperv_ref_tsc = mmap(NULL, sizeof(*hyperv_ref_tsc), PROT_READ,
187 /* Prevent the caller from re-entering. */
188 hyperv_ref_tsc = MAP_FAILED;
192 __vdso_hyperv_tsc(struct hyperv_reftsc *tsc_ref, u_int *tc)
194 uint64_t disc, ret, tsc, scale;
198 while ((seq = atomic_load_acq_int(&tsc_ref->tsc_seq)) != 0) {
199 scale = tsc_ref->tsc_scale;
200 ofs = tsc_ref->tsc_ofs;
205 /* ret = ((tsc * scale) >> 64) + ofs */
206 __asm__ __volatile__ ("mulq %3" :
207 "=d" (ret), "=a" (disc) :
208 "a" (tsc), "r" (scale));
211 atomic_thread_fence_acq();
212 if (tsc_ref->tsc_seq == seq) {
217 /* Sequence changed; re-sync. */
222 #endif /* WANT_HYPERV */
224 #pragma weak __vdso_gettc
226 __vdso_gettc(const struct vdso_timehands *th, u_int *tc)
231 switch (th->th_algo) {
232 case VDSO_TH_ALGO_X86_TSC:
233 *tc = th->th_x86_shift > 0 ? __vdso_gettc_rdtsc_low(th) :
236 case VDSO_TH_ALGO_X86_HPET:
237 idx = th->th_x86_hpet_idx;
238 if (idx >= HPET_DEV_MAP_MAX)
240 map = (volatile char *)atomic_load_acq_ptr(
241 (volatile uintptr_t *)&hpet_dev_map[idx]);
243 __vdso_init_hpet(idx);
244 map = (volatile char *)atomic_load_acq_ptr(
245 (volatile uintptr_t *)&hpet_dev_map[idx]);
247 if (map == MAP_FAILED)
249 *tc = *(volatile uint32_t *)(map + HPET_MAIN_COUNTER);
252 case VDSO_TH_ALGO_X86_HVTSC:
253 if (hyperv_ref_tsc == NULL)
254 __vdso_init_hyperv_tsc();
255 if (hyperv_ref_tsc == MAP_FAILED)
257 return (__vdso_hyperv_tsc(hyperv_ref_tsc, tc));
264 #pragma weak __vdso_gettimekeep
266 __vdso_gettimekeep(struct vdso_timekeep **tk)
269 return (_elf_aux_info(AT_TIMEKEEP, tk, sizeof(*tk)));