2 * Copyright (c) 2012 Konstantin Belousov <kib@FreeBSD.org>
3 * Copyright (c) 2016, 2017, 2019 The FreeBSD Foundation
6 * Portions of this software were developed by Konstantin Belousov
7 * under sponsorship from the FreeBSD Foundation.
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
34 #include <sys/param.h>
35 #include "namespace.h"
36 #include <sys/capsicum.h>
38 #include <sys/fcntl.h>
45 #include "un-namespace.h"
46 #include <machine/atomic.h>
47 #include <machine/cpufunc.h>
48 #include <machine/specialreg.h>
49 #include <dev/acpica/acpi_hpet.h>
51 #include <dev/hyperv/hyperv.h>
53 #include <x86/ifunc.h>
54 #include "libc_private.h"
57 rdtsc_low(const struct vdso_timehands *th)
61 __asm __volatile("rdtsc; shrd %%cl, %%edx, %0"
62 : "=a" (rv) : "c" (th->th_x86_shift) : "edx");
67 rdtscp_low(const struct vdso_timehands *th)
71 __asm __volatile("rdtscp; movl %%edi,%%ecx; shrd %%cl, %%edx, %0"
72 : "=a" (rv) : "D" (th->th_x86_shift) : "ecx", "edx");
77 rdtsc_low_mb_lfence(const struct vdso_timehands *th)
80 return (rdtsc_low(th));
84 rdtsc_low_mb_mfence(const struct vdso_timehands *th)
87 return (rdtsc_low(th));
91 rdtsc_low_mb_none(const struct vdso_timehands *th)
93 return (rdtsc_low(th));
97 rdtsc32_mb_lfence(void)
104 rdtsc32_mb_mfence(void)
111 rdtsc32_mb_none(void)
122 struct tsc_selector_tag {
123 u_int (*ts_rdtsc32)(void);
124 u_int (*ts_rdtsc_low)(const struct vdso_timehands *);
127 static const struct tsc_selector_tag tsc_selector[] = {
128 [0] = { /* Intel or AMD Zen+, LFENCE */
129 .ts_rdtsc32 = rdtsc32_mb_lfence,
130 .ts_rdtsc_low = rdtsc_low_mb_lfence,
132 [1] = { /* AMD, MFENCE */
133 .ts_rdtsc32 = rdtsc32_mb_mfence,
134 .ts_rdtsc_low = rdtsc_low_mb_mfence,
136 [2] = { /* No SSE2 */
137 .ts_rdtsc32 = rdtsc32_mb_none,
138 .ts_rdtsc_low = rdtsc_low_mb_none,
141 .ts_rdtsc32 = rdtscp32_,
142 .ts_rdtsc_low = rdtscp_low,
147 tsc_selector_idx(u_int cpu_feature)
149 u_int amd_feature, cpu_exthigh, cpu_id, p[4], v[3];
150 static const char amd_id[] = "AuthenticAMD";
151 static const char hygon_id[] = "HygonGenuine";
154 if (cpu_feature == 0)
155 return (2); /* should not happen due to RDTSC */
161 amd_cpu = memcmp(v, amd_id, sizeof(amd_id) - 1) == 0 ||
162 memcmp(v, hygon_id, sizeof(hygon_id) - 1) == 0;
167 if (amd_cpu && CPUID_TO_FAMILY(cpu_id) >= 0x17)
170 if (cpu_feature != 0) {
171 do_cpuid(0x80000000, p);
176 if (cpu_exthigh >= 0x80000001) {
177 do_cpuid(0x80000001, p);
183 if ((amd_feature & AMDID_RDTSCP) != 0)
185 if ((cpu_feature & CPUID_SSE2) == 0)
187 return (amd_cpu ? 1 : 0);
190 DEFINE_UIFUNC(static, u_int, __vdso_gettc_rdtsc_low,
191 (const struct vdso_timehands *th))
193 return (tsc_selector[tsc_selector_idx(cpu_feature)].ts_rdtsc_low);
196 DEFINE_UIFUNC(static, u_int, __vdso_gettc_rdtsc32, (void))
198 return (tsc_selector[tsc_selector_idx(cpu_feature)].ts_rdtsc32);
201 #define HPET_DEV_MAP_MAX 10
202 static volatile char *hpet_dev_map[HPET_DEV_MAP_MAX];
205 __vdso_init_hpet(uint32_t u)
207 static const char devprefix[] = "/dev/hpet";
208 char devname[64], *c, *c1, t;
209 volatile char *new_map, *old_map;
214 c1 = c = stpcpy(devname, devprefix);
217 *c++ = u1 % 10 + '0';
221 for (c--; c1 != c; c1++, c--) {
227 old_map = hpet_dev_map[u];
232 * Explicitely check for the capability mode to avoid
233 * triggering trap_enocap on the device open by absolute path.
235 if ((cap_getmode(&mode) == 0 && mode != 0) ||
236 (fd = _open(devname, O_RDONLY)) == -1) {
237 /* Prevent the caller from re-entering. */
238 atomic_cmpset_rel_ptr((volatile uintptr_t *)&hpet_dev_map[u],
239 (uintptr_t)old_map, (uintptr_t)MAP_FAILED);
243 new_map = mmap(NULL, PAGE_SIZE, PROT_READ, MAP_SHARED, fd, 0);
245 if (atomic_cmpset_rel_ptr((volatile uintptr_t *)&hpet_dev_map[u],
246 (uintptr_t)old_map, (uintptr_t)new_map) == 0 &&
247 new_map != MAP_FAILED)
248 munmap((void *)new_map, PAGE_SIZE);
253 #define HYPERV_REFTSC_DEVPATH "/dev/" HYPERV_REFTSC_DEVNAME
257 * We use 'NULL' for this variable to indicate that initialization
258 * is required. And if this variable is 'MAP_FAILED', then Hyper-V
259 * reference TSC can not be used, e.g. in misconfigured jail.
261 static struct hyperv_reftsc *hyperv_ref_tsc;
264 __vdso_init_hyperv_tsc(void)
269 if (cap_getmode(&mode) == 0 && mode != 0)
272 fd = _open(HYPERV_REFTSC_DEVPATH, O_RDONLY);
275 hyperv_ref_tsc = mmap(NULL, sizeof(*hyperv_ref_tsc), PROT_READ,
281 /* Prevent the caller from re-entering. */
282 hyperv_ref_tsc = MAP_FAILED;
286 __vdso_hyperv_tsc(struct hyperv_reftsc *tsc_ref, u_int *tc)
288 uint64_t disc, ret, tsc, scale;
292 while ((seq = atomic_load_acq_int(&tsc_ref->tsc_seq)) != 0) {
293 scale = tsc_ref->tsc_scale;
294 ofs = tsc_ref->tsc_ofs;
296 mfence(); /* XXXKIB */
299 /* ret = ((tsc * scale) >> 64) + ofs */
300 __asm__ __volatile__ ("mulq %3" :
301 "=d" (ret), "=a" (disc) :
302 "a" (tsc), "r" (scale));
305 atomic_thread_fence_acq();
306 if (tsc_ref->tsc_seq == seq) {
311 /* Sequence changed; re-sync. */
316 #endif /* WANT_HYPERV */
318 #pragma weak __vdso_gettc
320 __vdso_gettc(const struct vdso_timehands *th, u_int *tc)
325 switch (th->th_algo) {
326 case VDSO_TH_ALGO_X86_TSC:
327 *tc = th->th_x86_shift > 0 ? __vdso_gettc_rdtsc_low(th) :
328 __vdso_gettc_rdtsc32();
330 case VDSO_TH_ALGO_X86_HPET:
331 idx = th->th_x86_hpet_idx;
332 if (idx >= HPET_DEV_MAP_MAX)
334 map = (volatile char *)atomic_load_acq_ptr(
335 (volatile uintptr_t *)&hpet_dev_map[idx]);
337 __vdso_init_hpet(idx);
338 map = (volatile char *)atomic_load_acq_ptr(
339 (volatile uintptr_t *)&hpet_dev_map[idx]);
341 if (map == MAP_FAILED)
343 *tc = *(volatile uint32_t *)(map + HPET_MAIN_COUNTER);
346 case VDSO_TH_ALGO_X86_HVTSC:
347 if (hyperv_ref_tsc == NULL)
348 __vdso_init_hyperv_tsc();
349 if (hyperv_ref_tsc == MAP_FAILED)
351 return (__vdso_hyperv_tsc(hyperv_ref_tsc, tc));
358 #pragma weak __vdso_gettimekeep
360 __vdso_gettimekeep(struct vdso_timekeep **tk)
363 return (_elf_aux_info(AT_TIMEKEEP, tk, sizeof(*tk)));