From 216abc875738d0892729960b4c1ccfc1d8f3a1b8 Mon Sep 17 00:00:00 2001 From: kib Date: Sat, 22 Sep 2012 12:32:00 +0000 Subject: [PATCH] MFC r237433: Implement mechanism to export some kernel timekeeping data to usermode, using shared page. MFC r237474: Stop updating the struct vdso_timehands from even handler executed in the scheduled task from tc_windup(). Do it directly from tc_windup in interrupt context. git-svn-id: svn://svn.freebsd.org/base/stable/9@240815 ccf9f872-aa2e-dd11-9fc8-001c23d0bc1f --- sys/amd64/include/vdso.h | 6 ++ sys/arm/include/vdso.h | 34 ++++++++ sys/conf/files.arm | 2 + sys/conf/files.ia64 | 1 + sys/conf/files.mips | 2 + sys/conf/files.powerpc | 1 + sys/conf/files.sparc64 | 1 + sys/i386/include/vdso.h | 6 ++ sys/ia64/include/vdso.h | 41 ++++++++++ sys/kern/imgact_elf.c | 4 + sys/kern/kern_exec.c | 144 ++++++++++++++++++++++++++-------- sys/kern/kern_tc.c | 64 +++++++++++++++ sys/kern/subr_dummy_vdso_tc.c | 49 ++++++++++++ sys/mips/include/vdso.h | 41 ++++++++++ sys/pc98/include/vdso.h | 6 ++ sys/powerpc/include/vdso.h | 41 ++++++++++ sys/sparc64/include/vdso.h | 34 ++++++++ sys/sys/sysent.h | 6 +- sys/sys/vdso.h | 122 ++++++++++++++++++++++++++++ sys/x86/include/vdso.h | 42 ++++++++++ sys/x86/x86/tsc.c | 22 ++++++ 21 files changed, 636 insertions(+), 33 deletions(-) create mode 100644 sys/amd64/include/vdso.h create mode 100644 sys/arm/include/vdso.h create mode 100644 sys/i386/include/vdso.h create mode 100644 sys/ia64/include/vdso.h create mode 100644 sys/kern/subr_dummy_vdso_tc.c create mode 100644 sys/mips/include/vdso.h create mode 100644 sys/pc98/include/vdso.h create mode 100644 sys/powerpc/include/vdso.h create mode 100644 sys/sparc64/include/vdso.h create mode 100644 sys/sys/vdso.h create mode 100644 sys/x86/include/vdso.h diff --git a/sys/amd64/include/vdso.h b/sys/amd64/include/vdso.h new file mode 100644 index 000000000..b81c455a9 --- /dev/null +++ b/sys/amd64/include/vdso.h @@ -0,0 +1,6 @@ +/*- + * This file is in the public domain. + */ +/* $FreeBSD$ */ + +#include diff --git a/sys/arm/include/vdso.h b/sys/arm/include/vdso.h new file mode 100644 index 000000000..d6aa162a3 --- /dev/null +++ b/sys/arm/include/vdso.h @@ -0,0 +1,34 @@ +/*- + * Copyright 2012 Konstantin Belousov . + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _ARM_VDSO_H +#define _ARM_VDSO_H + +#define VDSO_TIMEHANDS_MD \ + uint32_t th_res[8]; + +#endif diff --git a/sys/conf/files.arm b/sys/conf/files.arm index 05fd8237b..202d369e7 100644 --- a/sys/conf/files.arm +++ b/sys/conf/files.arm @@ -76,3 +76,5 @@ libkern/umoddi3.c standard #XXX: We can't use these versions, as strcmp.c is included conf/files #libkern/arm/strcmp.S standard #libkern/arm/strncmp.S standard +# +kern/subr_dummy_vdso_tc.c standard diff --git a/sys/conf/files.ia64 b/sys/conf/files.ia64 index e34123065..0ec58de53 100644 --- a/sys/conf/files.ia64 +++ b/sys/conf/files.ia64 @@ -137,3 +137,4 @@ libkern/ia64/bswap16.S standard libkern/ia64/bswap32.S standard libkern/memmove.c standard libkern/memset.c standard +kern/subr_dummy_vdso_tc.c standard diff --git a/sys/conf/files.mips b/sys/conf/files.mips index 6438c492d..3b1335568 100644 --- a/sys/conf/files.mips +++ b/sys/conf/files.mips @@ -89,6 +89,8 @@ libkern/umoddi3.c optional isa_mips32 #libkern/mips/strcmp.S standard #libkern/mips/strncmp.S standard +kern/subr_dummy_vdso_tc.c standard + kern/kern_clocksource.c standard kern/link_elf_obj.c standard diff --git a/sys/conf/files.powerpc b/sys/conf/files.powerpc index af9969fb0..f4d6c3068 100644 --- a/sys/conf/files.powerpc +++ b/sys/conf/files.powerpc @@ -58,6 +58,7 @@ dev/tsec/if_tsec.c optional tsec dev/tsec/if_tsec_fdt.c optional tsec fdt dev/uart/uart_cpu_powerpc.c optional uart aim kern/kern_clocksource.c standard +kern/subr_dummy_vdso_tc.c standard kern/syscalls.c optional ktr libkern/ashldi3.c optional powerpc libkern/ashrdi3.c optional powerpc diff --git a/sys/conf/files.sparc64 b/sys/conf/files.sparc64 index 1ff61b114..27719b8b3 100644 --- a/sys/conf/files.sparc64 +++ b/sys/conf/files.sparc64 @@ -60,6 +60,7 @@ dev/syscons/scvtb.c optional sc dev/uart/uart_cpu_sparc64.c optional uart dev/uart/uart_kbd_sun.c optional uart sc kern/kern_clocksource.c standard +kern/subr_dummy_vdso_tc.c standard kern/syscalls.c optional ktr libkern/ffs.c standard libkern/ffsl.c standard diff --git a/sys/i386/include/vdso.h b/sys/i386/include/vdso.h new file mode 100644 index 000000000..b81c455a9 --- /dev/null +++ b/sys/i386/include/vdso.h @@ -0,0 +1,6 @@ +/*- + * This file is in the public domain. + */ +/* $FreeBSD$ */ + +#include diff --git a/sys/ia64/include/vdso.h b/sys/ia64/include/vdso.h new file mode 100644 index 000000000..4571daa29 --- /dev/null +++ b/sys/ia64/include/vdso.h @@ -0,0 +1,41 @@ +/*- + * Copyright 2012 Konstantin Belousov . + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _IA64_VDSO_H +#define _IA64_VDSO_H + +#define VDSO_TIMEHANDS_MD \ + uint32_t th_res[8]; + +#ifdef _KERNEL +#ifdef COMPAT_FREEBSD32 + +#define VDSO_TIMEHANDS_MD32 VDSO_TIMEHANDS_MD + +#endif +#endif +#endif diff --git a/sys/kern/imgact_elf.c b/sys/kern/imgact_elf.c index 3d39fe31f..b274c6363 100644 --- a/sys/kern/imgact_elf.c +++ b/sys/kern/imgact_elf.c @@ -1015,6 +1015,10 @@ __elfN(freebsd_fixup)(register_t **stack_base, struct image_params *imgp) AUXARGS_ENTRY(pos, AT_PAGESIZES, imgp->pagesizes); AUXARGS_ENTRY(pos, AT_PAGESIZESLEN, imgp->pagesizeslen); } + if (imgp->sysent->sv_timekeep_base != 0) { + AUXARGS_ENTRY(pos, AT_TIMEKEEP, + imgp->sysent->sv_timekeep_base); + } AUXARGS_ENTRY(pos, AT_STACKPROT, imgp->sysent->sv_shared_page_obj != NULL && imgp->stack_prot != 0 ? imgp->stack_prot : imgp->sysent->sv_stackprot); diff --git a/sys/kern/kern_exec.c b/sys/kern/kern_exec.c index 30f3a10e4..e2dc01995 100644 --- a/sys/kern/kern_exec.c +++ b/sys/kern/kern_exec.c @@ -28,6 +28,7 @@ __FBSDID("$FreeBSD$"); #include "opt_capsicum.h" +#include "opt_compat.h" #include "opt_hwpmc_hooks.h" #include "opt_kdtrace.h" #include "opt_ktrace.h" @@ -64,6 +65,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #ifdef KTRACE @@ -1517,42 +1519,13 @@ exec_unregister(execsw_arg) static struct sx shared_page_alloc_sx; static vm_object_t shared_page_obj; static int shared_page_free; - -struct sf_buf * -shared_page_write_start(int base) -{ - vm_page_t m; - struct sf_buf *s; - - VM_OBJECT_LOCK(shared_page_obj); - m = vm_page_grab(shared_page_obj, OFF_TO_IDX(base), VM_ALLOC_RETRY); - VM_OBJECT_UNLOCK(shared_page_obj); - s = sf_buf_alloc(m, SFB_DEFAULT); - return (s); -} - -void -shared_page_write_end(struct sf_buf *sf) -{ - vm_page_t m; - - m = sf_buf_page(sf); - sf_buf_free(sf); - VM_OBJECT_LOCK(shared_page_obj); - vm_page_wakeup(m); - VM_OBJECT_UNLOCK(shared_page_obj); -} +char *shared_page_mapping; void shared_page_write(int base, int size, const void *data) { - struct sf_buf *sf; - vm_offset_t sk; - sf = shared_page_write_start(base); - sk = sf_buf_kva(sf); - bcopy(data, (void *)(sk + (base & PAGE_MASK)), size); - shared_page_write_end(sf); + bcopy(data, shared_page_mapping + base, size); } static int @@ -1596,6 +1569,7 @@ static void shared_page_init(void *dummy __unused) { vm_page_t m; + vm_offset_t addr; sx_init(&shared_page_alloc_sx, "shpsx"); shared_page_obj = vm_pager_allocate(OBJT_PHYS, 0, PAGE_SIZE, @@ -1605,15 +1579,95 @@ shared_page_init(void *dummy __unused) VM_ALLOC_ZERO); m->valid = VM_PAGE_BITS_ALL; VM_OBJECT_UNLOCK(shared_page_obj); + addr = kmem_alloc_nofault(kernel_map, PAGE_SIZE); + pmap_qenter(addr, &m, 1); + shared_page_mapping = (char *)addr; } SYSINIT(shp, SI_SUB_EXEC, SI_ORDER_FIRST, (sysinit_cfunc_t)shared_page_init, NULL); +static void +timehands_update(struct sysentvec *sv) +{ + struct vdso_timehands th; + struct vdso_timekeep *tk; + uint32_t enabled, idx; + + enabled = tc_fill_vdso_timehands(&th); + tk = (struct vdso_timekeep *)(shared_page_mapping + + sv->sv_timekeep_off); + idx = sv->sv_timekeep_curr; + atomic_store_rel_32(&tk->tk_th[idx].th_gen, 0); + if (++idx >= VDSO_TH_NUM) + idx = 0; + sv->sv_timekeep_curr = idx; + if (++sv->sv_timekeep_gen == 0) + sv->sv_timekeep_gen = 1; + th.th_gen = 0; + if (enabled) + tk->tk_th[idx] = th; + tk->tk_enabled = enabled; + atomic_store_rel_32(&tk->tk_th[idx].th_gen, sv->sv_timekeep_gen); + tk->tk_current = idx; +} + +#ifdef COMPAT_FREEBSD32 +static void +timehands_update32(struct sysentvec *sv) +{ + struct vdso_timekeep32 *tk; + struct vdso_timehands32 th; + uint32_t enabled, idx; + + enabled = tc_fill_vdso_timehands32(&th); + tk = (struct vdso_timekeep32 *)(shared_page_mapping + + sv->sv_timekeep_off); + idx = sv->sv_timekeep_curr; + atomic_store_rel_32(&tk->tk_th[idx].th_gen, 0); + if (++idx >= VDSO_TH_NUM) + idx = 0; + sv->sv_timekeep_curr = idx; + if (++sv->sv_timekeep_gen == 0) + sv->sv_timekeep_gen = 1; + th.th_gen = 0; + if (enabled) + tk->tk_th[idx] = th; + tk->tk_enabled = enabled; + atomic_store_rel_32(&tk->tk_th[idx].th_gen, sv->sv_timekeep_gen); + tk->tk_current = idx; +} +#endif + +/* + * This is hackish, but easiest way to avoid creating list structures + * that needs to be iterated over from the hardclock interrupt + * context. + */ +static struct sysentvec *host_sysentvec; +#ifdef COMPAT_FREEBSD32 +static struct sysentvec *compat32_sysentvec; +#endif + +void +timekeep_push_vdso(void) +{ + + if (host_sysentvec != NULL && host_sysentvec->sv_timekeep_base != 0) + timehands_update(host_sysentvec); +#ifdef COMPAT_FREEBSD32 + if (compat32_sysentvec != NULL && + compat32_sysentvec->sv_timekeep_base != 0) + timehands_update32(compat32_sysentvec); +#endif +} + void exec_sysvec_init(void *param) { struct sysentvec *sv; + int tk_base; + uint32_t tk_ver; sv = (struct sysentvec *)param; @@ -1622,4 +1676,32 @@ exec_sysvec_init(void *param) sv->sv_shared_page_obj = shared_page_obj; sv->sv_sigcode_base = sv->sv_shared_page_base + shared_page_fill(*(sv->sv_szsigcode), 16, sv->sv_sigcode); + if ((sv->sv_flags & SV_ABI_MASK) != SV_ABI_FREEBSD) + return; + tk_ver = VDSO_TK_VER_CURR; +#ifdef COMPAT_FREEBSD32 + if ((sv->sv_flags & SV_ILP32) != 0) { + tk_base = shared_page_alloc(sizeof(struct vdso_timekeep32) + + sizeof(struct vdso_timehands32) * VDSO_TH_NUM, 16); + KASSERT(tk_base != -1, ("tk_base -1 for 32bit")); + shared_page_write(tk_base + offsetof(struct vdso_timekeep32, + tk_ver), sizeof(uint32_t), &tk_ver); + KASSERT(compat32_sysentvec == 0, + ("Native compat32 already registered")); + compat32_sysentvec = sv; + } else { +#endif + tk_base = shared_page_alloc(sizeof(struct vdso_timekeep) + + sizeof(struct vdso_timehands) * VDSO_TH_NUM, 16); + KASSERT(tk_base != -1, ("tk_base -1 for native")); + shared_page_write(tk_base + offsetof(struct vdso_timekeep, + tk_ver), sizeof(uint32_t), &tk_ver); + KASSERT(host_sysentvec == 0, ("Native already registered")); + host_sysentvec = sv; +#ifdef COMPAT_FREEBSD32 + } +#endif + sv->sv_timekeep_base = sv->sv_shared_page_base + tk_base; + sv->sv_timekeep_off = tk_base; + timekeep_push_vdso(); } diff --git a/sys/kern/kern_tc.c b/sys/kern/kern_tc.c index 0c520711e..d6b69bebe 100644 --- a/sys/kern/kern_tc.c +++ b/sys/kern/kern_tc.c @@ -10,6 +10,7 @@ #include __FBSDID("$FreeBSD$"); +#include "opt_compat.h" #include "opt_ntp.h" #include @@ -20,6 +21,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include /* * A large step happens on boot. This constant detects such steps. @@ -544,6 +546,7 @@ tc_windup(void) time_second = th->th_microtime.tv_sec; time_uptime = th->th_offset.sec; timehands = th; + timekeep_push_vdso(); } /* Report or change the active timecounter hardware. */ @@ -570,6 +573,7 @@ sysctl_kern_timecounter_hardware(SYSCTL_HANDLER_ARGS) (void)newtc->tc_get_timecount(newtc); timecounter = newtc; + timekeep_push_vdso(); return (0); } return (EINVAL); @@ -960,3 +964,63 @@ cputick2usec(uint64_t tick) } cpu_tick_f *cpu_ticks = tc_cpu_ticks; + +static int vdso_th_enable = 1; +static int +sysctl_fast_gettime(SYSCTL_HANDLER_ARGS) +{ + int old_vdso_th_enable, error; + + old_vdso_th_enable = vdso_th_enable; + error = sysctl_handle_int(oidp, &old_vdso_th_enable, 0, req); + if (error != 0) + return (error); + vdso_th_enable = old_vdso_th_enable; + timekeep_push_vdso(); + return (0); +} +SYSCTL_PROC(_kern_timecounter, OID_AUTO, fast_gettime, + CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, + NULL, 0, sysctl_fast_gettime, "I", "Enable fast time of day"); + +uint32_t +tc_fill_vdso_timehands(struct vdso_timehands *vdso_th) +{ + struct timehands *th; + uint32_t enabled; + + th = timehands; + vdso_th->th_algo = VDSO_TH_ALGO_1; + vdso_th->th_scale = th->th_scale; + vdso_th->th_offset_count = th->th_offset_count; + vdso_th->th_counter_mask = th->th_counter->tc_counter_mask; + vdso_th->th_offset = th->th_offset; + vdso_th->th_boottime = boottimebin; + enabled = cpu_fill_vdso_timehands(vdso_th); + if (!vdso_th_enable) + enabled = 0; + return (enabled); +} + +#ifdef COMPAT_FREEBSD32 +uint32_t +tc_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32) +{ + struct timehands *th; + uint32_t enabled; + + th = timehands; + vdso_th32->th_algo = VDSO_TH_ALGO_1; + *(uint64_t *)&vdso_th32->th_scale[0] = th->th_scale; + vdso_th32->th_offset_count = th->th_offset_count; + vdso_th32->th_counter_mask = th->th_counter->tc_counter_mask; + vdso_th32->th_offset.sec = th->th_offset.sec; + *(uint64_t *)&vdso_th32->th_offset.frac[0] = th->th_offset.frac; + vdso_th32->th_boottime.sec = boottimebin.sec; + *(uint64_t *)&vdso_th32->th_boottime.frac[0] = boottimebin.frac; + enabled = cpu_fill_vdso_timehands32(vdso_th32); + if (!vdso_th_enable) + enabled = 0; + return (enabled); +} +#endif diff --git a/sys/kern/subr_dummy_vdso_tc.c b/sys/kern/subr_dummy_vdso_tc.c new file mode 100644 index 000000000..9c845014f --- /dev/null +++ b/sys/kern/subr_dummy_vdso_tc.c @@ -0,0 +1,49 @@ +/*- + * Copyright 2012 Konstantin Belousov . + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include +__FBSDID("$FreeBSD$"); + +#include "opt_compat.h" + +#include +#include + +uint32_t +cpu_fill_vdso_timehands(struct vdso_timehands *vdso_th) +{ + + return (0); +} + +#ifdef COMPAT_FREEBSD32 +uint32_t +cpu_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32) +{ + + return (0); +} +#endif diff --git a/sys/mips/include/vdso.h b/sys/mips/include/vdso.h new file mode 100644 index 000000000..082df1ae5 --- /dev/null +++ b/sys/mips/include/vdso.h @@ -0,0 +1,41 @@ +/*- + * Copyright 2012 Konstantin Belousov . + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MIPS_VDSO_H +#define _MIPS_VDSO_H + +#define VDSO_TIMEHANDS_MD \ + uint32_t th_res[8]; + +#ifdef _KERNEL +#ifdef COMPAT_FREEBSD32 + +#define VDSO_TIMEHANDS_MD32 VDSO_TIMEHANDS_MD + +#endif +#endif +#endif diff --git a/sys/pc98/include/vdso.h b/sys/pc98/include/vdso.h new file mode 100644 index 000000000..b81c455a9 --- /dev/null +++ b/sys/pc98/include/vdso.h @@ -0,0 +1,6 @@ +/*- + * This file is in the public domain. + */ +/* $FreeBSD$ */ + +#include diff --git a/sys/powerpc/include/vdso.h b/sys/powerpc/include/vdso.h new file mode 100644 index 000000000..0bdafd982 --- /dev/null +++ b/sys/powerpc/include/vdso.h @@ -0,0 +1,41 @@ +/*- + * Copyright 2012 Konstantin Belousov . + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _POWERPC_VDSO_H +#define _POWERPC_VDSO_H + +#define VDSO_TIMEHANDS_MD \ + uint32_t th_res[8]; + +#ifdef _KERNEL +#ifdef COMPAT_FREEBSD32 + +#define VDSO_TIMEHANDS_MD32 VDSO_TIMEHANDS_MD + +#endif +#endif +#endif diff --git a/sys/sparc64/include/vdso.h b/sys/sparc64/include/vdso.h new file mode 100644 index 000000000..98b291e7d --- /dev/null +++ b/sys/sparc64/include/vdso.h @@ -0,0 +1,34 @@ +/*- + * Copyright 2012 Konstantin Belousov . + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SPARC64_VDSO_H +#define _SPARC64_VDSO_H + +#define VDSO_TIMEHANDS_MD \ + uint32_t th_res[8]; + +#endif diff --git a/sys/sys/sysent.h b/sys/sys/sysent.h index 9dd5c1caf..6de72d94e 100644 --- a/sys/sys/sysent.h +++ b/sys/sys/sysent.h @@ -124,6 +124,10 @@ struct sysentvec { vm_offset_t sv_shared_page_base; vm_offset_t sv_shared_page_len; vm_offset_t sv_sigcode_base; + vm_offset_t sv_timekeep_base; + int sv_timekeep_off; + int sv_timekeep_curr; + uint32_t sv_timekeep_gen; void *sv_shared_page_obj; void (*sv_schedtail)(struct thread *); }; @@ -261,8 +265,6 @@ int shared_page_alloc(int size, int align); int shared_page_fill(int size, int align, const void *data); void shared_page_write(int base, int size, const void *data); void exec_sysvec_init(void *param); -struct sf_buf *shared_page_write_start(int base); -void shared_page_write_end(struct sf_buf *sf); #define INIT_SYSENTVEC(name, sv) \ SYSINIT(name, SI_SUB_EXEC, SI_ORDER_ANY, \ diff --git a/sys/sys/vdso.h b/sys/sys/vdso.h new file mode 100644 index 000000000..653a60605 --- /dev/null +++ b/sys/sys/vdso.h @@ -0,0 +1,122 @@ +/*- + * Copyright 2012 Konstantin Belousov . + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SYS_VDSO_H +#define _SYS_VDSO_H + +#include +#include + +struct vdso_timehands { + uint32_t th_algo; + uint32_t th_gen; + uint64_t th_scale; + uint32_t th_offset_count; + uint32_t th_counter_mask; + struct bintime th_offset; + struct bintime th_boottime; + VDSO_TIMEHANDS_MD +}; + +struct vdso_timekeep { + uint32_t tk_ver; + uint32_t tk_enabled; + uint32_t tk_current; + struct vdso_timehands tk_th[]; +}; + +#define VDSO_TK_CURRENT_BUSY 0xffffffff +#define VDSO_TK_VER_1 0x1 +#define VDSO_TK_VER_CURR VDSO_TK_VER_1 +#define VDSO_TH_ALGO_1 0x1 + +#ifndef _KERNEL + +struct timespec; +struct timeval; +struct timezone; + +int __vdso_clock_gettime(clockid_t clock_id, struct timespec *ts); +#pragma weak __vdso_clock_gettime + +int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz); +#pragma weak __vdso_gettimeofday + +u_int __vdso_gettc(const struct vdso_timehands *vdso_th); +#pragma weak __vdso_gettc + +#endif + +#ifdef _KERNEL + +void timekeep_push_vdso(void); + +uint32_t tc_fill_vdso_timehands(struct vdso_timehands *vdso_th); + +/* + * The cpu_fill_vdso_timehands() function should fill MD-part of the + * struct vdso_timehands, which is both machine- and + * timecounter-depended. The return value should be 1 if fast + * userspace timecounter is enabled by hardware, and 0 otherwise. The + * global sysctl enable override is handled by machine-independed code + * after cpu_fill_vdso_timehands() call is made. + */ +uint32_t cpu_fill_vdso_timehands(struct vdso_timehands *vdso_th); + +#define VDSO_TH_NUM 4 + +#ifdef COMPAT_FREEBSD32 +struct bintime32 { + uint32_t sec; + uint32_t frac[2]; +}; + +struct vdso_timehands32 { + uint32_t th_algo; + uint32_t th_gen; + uint32_t th_scale[2]; + uint32_t th_offset_count; + uint32_t th_counter_mask; + struct bintime32 th_offset; + struct bintime32 th_boottime; + VDSO_TIMEHANDS_MD32 +}; + +struct vdso_timekeep32 { + uint32_t tk_ver; + uint32_t tk_enabled; + uint32_t tk_current; + struct vdso_timehands32 tk_th[]; +}; + +uint32_t tc_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32); +uint32_t cpu_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32); + +#endif +#endif + +#endif diff --git a/sys/x86/include/vdso.h b/sys/x86/include/vdso.h new file mode 100644 index 000000000..11714205d --- /dev/null +++ b/sys/x86/include/vdso.h @@ -0,0 +1,42 @@ +/*- + * Copyright 2012 Konstantin Belousov . + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _X86_VDSO_H +#define _X86_VDSO_H + +#define VDSO_TIMEHANDS_MD \ + uint32_t th_x86_shift; \ + uint32_t th_res[7]; + +#ifdef _KERNEL +#ifdef COMPAT_FREEBSD32 + +#define VDSO_TIMEHANDS_MD32 VDSO_TIMEHANDS_MD + +#endif +#endif +#endif diff --git a/sys/x86/x86/tsc.c b/sys/x86/x86/tsc.c index 6b40397eb..c253a9629 100644 --- a/sys/x86/x86/tsc.c +++ b/sys/x86/x86/tsc.c @@ -27,6 +27,7 @@ #include __FBSDID("$FreeBSD$"); +#include "opt_compat.h" #include "opt_clock.h" #include @@ -41,6 +42,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include @@ -606,3 +608,23 @@ tsc_get_timecount_low(struct timecounter *tc) : "=a" (rv) : "c" ((int)(intptr_t)tc->tc_priv) : "edx"); return (rv); } + +uint32_t +cpu_fill_vdso_timehands(struct vdso_timehands *vdso_th) +{ + + vdso_th->th_x86_shift = (int)(intptr_t)timecounter->tc_priv; + bzero(vdso_th->th_res, sizeof(vdso_th->th_res)); + return (timecounter == &tsc_timecounter); +} + +#ifdef COMPAT_FREEBSD32 +uint32_t +cpu_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32) +{ + + vdso_th32->th_x86_shift = (int)(intptr_t)timecounter->tc_priv; + bzero(vdso_th32->th_res, sizeof(vdso_th32->th_res)); + return (timecounter == &tsc_timecounter); +} +#endif -- 2.45.0