]> CyberLeo.Net >> Repos - FreeBSD/FreeBSD.git/blob - sys/kern/kern_sharedpage.c
Merge OpenSSL 1.1.1f.
[FreeBSD/FreeBSD.git] / sys / kern / kern_sharedpage.c
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2010, 2012 Konstantin Belousov <kib@FreeBSD.org>
5  * Copyright (c) 2015 The FreeBSD Foundation
6  * All rights reserved.
7  *
8  * Portions of this software were developed by Konstantin Belousov
9  * under sponsorship from the FreeBSD Foundation.
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions
13  * are met:
14  * 1. Redistributions of source code must retain the above copyright
15  *    notice, this list of conditions and the following disclaimer.
16  * 2. Redistributions in binary form must reproduce the above copyright
17  *    notice, this list of conditions and the following disclaimer in the
18  *    documentation and/or other materials provided with the distribution.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
21  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
24  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30  * SUCH DAMAGE.
31  */
32
33 #include <sys/cdefs.h>
34 __FBSDID("$FreeBSD$");
35
36 #include "opt_vm.h"
37
38 #include <sys/param.h>
39 #include <sys/systm.h>
40 #include <sys/kernel.h>
41 #include <sys/lock.h>
42 #include <sys/malloc.h>
43 #include <sys/rwlock.h>
44 #include <sys/sysent.h>
45 #include <sys/sysctl.h>
46 #include <sys/vdso.h>
47
48 #include <vm/vm.h>
49 #include <vm/vm_param.h>
50 #include <vm/pmap.h>
51 #include <vm/vm_extern.h>
52 #include <vm/vm_kern.h>
53 #include <vm/vm_map.h>
54 #include <vm/vm_object.h>
55 #include <vm/vm_page.h>
56 #include <vm/vm_pager.h>
57
58 static struct sx shared_page_alloc_sx;
59 static vm_object_t shared_page_obj;
60 static int shared_page_free;
61 char *shared_page_mapping;
62
63 void
64 shared_page_write(int base, int size, const void *data)
65 {
66
67         bcopy(data, shared_page_mapping + base, size);
68 }
69
70 static int
71 shared_page_alloc_locked(int size, int align)
72 {
73         int res;
74
75         res = roundup(shared_page_free, align);
76         if (res + size >= IDX_TO_OFF(shared_page_obj->size))
77                 res = -1;
78         else
79                 shared_page_free = res + size;
80         return (res);
81 }
82
83 int
84 shared_page_alloc(int size, int align)
85 {
86         int res;
87
88         sx_xlock(&shared_page_alloc_sx);
89         res = shared_page_alloc_locked(size, align);
90         sx_xunlock(&shared_page_alloc_sx);
91         return (res);
92 }
93
94 int
95 shared_page_fill(int size, int align, const void *data)
96 {
97         int res;
98
99         sx_xlock(&shared_page_alloc_sx);
100         res = shared_page_alloc_locked(size, align);
101         if (res != -1)
102                 shared_page_write(res, size, data);
103         sx_xunlock(&shared_page_alloc_sx);
104         return (res);
105 }
106
107 static void
108 shared_page_init(void *dummy __unused)
109 {
110         vm_page_t m;
111         vm_offset_t addr;
112
113         sx_init(&shared_page_alloc_sx, "shpsx");
114         shared_page_obj = vm_pager_allocate(OBJT_PHYS, 0, PAGE_SIZE,
115             VM_PROT_DEFAULT, 0, NULL);
116         VM_OBJECT_WLOCK(shared_page_obj);
117         m = vm_page_grab(shared_page_obj, 0, VM_ALLOC_ZERO);
118         VM_OBJECT_WUNLOCK(shared_page_obj);
119         vm_page_valid(m);
120         vm_page_xunbusy(m);
121         addr = kva_alloc(PAGE_SIZE);
122         pmap_qenter(addr, &m, 1);
123         shared_page_mapping = (char *)addr;
124 }
125
126 SYSINIT(shp, SI_SUB_EXEC, SI_ORDER_FIRST, (sysinit_cfunc_t)shared_page_init,
127     NULL);
128
129 /*
130  * Push the timehands update to the shared page.
131  *
132  * The lockless update scheme is similar to the one used to update the
133  * in-kernel timehands, see sys/kern/kern_tc.c:tc_windup() (which
134  * calls us after the timehands are updated).
135  */
136 static void
137 timehands_update(struct vdso_sv_tk *svtk)
138 {
139         struct vdso_timehands th;
140         struct vdso_timekeep *tk;
141         uint32_t enabled, idx;
142
143         enabled = tc_fill_vdso_timehands(&th);
144         th.th_gen = 0;
145         idx = svtk->sv_timekeep_curr;
146         if (++idx >= VDSO_TH_NUM)
147                 idx = 0;
148         svtk->sv_timekeep_curr = idx;
149         if (++svtk->sv_timekeep_gen == 0)
150                 svtk->sv_timekeep_gen = 1;
151
152         tk = (struct vdso_timekeep *)(shared_page_mapping +
153             svtk->sv_timekeep_off);
154         tk->tk_th[idx].th_gen = 0;
155         atomic_thread_fence_rel();
156         if (enabled)
157                 tk->tk_th[idx] = th;
158         atomic_store_rel_32(&tk->tk_th[idx].th_gen, svtk->sv_timekeep_gen);
159         atomic_store_rel_32(&tk->tk_current, idx);
160
161         /*
162          * The ordering of the assignment to tk_enabled relative to
163          * the update of the vdso_timehands is not important.
164          */
165         tk->tk_enabled = enabled;
166 }
167
168 #ifdef COMPAT_FREEBSD32
169 static void
170 timehands_update32(struct vdso_sv_tk *svtk)
171 {
172         struct vdso_timehands32 th;
173         struct vdso_timekeep32 *tk;
174         uint32_t enabled, idx;
175
176         enabled = tc_fill_vdso_timehands32(&th);
177         th.th_gen = 0;
178         idx = svtk->sv_timekeep_curr;
179         if (++idx >= VDSO_TH_NUM)
180                 idx = 0;
181         svtk->sv_timekeep_curr = idx;
182         if (++svtk->sv_timekeep_gen == 0)
183                 svtk->sv_timekeep_gen = 1;
184
185         tk = (struct vdso_timekeep32 *)(shared_page_mapping +
186             svtk->sv_timekeep_off);
187         tk->tk_th[idx].th_gen = 0;
188         atomic_thread_fence_rel();
189         if (enabled)
190                 tk->tk_th[idx] = th;
191         atomic_store_rel_32(&tk->tk_th[idx].th_gen, svtk->sv_timekeep_gen);
192         atomic_store_rel_32(&tk->tk_current, idx);
193         tk->tk_enabled = enabled;
194 }
195 #endif
196
197 /*
198  * This is hackish, but easiest way to avoid creating list structures
199  * that needs to be iterated over from the hardclock interrupt
200  * context.
201  */
202 static struct vdso_sv_tk *host_svtk;
203 #ifdef COMPAT_FREEBSD32
204 static struct vdso_sv_tk *compat32_svtk;
205 #endif
206
207 void
208 timekeep_push_vdso(void)
209 {
210
211         if (host_svtk != NULL)
212                 timehands_update(host_svtk);
213 #ifdef COMPAT_FREEBSD32
214         if (compat32_svtk != NULL)
215                 timehands_update32(compat32_svtk);
216 #endif
217 }
218
219 struct vdso_sv_tk *
220 alloc_sv_tk(void)
221 {
222         struct vdso_sv_tk *svtk;
223         int tk_base;
224         uint32_t tk_ver;
225
226         tk_ver = VDSO_TK_VER_CURR;
227         svtk = malloc(sizeof(struct vdso_sv_tk), M_TEMP, M_WAITOK | M_ZERO);
228         tk_base = shared_page_alloc(sizeof(struct vdso_timekeep) +
229             sizeof(struct vdso_timehands) * VDSO_TH_NUM, 16);
230         KASSERT(tk_base != -1, ("tk_base -1 for native"));
231         shared_page_write(tk_base + offsetof(struct vdso_timekeep, tk_ver),
232             sizeof(uint32_t), &tk_ver);
233         svtk->sv_timekeep_off = tk_base;
234         timekeep_push_vdso();
235         return (svtk);
236 }
237
238 #ifdef COMPAT_FREEBSD32
239 struct vdso_sv_tk *
240 alloc_sv_tk_compat32(void)
241 {
242         struct vdso_sv_tk *svtk;
243         int tk_base;
244         uint32_t tk_ver;
245
246         svtk = malloc(sizeof(struct vdso_sv_tk), M_TEMP, M_WAITOK | M_ZERO);
247         tk_ver = VDSO_TK_VER_CURR;
248         tk_base = shared_page_alloc(sizeof(struct vdso_timekeep32) +
249             sizeof(struct vdso_timehands32) * VDSO_TH_NUM, 16);
250         KASSERT(tk_base != -1, ("tk_base -1 for 32bit"));
251         shared_page_write(tk_base + offsetof(struct vdso_timekeep32,
252             tk_ver), sizeof(uint32_t), &tk_ver);
253         svtk->sv_timekeep_off = tk_base;
254         timekeep_push_vdso();
255         return (svtk);
256 }
257 #endif
258
259 void
260 exec_sysvec_init(void *param)
261 {
262         struct sysentvec *sv;
263
264         sv = (struct sysentvec *)param;
265         if ((sv->sv_flags & SV_SHP) == 0)
266                 return;
267         sv->sv_shared_page_obj = shared_page_obj;
268         sv->sv_sigcode_base = sv->sv_shared_page_base +
269             shared_page_fill(*(sv->sv_szsigcode), 16, sv->sv_sigcode);
270         if ((sv->sv_flags & SV_ABI_MASK) != SV_ABI_FREEBSD)
271                 return;
272         if ((sv->sv_flags & SV_TIMEKEEP) != 0) {
273 #ifdef COMPAT_FREEBSD32
274                 if ((sv->sv_flags & SV_ILP32) != 0) {
275                         KASSERT(compat32_svtk == NULL,
276                             ("Compat32 already registered"));
277                         compat32_svtk = alloc_sv_tk_compat32();
278                         sv->sv_timekeep_base = sv->sv_shared_page_base +
279                             compat32_svtk->sv_timekeep_off;
280                 } else {
281 #endif
282                         KASSERT(host_svtk == NULL, ("Host already registered"));
283                         host_svtk = alloc_sv_tk();
284                         sv->sv_timekeep_base = sv->sv_shared_page_base +
285                             host_svtk->sv_timekeep_off;
286 #ifdef COMPAT_FREEBSD32
287                 }
288 #endif
289         }
290 }