2 * Copyright (c) 2004 Doug Rabson
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * Define stubs for TLS internals so that programs and libraries can
31 * link. These functions will be replaced by functional versions at
32 * runtime from ld-elf.so.1.
35 #include <sys/cdefs.h>
36 #include <sys/param.h>
41 #include "libc_private.h"
43 /* Provided by jemalloc to avoid bootstrapping issues. */
44 void *__je_bootstrap_malloc(size_t size);
45 void *__je_bootstrap_calloc(size_t num, size_t size);
46 void __je_bootstrap_free(void *ptr);
48 __weak_reference(__libc_allocate_tls, _rtld_allocate_tls);
49 __weak_reference(__libc_free_tls, _rtld_free_tls);
53 __weak_reference(___libc_tls_get_addr, ___tls_get_addr);
54 __attribute__((__regparm__(1))) void * ___libc_tls_get_addr(void *);
58 void * __libc_tls_get_addr(void *);
59 __weak_reference(__libc_tls_get_addr, __tls_get_addr);
61 void *_rtld_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign);
62 void _rtld_free_tls(void *tls, size_t tcbsize, size_t tcbalign);
63 void *__libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign);
64 void __libc_free_tls(void *tls, size_t tcbsize, size_t tcbalign);
66 #if defined(__amd64__)
67 #define TLS_TCB_ALIGN 16
68 #elif defined(__aarch64__) || defined(__arm__) || defined(__i386__) || \
69 defined(__mips__) || defined(__powerpc__) || defined(__riscv__) || \
71 #define TLS_TCB_ALIGN sizeof(void *)
73 #error TLS_TCB_ALIGN undefined for target architecture
76 #if defined(__aarch64__) || defined(__arm__) || defined(__mips__) || \
77 defined(__powerpc__) || defined(__riscv__)
80 #if defined(__i386__) || defined(__amd64__) || defined(__sparc64__)
81 #define TLS_VARIANT_II
86 static size_t tls_static_space;
87 static size_t tls_init_size;
88 static void *tls_init;
95 __attribute__((__regparm__(1)))
97 ___libc_tls_get_addr(void *ti __unused)
105 __libc_tls_get_addr(void *ti __unused)
114 #define TLS_TCB_SIZE (2 * sizeof(void *))
117 * Free Static TLS using the Variant I method.
120 __libc_free_tls(void *tcb, size_t tcbsize, size_t tcbalign __unused)
125 tls = (Elf_Addr **)((Elf_Addr)tcb + tcbsize - TLS_TCB_SIZE);
127 __je_bootstrap_free(dtv);
128 __je_bootstrap_free(tcb);
132 * Allocate Static TLS using the Variant I method.
135 __libc_allocate_tls(void *oldtcb, size_t tcbsize, size_t tcbalign __unused)
141 if (oldtcb != NULL && tcbsize == TLS_TCB_SIZE)
144 tcb = __je_bootstrap_calloc(1, tls_static_space + tcbsize - TLS_TCB_SIZE);
145 tls = (Elf_Addr **)(tcb + tcbsize - TLS_TCB_SIZE);
147 if (oldtcb != NULL) {
148 memcpy(tls, oldtcb, tls_static_space);
149 __je_bootstrap_free(oldtcb);
151 /* Adjust the DTV. */
153 dtv[2] = (Elf_Addr)tls + TLS_TCB_SIZE;
155 dtv = __je_bootstrap_malloc(3 * sizeof(Elf_Addr));
159 dtv[2] = (Elf_Addr)tls + TLS_TCB_SIZE;
161 if (tls_init_size > 0)
162 memcpy((void*)dtv[2], tls_init, tls_init_size);
163 if (tls_static_space > tls_init_size)
164 memset((void*)(dtv[2] + tls_init_size), 0,
165 tls_static_space - tls_init_size);
173 #ifdef TLS_VARIANT_II
175 #define TLS_TCB_SIZE (3 * sizeof(Elf_Addr))
178 * Free Static TLS using the Variant II method.
181 __libc_free_tls(void *tcb, size_t tcbsize __unused, size_t tcbalign)
185 Elf_Addr tlsstart, tlsend;
188 * Figure out the size of the initial TLS block so that we can
189 * find stuff which ___tls_get_addr() allocated dynamically.
191 size = roundup2(tls_static_space, tcbalign);
193 dtv = ((Elf_Addr**)tcb)[1];
194 tlsend = (Elf_Addr) tcb;
195 tlsstart = tlsend - size;
196 __je_bootstrap_free((void*) tlsstart);
197 __je_bootstrap_free(dtv);
201 * Allocate Static TLS using the Variant II method.
204 __libc_allocate_tls(void *oldtls, size_t tcbsize, size_t tcbalign)
209 Elf_Addr segbase, oldsegbase;
211 size = roundup2(tls_static_space, tcbalign);
213 if (tcbsize < 2 * sizeof(Elf_Addr))
214 tcbsize = 2 * sizeof(Elf_Addr);
215 tls = __je_bootstrap_calloc(1, size + tcbsize);
216 dtv = __je_bootstrap_malloc(3 * sizeof(Elf_Addr));
218 segbase = (Elf_Addr)(tls + size);
219 ((Elf_Addr*)segbase)[0] = segbase;
220 ((Elf_Addr*)segbase)[1] = (Elf_Addr) dtv;
224 dtv[2] = segbase - tls_static_space;
228 * Copy the static TLS block over whole.
230 oldsegbase = (Elf_Addr) oldtls;
231 memcpy((void *)(segbase - tls_static_space),
232 (const void *)(oldsegbase - tls_static_space),
236 * We assume that this block was the one we created with
237 * allocate_initial_tls().
239 _rtld_free_tls(oldtls, 2*sizeof(Elf_Addr), sizeof(Elf_Addr));
241 memcpy((void *)(segbase - tls_static_space),
242 tls_init, tls_init_size);
243 memset((void *)(segbase - tls_static_space + tls_init_size),
244 0, tls_static_space - tls_init_size);
247 return (void*) segbase;
250 #endif /* TLS_VARIANT_II */
255 __libc_allocate_tls(void *oldtls __unused, size_t tcbsize __unused,
256 size_t tcbalign __unused)
262 __libc_free_tls(void *tcb __unused, size_t tcbsize __unused,
263 size_t tcbalign __unused)
269 extern char **environ;
276 Elf_Auxinfo *aux, *auxp;
282 sp = (Elf_Addr *) environ;
285 aux = (Elf_Auxinfo *) sp;
288 for (auxp = aux; auxp->a_type != AT_NULL; auxp++) {
289 switch (auxp->a_type) {
291 phdr = auxp->a_un.a_ptr;
295 phent = auxp->a_un.a_val;
299 phnum = auxp->a_un.a_val;
303 if (phdr == NULL || phent != sizeof(Elf_Phdr) || phnum == 0)
306 for (i = 0; (unsigned) i < phnum; i++) {
307 if (phdr[i].p_type == PT_TLS) {
308 tls_static_space = roundup2(phdr[i].p_memsz,
310 tls_init_size = phdr[i].p_filesz;
311 tls_init = (void*) phdr[i].p_vaddr;
317 * tls_static_space should include space for TLS structure
319 tls_static_space += TLS_TCB_SIZE;
322 tls = _rtld_allocate_tls(NULL, TLS_TCB_SIZE, TLS_TCB_ALIGN);