1 //===-------- cfi.cpp -----------------------------------------------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file implements the runtime support for the cross-DSO CFI.
11 //===----------------------------------------------------------------------===//
16 #include "sanitizer_common/sanitizer_common.h"
18 #include <sys/link_elf.h>
26 typedef ElfW(Phdr) Elf_Phdr;
27 typedef ElfW(Ehdr) Elf_Ehdr;
28 typedef ElfW(Addr) Elf_Addr;
29 typedef ElfW(Sym) Elf_Sym;
30 typedef ElfW(Dyn) Elf_Dyn;
31 #elif SANITIZER_FREEBSD
32 #if SANITIZER_WORDSIZE == 64
33 #define ElfW64_Dyn Elf_Dyn
34 #define ElfW64_Sym Elf_Sym
36 #define ElfW32_Dyn Elf_Dyn
37 #define ElfW32_Sym Elf_Sym
41 #include "interception/interception.h"
42 #include "sanitizer_common/sanitizer_flag_parser.h"
43 #include "ubsan/ubsan_init.h"
44 #include "ubsan/ubsan_flags.h"
46 #ifdef CFI_ENABLE_DIAG
47 #include "ubsan/ubsan_handlers.h"
50 using namespace __sanitizer;
54 #define kCfiShadowLimitsStorageSize 4096 // 1 page
55 // Lets hope that the data segment is mapped with 4K pages.
56 // The pointer to the cfi shadow region is stored at the start of this page.
57 // The rest of the page is unused and re-mapped read-only.
59 char space[kCfiShadowLimitsStorageSize];
64 } cfi_shadow_limits_storage
65 __attribute__((aligned(kCfiShadowLimitsStorageSize)));
66 static constexpr uptr kShadowGranularity = 12;
67 static constexpr uptr kShadowAlign = 1UL << kShadowGranularity; // 4096
69 static constexpr uint16_t kInvalidShadow = 0;
70 static constexpr uint16_t kUncheckedShadow = 0xFFFFU;
72 // Get the start address of the CFI shadow region.
74 return cfi_shadow_limits_storage.limits.start;
77 uptr GetShadowSize() {
78 return cfi_shadow_limits_storage.limits.size;
81 // This will only work while the shadow is not allocated.
82 void SetShadowSize(uptr size) {
83 cfi_shadow_limits_storage.limits.size = size;
86 uptr MemToShadowOffset(uptr x) {
87 return (x >> kShadowGranularity) << 1;
90 uint16_t *MemToShadow(uptr x, uptr shadow_base) {
91 return (uint16_t *)(shadow_base + MemToShadowOffset(x));
94 typedef int (*CFICheckFn)(u64, void *, void *);
96 // This class reads and decodes the shadow contents.
100 explicit ShadowValue(uptr addr, uint16_t v) : addr(addr), v(v) {}
103 bool is_invalid() const { return v == kInvalidShadow; }
105 bool is_unchecked() const { return v == kUncheckedShadow; }
107 CFICheckFn get_cfi_check() const {
108 assert(!is_invalid() && !is_unchecked());
109 uptr aligned_addr = addr & ~(kShadowAlign - 1);
110 uptr p = aligned_addr - (((uptr)v - 1) << kShadowGranularity);
111 return reinterpret_cast<CFICheckFn>(p);
114 // Load a shadow value for the given application memory address.
115 static const ShadowValue load(uptr addr) {
116 uptr shadow_base = GetShadow();
117 uptr shadow_offset = MemToShadowOffset(addr);
118 if (shadow_offset > GetShadowSize())
119 return ShadowValue(addr, kInvalidShadow);
122 addr, *reinterpret_cast<uint16_t *>(shadow_base + shadow_offset));
126 class ShadowBuilder {
130 // Allocate a new empty shadow (for the entire address space) on the side.
132 // Mark the given address range as unchecked.
133 // This is used for uninstrumented libraries like libc.
134 // Any CFI check with a target in that range will pass.
135 void AddUnchecked(uptr begin, uptr end);
136 // Mark the given address range as belonging to a library with the given
137 // cfi_check function.
138 void Add(uptr begin, uptr end, uptr cfi_check);
139 // Finish shadow construction. Atomically switch the current active shadow
140 // region with the newly constructed one and deallocate the former.
144 void ShadowBuilder::Start() {
145 shadow_ = (uptr)MmapNoReserveOrDie(GetShadowSize(), "CFI shadow");
146 VReport(1, "CFI: shadow at %zx .. %zx\n", shadow_, shadow_ + GetShadowSize());
149 void ShadowBuilder::AddUnchecked(uptr begin, uptr end) {
150 uint16_t *shadow_begin = MemToShadow(begin, shadow_);
151 uint16_t *shadow_end = MemToShadow(end - 1, shadow_) + 1;
152 // memset takes a byte, so our unchecked shadow value requires both bytes to
153 // be the same. Make sure we're ok during compilation.
154 static_assert((kUncheckedShadow & 0xff) == ((kUncheckedShadow >> 8) & 0xff),
155 "Both bytes of the 16-bit value must be the same!");
156 memset(shadow_begin, kUncheckedShadow & 0xff,
157 (shadow_end - shadow_begin) * sizeof(*shadow_begin));
160 void ShadowBuilder::Add(uptr begin, uptr end, uptr cfi_check) {
161 assert((cfi_check & (kShadowAlign - 1)) == 0);
163 // Don't fill anything below cfi_check. We can not represent those addresses
164 // in the shadow, and must make sure at codegen to place all valid call
165 // targets above cfi_check.
166 begin = Max(begin, cfi_check);
167 uint16_t *s = MemToShadow(begin, shadow_);
168 uint16_t *s_end = MemToShadow(end - 1, shadow_) + 1;
169 uint16_t sv = ((begin - cfi_check) >> kShadowGranularity) + 1;
170 for (; s < s_end; s++, sv++)
174 #if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD
175 void ShadowBuilder::Install() {
176 MprotectReadOnly(shadow_, GetShadowSize());
177 uptr main_shadow = GetShadow();
181 void *res = mremap((void *)shadow_, GetShadowSize(), GetShadowSize(),
182 MREMAP_MAYMOVE | MREMAP_FIXED, (void *)main_shadow);
183 CHECK(res != MAP_FAILED);
184 #elif SANITIZER_NETBSD
185 void *res = mremap((void *)shadow_, GetShadowSize(), (void *)main_shadow,
186 GetShadowSize(), MAP_FIXED);
187 CHECK(res != MAP_FAILED);
189 void *res = MmapFixedOrDie(shadow_, GetShadowSize(), "cfi shadow");
190 CHECK(res != MAP_FAILED);
191 ::memcpy(&shadow_, &main_shadow, GetShadowSize());
195 CHECK_EQ(kCfiShadowLimitsStorageSize, GetPageSizeCached());
196 CHECK_EQ(0, GetShadow());
197 cfi_shadow_limits_storage.limits.start = shadow_;
198 MprotectReadOnly((uptr)&cfi_shadow_limits_storage,
199 sizeof(cfi_shadow_limits_storage));
200 CHECK_EQ(shadow_, GetShadow());
204 #error not implemented
207 // This is a workaround for a glibc bug:
208 // https://sourceware.org/bugzilla/show_bug.cgi?id=15199
209 // Other platforms can, hopefully, just do
210 // dlopen(RTLD_NOLOAD | RTLD_LAZY)
211 // dlsym("__cfi_check").
212 uptr find_cfi_check_in_dso(dl_phdr_info *info) {
213 const Elf_Dyn *dynamic = nullptr;
214 for (int i = 0; i < info->dlpi_phnum; ++i) {
215 if (info->dlpi_phdr[i].p_type == PT_DYNAMIC) {
217 (const Elf_Dyn *)(info->dlpi_addr + info->dlpi_phdr[i].p_vaddr);
221 if (!dynamic) return 0;
222 uptr strtab = 0, symtab = 0, strsz = 0;
223 for (const Elf_Dyn *p = dynamic; p->d_tag != PT_NULL; ++p) {
224 if (p->d_tag == DT_SYMTAB)
225 symtab = p->d_un.d_ptr;
226 else if (p->d_tag == DT_STRTAB)
227 strtab = p->d_un.d_ptr;
228 else if (p->d_tag == DT_STRSZ)
229 strsz = p->d_un.d_ptr;
232 if (symtab > strtab) {
233 VReport(1, "Can not handle: symtab > strtab (%p > %zx)\n", symtab, strtab);
237 // Verify that strtab and symtab are inside of the same LOAD segment.
238 // This excludes VDSO, which has (very high) bogus strtab and symtab pointers.
240 for (phdr_idx = 0; phdr_idx < info->dlpi_phnum; phdr_idx++) {
241 const Elf_Phdr *phdr = &info->dlpi_phdr[phdr_idx];
242 if (phdr->p_type == PT_LOAD) {
243 uptr beg = info->dlpi_addr + phdr->p_vaddr;
244 uptr end = beg + phdr->p_memsz;
245 if (strtab >= beg && strtab + strsz < end && symtab >= beg &&
250 if (phdr_idx == info->dlpi_phnum) {
251 // Nope, either different segments or just bogus pointers.
252 // Can not handle this.
253 VReport(1, "Can not handle: symtab %p, strtab %zx\n", symtab, strtab);
257 for (const Elf_Sym *p = (const Elf_Sym *)symtab; (Elf_Addr)p < strtab;
259 // There is no reliable way to find the end of the symbol table. In
260 // lld-produces files, there are other sections between symtab and strtab.
261 // Stop looking when the symbol name is not inside strtab.
262 if (p->st_name >= strsz) break;
263 char *name = (char*)(strtab + p->st_name);
264 if (strcmp(name, "__cfi_check") == 0) {
265 assert(p->st_info == ELF32_ST_INFO(STB_GLOBAL, STT_FUNC) ||
266 p->st_info == ELF32_ST_INFO(STB_WEAK, STT_FUNC));
267 uptr addr = info->dlpi_addr + p->st_value;
274 int dl_iterate_phdr_cb(dl_phdr_info *info, size_t size, void *data) {
275 uptr cfi_check = find_cfi_check_in_dso(info);
277 VReport(1, "Module '%s' __cfi_check %zx\n", info->dlpi_name, cfi_check);
279 ShadowBuilder *b = reinterpret_cast<ShadowBuilder *>(data);
281 for (int i = 0; i < info->dlpi_phnum; i++) {
282 const Elf_Phdr *phdr = &info->dlpi_phdr[i];
283 if (phdr->p_type == PT_LOAD) {
284 // Jump tables are in the executable segment.
285 // VTables are in the non-executable one.
286 // Need to fill shadow for both.
287 // FIXME: reject writable if vtables are in the r/o segment. Depend on
289 uptr cur_beg = info->dlpi_addr + phdr->p_vaddr;
290 uptr cur_end = cur_beg + phdr->p_memsz;
292 VReport(1, " %zx .. %zx\n", cur_beg, cur_end);
293 b->Add(cur_beg, cur_end, cfi_check);
295 b->AddUnchecked(cur_beg, cur_end);
302 // Init or update shadow for the current set of loaded libraries.
303 void UpdateShadow() {
306 dl_iterate_phdr(dl_iterate_phdr_cb, &b);
311 CHECK_EQ(0, GetShadow());
312 CHECK_EQ(0, GetShadowSize());
314 uptr vma = GetMaxUserVirtualAddress();
315 // Shadow is 2 -> 2**kShadowGranularity.
316 SetShadowSize((vma >> (kShadowGranularity - 1)) + 1);
317 VReport(1, "CFI: VMA size %zx, shadow size %zx\n", vma, GetShadowSize());
322 THREADLOCAL int in_loader;
323 BlockingMutex shadow_update_lock(LINKER_INITIALIZED);
326 if (in_loader == 0) {
327 shadow_update_lock.Lock();
333 CHECK(in_loader > 0);
336 if (in_loader == 0) {
337 shadow_update_lock.Unlock();
341 ALWAYS_INLINE void CfiSlowPathCommon(u64 CallSiteTypeId, void *Ptr,
343 uptr Addr = (uptr)Ptr;
344 VReport(3, "__cfi_slowpath: %llx, %p\n", CallSiteTypeId, Ptr);
345 ShadowValue sv = ShadowValue::load(Addr);
346 if (sv.is_invalid()) {
347 VReport(1, "CFI: invalid memory region for a check target: %p\n", Ptr);
348 #ifdef CFI_ENABLE_DIAG
350 __ubsan_handle_cfi_check_fail(
351 reinterpret_cast<__ubsan::CFICheckFailData *>(DiagData), Addr, false);
357 if (sv.is_unchecked()) {
358 VReport(2, "CFI: unchecked call (shadow=FFFF): %p\n", Ptr);
361 CFICheckFn cfi_check = sv.get_cfi_check();
362 VReport(2, "__cfi_check at %p\n", cfi_check);
363 cfi_check(CallSiteTypeId, Ptr, DiagData);
366 void InitializeFlags() {
367 SetCommonFlagsDefaults();
368 #ifdef CFI_ENABLE_DIAG
369 __ubsan::Flags *uf = __ubsan::flags();
373 FlagParser cfi_parser;
374 RegisterCommonFlags(&cfi_parser);
375 cfi_parser.ParseStringFromEnv("CFI_OPTIONS");
377 #ifdef CFI_ENABLE_DIAG
378 FlagParser ubsan_parser;
379 __ubsan::RegisterUbsanFlags(&ubsan_parser, uf);
380 RegisterCommonFlags(&ubsan_parser);
382 const char *ubsan_default_options = __ubsan::MaybeCallUbsanDefaultOptions();
383 ubsan_parser.ParseString(ubsan_default_options);
384 ubsan_parser.ParseStringFromEnv("UBSAN_OPTIONS");
387 InitializeCommonFlags();
390 ReportUnrecognizedFlags();
392 if (common_flags()->help) {
393 cfi_parser.PrintFlagDescriptions();
399 using namespace __cfi;
401 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
402 __cfi_slowpath(u64 CallSiteTypeId, void *Ptr) {
403 CfiSlowPathCommon(CallSiteTypeId, Ptr, nullptr);
406 #ifdef CFI_ENABLE_DIAG
407 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void
408 __cfi_slowpath_diag(u64 CallSiteTypeId, void *Ptr, void *DiagData) {
409 CfiSlowPathCommon(CallSiteTypeId, Ptr, DiagData);
413 static void EnsureInterceptorsInitialized();
415 // Setup shadow for dlopen()ed libraries.
416 // The actual shadow setup happens after dlopen() returns, which means that
417 // a library can not be a target of any CFI checks while its constructors are
418 // running. It's unclear how to fix this without some extra help from libc.
419 // In glibc, mmap inside dlopen is not interceptable.
420 // Maybe a seccomp-bpf filter?
421 // We could insert a high-priority constructor into the library, but that would
422 // not help with the uninstrumented libraries.
423 INTERCEPTOR(void*, dlopen, const char *filename, int flag) {
424 EnsureInterceptorsInitialized();
426 void *handle = REAL(dlopen)(filename, flag);
431 INTERCEPTOR(int, dlclose, void *handle) {
432 EnsureInterceptorsInitialized();
434 int res = REAL(dlclose)(handle);
439 static BlockingMutex interceptor_init_lock(LINKER_INITIALIZED);
440 static bool interceptors_inited = false;
442 static void EnsureInterceptorsInitialized() {
443 BlockingMutexLock lock(&interceptor_init_lock);
444 if (interceptors_inited)
447 INTERCEPT_FUNCTION(dlopen);
448 INTERCEPT_FUNCTION(dlclose);
450 interceptors_inited = true;
453 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
454 #if !SANITIZER_CAN_USE_PREINIT_ARRAY
455 // On ELF platforms, the constructor is invoked using .preinit_array (see below)
456 __attribute__((constructor(0)))
459 SanitizerToolName = "CFI";
463 #ifdef CFI_ENABLE_DIAG
464 __ubsan::InitAsPlugin();
468 #if SANITIZER_CAN_USE_PREINIT_ARRAY
469 // On ELF platforms, run cfi initialization before any other constructors.
470 // On other platforms we use the constructor attribute to arrange to run our
471 // initialization early.
473 __attribute__((section(".preinit_array"),
474 used)) void (*__cfi_preinit)(void) = __cfi_init;