1 /*===- GCDAProfiling.c - Support library for GCDA file emission -----------===*\
3 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 |* See https://llvm.org/LICENSE.txt for license information.
5 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 |*===----------------------------------------------------------------------===*|
9 |* This file implements the call back routines for the gcov profiling
10 |* instrumentation pass. Link against this library when running code through
11 |* the -insert-gcov-profiling LLVM pass.
13 |* We emit files in a corrupt version of GCOV's "gcda" file format. These files
14 |* are only close enough that LCOV will happily parse them. Anything that lcov
15 |* ignores is missing.
17 |* TODO: gcov is multi-process safe by having each exit open the existing file
18 |* and append to it. We'd like to achieve that and be thread-safe too.
20 \*===----------------------------------------------------------------------===*/
22 #if !defined(__Fuchsia__)
31 #define WIN32_LEAN_AND_MEAN
33 #include "WindowsMMap.h"
37 #include <sys/types.h>
41 #if defined(__FreeBSD__) && defined(__i386__)
42 #define I386_FREEBSD 1
44 #define I386_FREEBSD 0
47 #if !defined(_MSC_VER) && !I386_FREEBSD
52 typedef unsigned char uint8_t;
53 typedef unsigned int uint32_t;
54 typedef unsigned long long uint64_t;
56 /* System headers define 'size_t' incorrectly on x64 FreeBSD (prior to
57 * FreeBSD 10, r232261) when compiled in 32-bit mode.
59 typedef unsigned char uint8_t;
60 typedef unsigned int uint32_t;
61 typedef unsigned long long uint64_t;
64 #include "InstrProfiling.h"
65 #include "InstrProfilingUtil.h"
67 /* #define DEBUG_GCDAPROFILING */
70 GCOV_DATA_MAGIC = 0x67636461, // "gcda"
72 GCOV_TAG_FUNCTION = 0x01000000,
73 GCOV_TAG_COUNTER_ARCS = 0x01a10000,
74 // GCOV_TAG_OBJECT_SUMMARY superseded GCOV_TAG_PROGRAM_SUMMARY in GCC 9.
75 GCOV_TAG_OBJECT_SUMMARY = 0xa1000000,
76 GCOV_TAG_PROGRAM_SUMMARY = 0xa3000000,
80 * --- GCOV file format I/O primitives ---
84 * The current file name we're outputting. Used primarily for error logging.
86 static char *filename = NULL;
89 * The current file we're outputting.
91 static FILE *output_file = NULL;
94 * Buffer that we write things into.
96 #define WRITE_BUFFER_SIZE (128 * 1024)
97 static unsigned char *write_buffer = NULL;
98 static uint64_t cur_buffer_size = 0;
99 static uint64_t cur_pos = 0;
100 static uint64_t file_size = 0;
101 static int new_file = 0;
102 static int gcov_version;
104 static HANDLE mmap_handle = NULL;
108 typedef void (*fn_ptr)();
110 typedef void* dynamic_object_id;
111 // The address of this variable identifies a given dynamic object.
112 static dynamic_object_id current_id;
113 #define CURRENT_ID (¤t_id)
116 dynamic_object_id id;
118 struct fn_node* next;
122 struct fn_node *head, *tail;
126 * A list of functions to write out the data, shared between all dynamic objects.
128 struct fn_list writeout_fn_list;
131 * A list of flush functions that our __gcov_flush() function should call, shared between all dynamic objects.
133 struct fn_list flush_fn_list;
136 * A list of reset functions, shared between all dynamic objects.
138 struct fn_list reset_fn_list;
140 static void fn_list_insert(struct fn_list* list, fn_ptr fn) {
141 struct fn_node* new_node = malloc(sizeof(struct fn_node));
143 new_node->next = NULL;
144 new_node->id = CURRENT_ID;
147 list->head = list->tail = new_node;
149 list->tail->next = new_node;
150 list->tail = new_node;
154 static void fn_list_remove(struct fn_list* list) {
155 struct fn_node* curr = list->head;
156 struct fn_node* prev = NULL;
157 struct fn_node* next = NULL;
162 if (curr->id == CURRENT_ID) {
163 if (curr == list->head) {
167 if (curr == list->tail) {
184 static void resize_write_buffer(uint64_t size) {
185 if (!new_file) return;
187 if (size <= cur_buffer_size) return;
188 size = (size - 1) / WRITE_BUFFER_SIZE + 1;
189 size *= WRITE_BUFFER_SIZE;
190 write_buffer = realloc(write_buffer, size);
191 cur_buffer_size = size;
194 static void write_bytes(const char *s, size_t len) {
195 resize_write_buffer(len);
196 memcpy(&write_buffer[cur_pos], s, len);
200 static void write_32bit_value(uint32_t i) {
201 write_bytes((char*)&i, 4);
204 static void write_64bit_value(uint64_t i) {
205 // GCOV uses a lo-/hi-word format even on big-endian systems.
206 // See also GCOVBuffer::readInt64 in LLVM.
207 uint32_t lo = (uint32_t) i;
208 uint32_t hi = (uint32_t) (i >> 32);
209 write_32bit_value(lo);
210 write_32bit_value(hi);
213 static uint32_t read_32bit_value() {
219 val = *(uint32_t*)&write_buffer[cur_pos];
224 static uint64_t read_64bit_value() {
225 // GCOV uses a lo-/hi-word format even on big-endian systems.
226 // See also GCOVBuffer::readInt64 in LLVM.
227 uint32_t lo = read_32bit_value();
228 uint32_t hi = read_32bit_value();
229 return ((uint64_t)hi << 32) | ((uint64_t)lo);
232 static char *mangle_filename(const char *orig_filename) {
236 const char *prefix = lprofGetPathPrefix(&prefix_strip, &prefix_len);
239 return strdup(orig_filename);
241 new_filename = malloc(prefix_len + 1 + strlen(orig_filename) + 1);
242 lprofApplyPathPrefix(new_filename, orig_filename, prefix, prefix_len,
248 static int map_file() {
249 fseek(output_file, 0L, SEEK_END);
250 file_size = ftell(output_file);
252 /* A size of 0 means the file has been created just now (possibly by another
253 * process in lock-after-open race condition). No need to mmap. */
260 mmap_fd = INVALID_HANDLE_VALUE;
262 mmap_fd = (HANDLE)_get_osfhandle(fd);
264 mmap_handle = CreateFileMapping(mmap_fd, NULL, PAGE_READWRITE, DWORD_HI(file_size), DWORD_LO(file_size), NULL);
265 if (mmap_handle == NULL) {
266 fprintf(stderr, "profiling: %s: cannot create file mapping: %lu\n",
267 filename, GetLastError());
271 write_buffer = MapViewOfFile(mmap_handle, FILE_MAP_WRITE, 0, 0, file_size);
272 if (write_buffer == NULL) {
273 fprintf(stderr, "profiling: %s: cannot map: %lu\n", filename,
275 CloseHandle(mmap_handle);
279 write_buffer = mmap(0, file_size, PROT_READ | PROT_WRITE,
280 MAP_FILE | MAP_SHARED, fd, 0);
281 if (write_buffer == (void *)-1) {
283 fprintf(stderr, "profiling: %s: cannot map: %s\n", filename,
292 static void unmap_file() {
294 if (!FlushViewOfFile(write_buffer, file_size)) {
295 fprintf(stderr, "profiling: %s: cannot flush mapped view: %lu\n", filename,
299 if (!UnmapViewOfFile(write_buffer)) {
300 fprintf(stderr, "profiling: %s: cannot unmap mapped view: %lu\n", filename,
304 if (!CloseHandle(mmap_handle)) {
305 fprintf(stderr, "profiling: %s: cannot close file mapping handle: %lu\n",
306 filename, GetLastError());
311 if (msync(write_buffer, file_size, MS_SYNC) == -1) {
313 fprintf(stderr, "profiling: %s: cannot msync: %s\n", filename,
317 /* We explicitly ignore errors from unmapping because at this point the data
318 * is written and we don't care.
320 (void)munmap(write_buffer, file_size);
328 * --- LLVM line counter API ---
331 /* A file in this case is a translation unit. Each .o file built with line
332 * profiling enabled will emit to a different file. Only one file may be
335 COMPILER_RT_VISIBILITY
336 void llvm_gcda_start_file(const char *orig_filename, uint32_t version,
338 const char *mode = "r+b";
339 filename = mangle_filename(orig_filename);
341 /* Try just opening the file. */
342 fd = open(filename, O_RDWR | O_BINARY);
345 /* Try creating the file. */
346 fd = open(filename, O_RDWR | O_CREAT | O_EXCL | O_BINARY, 0644);
350 /* Try creating the directories first then opening the file. */
351 __llvm_profile_recursive_mkdir(filename);
352 fd = open(filename, O_RDWR | O_CREAT | O_EXCL | O_BINARY, 0644);
356 /* Another process may have created the file just now.
357 * Try opening it without O_CREAT and O_EXCL. */
358 fd = open(filename, O_RDWR | O_BINARY);
360 /* Bah! It's hopeless. */
362 fprintf(stderr, "profiling: %s: cannot open: %s\n", filename,
370 /* Try to flock the file to serialize concurrent processes writing out to the
371 * same GCDA. This can fail if the filesystem doesn't support it, but in that
372 * case we'll just carry on with the old racy behaviour and hope for the best.
375 output_file = fdopen(fd, mode);
377 /* Initialize the write buffer. */
383 if (map_file() == -1) {
384 /* The file has been created just now (file_size == 0) or mmap failed
385 * unexpectedly. In the latter case, try to recover by clobbering. */
388 resize_write_buffer(WRITE_BUFFER_SIZE);
389 memset(write_buffer, 0, WRITE_BUFFER_SIZE);
392 /* gcda file, version, stamp checksum. */
394 uint8_t c3 = version >> 24;
395 uint8_t c2 = (version >> 16) & 255;
396 uint8_t c1 = (version >> 8) & 255;
397 gcov_version = c3 >= 'A' ? (c3 - 'A') * 100 + (c2 - '0') * 10 + c1 - '0'
398 : (c3 - '0') * 10 + c1 - '0';
400 write_32bit_value(GCOV_DATA_MAGIC);
401 write_32bit_value(version);
402 write_32bit_value(checksum);
404 #ifdef DEBUG_GCDAPROFILING
405 fprintf(stderr, "llvmgcda: [%s]\n", orig_filename);
409 /* Given an array of pointers to counters (counters), increment the n-th one,
410 * where we're also given a pointer to n (predecessor).
412 COMPILER_RT_VISIBILITY
413 void llvm_gcda_increment_indirect_counter(uint32_t *predecessor,
414 uint64_t **counters) {
419 if (pred == 0xffffffff)
421 counter = counters[pred];
423 /* Don't crash if the pred# is out of sync. This can happen due to threads,
424 or because of a TODO in GCOVProfiling.cpp buildEdgeLookupTable(). */
427 #ifdef DEBUG_GCDAPROFILING
430 "llvmgcda: increment_indirect_counter counters=%08llx, pred=%u\n",
431 *counter, *predecessor);
435 COMPILER_RT_VISIBILITY
436 void llvm_gcda_emit_function(uint32_t ident, uint32_t func_checksum,
437 uint32_t cfg_checksum) {
439 int use_extra_checksum = gcov_version >= 47;
441 if (use_extra_checksum)
443 #ifdef DEBUG_GCDAPROFILING
444 fprintf(stderr, "llvmgcda: function id=0x%08x\n", ident);
446 if (!output_file) return;
449 write_32bit_value(GCOV_TAG_FUNCTION);
450 write_32bit_value(len);
451 write_32bit_value(ident);
452 write_32bit_value(func_checksum);
453 if (use_extra_checksum)
454 write_32bit_value(cfg_checksum);
457 COMPILER_RT_VISIBILITY
458 void llvm_gcda_emit_arcs(uint32_t num_counters, uint64_t *counters) {
460 uint64_t *old_ctrs = NULL;
462 uint64_t save_cur_pos = cur_pos;
464 if (!output_file) return;
466 val = read_32bit_value();
468 if (val != (uint32_t)-1) {
469 /* There are counters present in the file. Merge them. */
470 if (val != GCOV_TAG_COUNTER_ARCS) {
471 fprintf(stderr, "profiling: %s: cannot merge previous GCDA file: "
472 "corrupt arc tag (0x%08x)\n",
477 val = read_32bit_value();
478 if (val == (uint32_t)-1 || val / 2 != num_counters) {
479 fprintf(stderr, "profiling: %s: cannot merge previous GCDA file: "
480 "mismatched number of counters (%d)\n",
485 old_ctrs = malloc(sizeof(uint64_t) * num_counters);
486 for (i = 0; i < num_counters; ++i)
487 old_ctrs[i] = read_64bit_value();
490 cur_pos = save_cur_pos;
492 /* Counter #1 (arcs) tag */
493 write_32bit_value(GCOV_TAG_COUNTER_ARCS);
494 write_32bit_value(num_counters * 2);
495 for (i = 0; i < num_counters; ++i) {
496 counters[i] += (old_ctrs ? old_ctrs[i] : 0);
497 write_64bit_value(counters[i]);
502 #ifdef DEBUG_GCDAPROFILING
503 fprintf(stderr, "llvmgcda: %u arcs\n", num_counters);
504 for (i = 0; i < num_counters; ++i)
505 fprintf(stderr, "llvmgcda: %llu\n", (unsigned long long)counters[i]);
509 COMPILER_RT_VISIBILITY
510 void llvm_gcda_summary_info() {
512 static uint32_t run_counted = 0; // We only want to increase the run count once.
514 uint64_t save_cur_pos = cur_pos;
516 if (!output_file) return;
518 val = read_32bit_value();
520 if (val != (uint32_t)-1) {
521 /* There are counters present in the file. Merge them. */
522 if (val != (gcov_version >= 90 ? GCOV_TAG_OBJECT_SUMMARY
523 : GCOV_TAG_PROGRAM_SUMMARY)) {
525 "profiling: %s: cannot merge previous run count: "
526 "corrupt object tag (0x%08x)\n",
531 val = read_32bit_value(); /* length */
533 if (gcov_version < 90) {
536 prev_runs = read_32bit_value();
538 prev_runs = read_32bit_value();
541 for (uint32_t i = gcov_version < 90 ? 3 : 2; i < val; ++i)
543 /* Add previous run count to new counter, if not already counted before. */
544 runs = run_counted ? prev_runs : prev_runs + 1;
547 cur_pos = save_cur_pos;
549 if (gcov_version >= 90) {
550 write_32bit_value(GCOV_TAG_OBJECT_SUMMARY);
551 write_32bit_value(2);
552 write_32bit_value(runs);
553 write_32bit_value(0); // sum_max
555 // Before gcov 4.8 (r190952), GCOV_TAG_SUMMARY_LENGTH was 9. r190952 set
556 // GCOV_TAG_SUMMARY_LENGTH to 22. We simply use the smallest length which
557 // can make gcov read "Runs:".
558 write_32bit_value(GCOV_TAG_PROGRAM_SUMMARY);
559 write_32bit_value(3);
560 write_32bit_value(0);
561 write_32bit_value(0);
562 write_32bit_value(runs);
567 #ifdef DEBUG_GCDAPROFILING
568 fprintf(stderr, "llvmgcda: %u runs\n", runs);
572 COMPILER_RT_VISIBILITY
573 void llvm_gcda_end_file() {
574 /* Write out EOF record. */
576 write_bytes("\0\0\0\0\0\0\0\0", 8);
579 fwrite(write_buffer, cur_pos, 1, output_file);
593 #ifdef DEBUG_GCDAPROFILING
594 fprintf(stderr, "llvmgcda: -----\n");
598 COMPILER_RT_VISIBILITY
599 void llvm_register_writeout_function(fn_ptr fn) {
600 fn_list_insert(&writeout_fn_list, fn);
603 COMPILER_RT_VISIBILITY
604 void llvm_writeout_files(void) {
605 struct fn_node *curr = writeout_fn_list.head;
608 if (curr->id == CURRENT_ID) {
616 // __attribute__((destructor)) and destructors whose priorities are greater than
617 // 100 run before this function and can thus be tracked. The priority is
618 // compatible with GCC 7 onwards.
620 #pragma GCC diagnostic ignored "-Wprio-ctor-dtor"
622 __attribute__((destructor(100)))
624 static void llvm_writeout_and_clear(void) {
625 llvm_writeout_files();
626 fn_list_remove(&writeout_fn_list);
629 COMPILER_RT_VISIBILITY
630 void llvm_register_flush_function(fn_ptr fn) {
631 fn_list_insert(&flush_fn_list, fn);
634 void __gcov_flush() {
635 struct fn_node* curr = flush_fn_list.head;
643 COMPILER_RT_VISIBILITY
644 void llvm_delete_flush_function_list(void) {
645 fn_list_remove(&flush_fn_list);
648 COMPILER_RT_VISIBILITY
649 void llvm_register_reset_function(fn_ptr fn) {
650 fn_list_insert(&reset_fn_list, fn);
653 COMPILER_RT_VISIBILITY
654 void llvm_delete_reset_function_list(void) { fn_list_remove(&reset_fn_list); }
656 COMPILER_RT_VISIBILITY
657 void llvm_reset_counters(void) {
658 struct fn_node *curr = reset_fn_list.head;
661 if (curr->id == CURRENT_ID) {
669 COMPILER_RT_VISIBILITY
670 pid_t __gcov_fork() {
671 pid_t parent_pid = getpid();
675 pid_t child_pid = getpid();
676 if (child_pid != parent_pid) {
677 // The pid changed so we've a fork (one could have its own fork function)
678 // Just reset the counters for this child process
680 llvm_reset_counters();
687 COMPILER_RT_VISIBILITY
688 void llvm_gcov_init(fn_ptr wfn, fn_ptr ffn, fn_ptr rfn) {
689 static int atexit_ran = 0;
692 llvm_register_writeout_function(wfn);
695 llvm_register_flush_function(ffn);
698 llvm_register_reset_function(rfn);
700 if (atexit_ran == 0) {
703 /* Make sure we write out the data and delete the data structures. */
704 atexit(llvm_delete_reset_function_list);
705 atexit(llvm_delete_flush_function_list);
707 atexit(llvm_writeout_and_clear);