1 //===- MemorySanitizer.cpp - detector of uninitialized reads --------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
11 /// This file is a part of MemorySanitizer, a detector of uninitialized
14 /// The algorithm of the tool is similar to Memcheck
15 /// (http://goo.gl/QKbem). We associate a few shadow bits with every
16 /// byte of the application memory, poison the shadow of the malloc-ed
17 /// or alloca-ed memory, load the shadow bits on every memory read,
18 /// propagate the shadow bits through some of the arithmetic
19 /// instruction (including MOV), store the shadow bits on every memory
20 /// write, report a bug on some other instructions (e.g. JMP) if the
21 /// associated shadow is poisoned.
23 /// But there are differences too. The first and the major one:
24 /// compiler instrumentation instead of binary instrumentation. This
25 /// gives us much better register allocation, possible compiler
26 /// optimizations and a fast start-up. But this brings the major issue
27 /// as well: msan needs to see all program events, including system
28 /// calls and reads/writes in system libraries, so we either need to
29 /// compile *everything* with msan or use a binary translation
30 /// component (e.g. DynamoRIO) to instrument pre-built libraries.
31 /// Another difference from Memcheck is that we use 8 shadow bits per
32 /// byte of application memory and use a direct shadow mapping. This
33 /// greatly simplifies the instrumentation code and avoids races on
34 /// shadow updates (Memcheck is single-threaded so races are not a
35 /// concern there. Memcheck uses 2 shadow bits per byte with a slow
36 /// path storage that uses 8 bits per byte).
38 /// The default value of shadow is 0, which means "clean" (not poisoned).
40 /// Every module initializer should call __msan_init to ensure that the
41 /// shadow memory is ready. On error, __msan_warning is called. Since
42 /// parameters and return values may be passed via registers, we have a
43 /// specialized thread-local shadow for return values
44 /// (__msan_retval_tls) and parameters (__msan_param_tls).
48 /// MemorySanitizer can track origins (allocation points) of all uninitialized
49 /// values. This behavior is controlled with a flag (msan-track-origins) and is
50 /// disabled by default.
52 /// Origins are 4-byte values created and interpreted by the runtime library.
53 /// They are stored in a second shadow mapping, one 4-byte value for 4 bytes
54 /// of application memory. Propagation of origins is basically a bunch of
55 /// "select" instructions that pick the origin of a dirty argument, if an
56 /// instruction has one.
58 /// Every 4 aligned, consecutive bytes of application memory have one origin
59 /// value associated with them. If these bytes contain uninitialized data
60 /// coming from 2 different allocations, the last store wins. Because of this,
61 /// MemorySanitizer reports can show unrelated origins, but this is unlikely in
64 /// Origins are meaningless for fully initialized values, so MemorySanitizer
65 /// avoids storing origin to memory when a fully initialized value is stored.
66 /// This way it avoids needless overwritting origin of the 4-byte region on
67 /// a short (i.e. 1 byte) clean store, and it is also good for performance.
71 /// Ideally, every atomic store of application value should update the
72 /// corresponding shadow location in an atomic way. Unfortunately, atomic store
73 /// of two disjoint locations can not be done without severe slowdown.
75 /// Therefore, we implement an approximation that may err on the safe side.
76 /// In this implementation, every atomically accessed location in the program
77 /// may only change from (partially) uninitialized to fully initialized, but
78 /// not the other way around. We load the shadow _after_ the application load,
79 /// and we store the shadow _before_ the app store. Also, we always store clean
80 /// shadow (if the application store is atomic). This way, if the store-load
81 /// pair constitutes a happens-before arc, shadow store and load are correctly
82 /// ordered such that the load will get either the value that was stored, or
83 /// some later value (which is always clean).
85 /// This does not work very well with Compare-And-Swap (CAS) and
86 /// Read-Modify-Write (RMW) operations. To follow the above logic, CAS and RMW
87 /// must store the new shadow before the app operation, and load the shadow
88 /// after the app operation. Computers don't work this way. Current
89 /// implementation ignores the load aspect of CAS/RMW, always returning a clean
90 /// value. It implements the store part as a simple atomic store by storing a
93 //===----------------------------------------------------------------------===//
95 #include "llvm/ADT/APInt.h"
96 #include "llvm/ADT/ArrayRef.h"
97 #include "llvm/ADT/DepthFirstIterator.h"
98 #include "llvm/ADT/SmallString.h"
99 #include "llvm/ADT/SmallVector.h"
100 #include "llvm/ADT/StringExtras.h"
101 #include "llvm/ADT/StringRef.h"
102 #include "llvm/ADT/Triple.h"
103 #include "llvm/Analysis/TargetLibraryInfo.h"
104 #include "llvm/Transforms/Utils/Local.h"
105 #include "llvm/IR/Argument.h"
106 #include "llvm/IR/Attributes.h"
107 #include "llvm/IR/BasicBlock.h"
108 #include "llvm/IR/CallSite.h"
109 #include "llvm/IR/CallingConv.h"
110 #include "llvm/IR/Constant.h"
111 #include "llvm/IR/Constants.h"
112 #include "llvm/IR/DataLayout.h"
113 #include "llvm/IR/DerivedTypes.h"
114 #include "llvm/IR/Function.h"
115 #include "llvm/IR/GlobalValue.h"
116 #include "llvm/IR/GlobalVariable.h"
117 #include "llvm/IR/IRBuilder.h"
118 #include "llvm/IR/InlineAsm.h"
119 #include "llvm/IR/InstVisitor.h"
120 #include "llvm/IR/InstrTypes.h"
121 #include "llvm/IR/Instruction.h"
122 #include "llvm/IR/Instructions.h"
123 #include "llvm/IR/IntrinsicInst.h"
124 #include "llvm/IR/Intrinsics.h"
125 #include "llvm/IR/LLVMContext.h"
126 #include "llvm/IR/MDBuilder.h"
127 #include "llvm/IR/Module.h"
128 #include "llvm/IR/Type.h"
129 #include "llvm/IR/Value.h"
130 #include "llvm/IR/ValueMap.h"
131 #include "llvm/Pass.h"
132 #include "llvm/Support/AtomicOrdering.h"
133 #include "llvm/Support/Casting.h"
134 #include "llvm/Support/CommandLine.h"
135 #include "llvm/Support/Compiler.h"
136 #include "llvm/Support/Debug.h"
137 #include "llvm/Support/ErrorHandling.h"
138 #include "llvm/Support/MathExtras.h"
139 #include "llvm/Support/raw_ostream.h"
140 #include "llvm/Transforms/Instrumentation.h"
141 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
142 #include "llvm/Transforms/Utils/ModuleUtils.h"
151 using namespace llvm;
153 #define DEBUG_TYPE "msan"
155 static const unsigned kOriginSize = 4;
156 static const unsigned kMinOriginAlignment = 4;
157 static const unsigned kShadowTLSAlignment = 8;
159 // These constants must be kept in sync with the ones in msan.h.
160 static const unsigned kParamTLSSize = 800;
161 static const unsigned kRetvalTLSSize = 800;
163 // Accesses sizes are powers of two: 1, 2, 4, 8.
164 static const size_t kNumberOfAccessSizes = 4;
166 /// Track origins of uninitialized values.
168 /// Adds a section to MemorySanitizer report that points to the allocation
169 /// (stack or heap) the uninitialized bits came from originally.
170 static cl::opt<int> ClTrackOrigins("msan-track-origins",
171 cl::desc("Track origins (allocation sites) of poisoned memory"),
172 cl::Hidden, cl::init(0));
174 static cl::opt<bool> ClKeepGoing("msan-keep-going",
175 cl::desc("keep going after reporting a UMR"),
176 cl::Hidden, cl::init(false));
178 static cl::opt<bool> ClPoisonStack("msan-poison-stack",
179 cl::desc("poison uninitialized stack variables"),
180 cl::Hidden, cl::init(true));
182 static cl::opt<bool> ClPoisonStackWithCall("msan-poison-stack-with-call",
183 cl::desc("poison uninitialized stack variables with a call"),
184 cl::Hidden, cl::init(false));
186 static cl::opt<int> ClPoisonStackPattern("msan-poison-stack-pattern",
187 cl::desc("poison uninitialized stack variables with the given pattern"),
188 cl::Hidden, cl::init(0xff));
190 static cl::opt<bool> ClPoisonUndef("msan-poison-undef",
191 cl::desc("poison undef temps"),
192 cl::Hidden, cl::init(true));
194 static cl::opt<bool> ClHandleICmp("msan-handle-icmp",
195 cl::desc("propagate shadow through ICmpEQ and ICmpNE"),
196 cl::Hidden, cl::init(true));
198 static cl::opt<bool> ClHandleICmpExact("msan-handle-icmp-exact",
199 cl::desc("exact handling of relational integer ICmp"),
200 cl::Hidden, cl::init(false));
202 // When compiling the Linux kernel, we sometimes see false positives related to
203 // MSan being unable to understand that inline assembly calls may initialize
205 // This flag makes the compiler conservatively unpoison every memory location
206 // passed into an assembly call. Note that this may cause false positives.
207 // Because it's impossible to figure out the array sizes, we can only unpoison
208 // the first sizeof(type) bytes for each type* pointer.
209 static cl::opt<bool> ClHandleAsmConservative(
210 "msan-handle-asm-conservative",
211 cl::desc("conservative handling of inline assembly"), cl::Hidden,
214 // This flag controls whether we check the shadow of the address
215 // operand of load or store. Such bugs are very rare, since load from
216 // a garbage address typically results in SEGV, but still happen
217 // (e.g. only lower bits of address are garbage, or the access happens
218 // early at program startup where malloc-ed memory is more likely to
219 // be zeroed. As of 2012-08-28 this flag adds 20% slowdown.
220 static cl::opt<bool> ClCheckAccessAddress("msan-check-access-address",
221 cl::desc("report accesses through a pointer which has poisoned shadow"),
222 cl::Hidden, cl::init(true));
224 static cl::opt<bool> ClDumpStrictInstructions("msan-dump-strict-instructions",
225 cl::desc("print out instructions with default strict semantics"),
226 cl::Hidden, cl::init(false));
228 static cl::opt<int> ClInstrumentationWithCallThreshold(
229 "msan-instrumentation-with-call-threshold",
231 "If the function being instrumented requires more than "
232 "this number of checks and origin stores, use callbacks instead of "
233 "inline checks (-1 means never use callbacks)."),
234 cl::Hidden, cl::init(3500));
236 // This is an experiment to enable handling of cases where shadow is a non-zero
237 // compile-time constant. For some unexplainable reason they were silently
238 // ignored in the instrumentation.
239 static cl::opt<bool> ClCheckConstantShadow("msan-check-constant-shadow",
240 cl::desc("Insert checks for constant shadow values"),
241 cl::Hidden, cl::init(false));
243 // This is off by default because of a bug in gold:
244 // https://sourceware.org/bugzilla/show_bug.cgi?id=19002
245 static cl::opt<bool> ClWithComdat("msan-with-comdat",
246 cl::desc("Place MSan constructors in comdat sections"),
247 cl::Hidden, cl::init(false));
249 // These options allow to specify custom memory map parameters
250 // See MemoryMapParams for details.
251 static cl::opt<unsigned long long> ClAndMask("msan-and-mask",
252 cl::desc("Define custom MSan AndMask"),
253 cl::Hidden, cl::init(0));
255 static cl::opt<unsigned long long> ClXorMask("msan-xor-mask",
256 cl::desc("Define custom MSan XorMask"),
257 cl::Hidden, cl::init(0));
259 static cl::opt<unsigned long long> ClShadowBase("msan-shadow-base",
260 cl::desc("Define custom MSan ShadowBase"),
261 cl::Hidden, cl::init(0));
263 static cl::opt<unsigned long long> ClOriginBase("msan-origin-base",
264 cl::desc("Define custom MSan OriginBase"),
265 cl::Hidden, cl::init(0));
267 static const char *const kMsanModuleCtorName = "msan.module_ctor";
268 static const char *const kMsanInitName = "__msan_init";
272 // Memory map parameters used in application-to-shadow address calculation.
273 // Offset = (Addr & ~AndMask) ^ XorMask
274 // Shadow = ShadowBase + Offset
275 // Origin = OriginBase + Offset
276 struct MemoryMapParams {
283 struct PlatformMemoryMapParams {
284 const MemoryMapParams *bits32;
285 const MemoryMapParams *bits64;
288 } // end anonymous namespace
291 static const MemoryMapParams Linux_I386_MemoryMapParams = {
292 0x000080000000, // AndMask
293 0, // XorMask (not used)
294 0, // ShadowBase (not used)
295 0x000040000000, // OriginBase
299 static const MemoryMapParams Linux_X86_64_MemoryMapParams = {
300 #ifdef MSAN_LINUX_X86_64_OLD_MAPPING
301 0x400000000000, // AndMask
302 0, // XorMask (not used)
303 0, // ShadowBase (not used)
304 0x200000000000, // OriginBase
306 0, // AndMask (not used)
307 0x500000000000, // XorMask
308 0, // ShadowBase (not used)
309 0x100000000000, // OriginBase
314 static const MemoryMapParams Linux_MIPS64_MemoryMapParams = {
315 0, // AndMask (not used)
316 0x008000000000, // XorMask
317 0, // ShadowBase (not used)
318 0x002000000000, // OriginBase
322 static const MemoryMapParams Linux_PowerPC64_MemoryMapParams = {
323 0xE00000000000, // AndMask
324 0x100000000000, // XorMask
325 0x080000000000, // ShadowBase
326 0x1C0000000000, // OriginBase
330 static const MemoryMapParams Linux_AArch64_MemoryMapParams = {
331 0, // AndMask (not used)
332 0x06000000000, // XorMask
333 0, // ShadowBase (not used)
334 0x01000000000, // OriginBase
338 static const MemoryMapParams FreeBSD_I386_MemoryMapParams = {
339 0x000180000000, // AndMask
340 0x000040000000, // XorMask
341 0x000020000000, // ShadowBase
342 0x000700000000, // OriginBase
346 static const MemoryMapParams FreeBSD_X86_64_MemoryMapParams = {
347 0xc00000000000, // AndMask
348 0x200000000000, // XorMask
349 0x100000000000, // ShadowBase
350 0x380000000000, // OriginBase
354 static const MemoryMapParams NetBSD_X86_64_MemoryMapParams = {
356 0x500000000000, // XorMask
358 0x100000000000, // OriginBase
361 static const PlatformMemoryMapParams Linux_X86_MemoryMapParams = {
362 &Linux_I386_MemoryMapParams,
363 &Linux_X86_64_MemoryMapParams,
366 static const PlatformMemoryMapParams Linux_MIPS_MemoryMapParams = {
368 &Linux_MIPS64_MemoryMapParams,
371 static const PlatformMemoryMapParams Linux_PowerPC_MemoryMapParams = {
373 &Linux_PowerPC64_MemoryMapParams,
376 static const PlatformMemoryMapParams Linux_ARM_MemoryMapParams = {
378 &Linux_AArch64_MemoryMapParams,
381 static const PlatformMemoryMapParams FreeBSD_X86_MemoryMapParams = {
382 &FreeBSD_I386_MemoryMapParams,
383 &FreeBSD_X86_64_MemoryMapParams,
386 static const PlatformMemoryMapParams NetBSD_X86_MemoryMapParams = {
388 &NetBSD_X86_64_MemoryMapParams,
393 /// An instrumentation pass implementing detection of uninitialized
396 /// MemorySanitizer: instrument the code in module to find
397 /// uninitialized reads.
398 class MemorySanitizer : public FunctionPass {
400 // Pass identification, replacement for typeid.
403 MemorySanitizer(int TrackOrigins = 0, bool Recover = false)
405 TrackOrigins(std::max(TrackOrigins, (int)ClTrackOrigins)),
406 Recover(Recover || ClKeepGoing) {}
408 StringRef getPassName() const override { return "MemorySanitizer"; }
410 void getAnalysisUsage(AnalysisUsage &AU) const override {
411 AU.addRequired<TargetLibraryInfoWrapperPass>();
414 bool runOnFunction(Function &F) override;
415 bool doInitialization(Module &M) override;
418 friend struct MemorySanitizerVisitor;
419 friend struct VarArgAMD64Helper;
420 friend struct VarArgMIPS64Helper;
421 friend struct VarArgAArch64Helper;
422 friend struct VarArgPowerPC64Helper;
424 void initializeCallbacks(Module &M);
425 void createUserspaceApi(Module &M);
427 /// Track origins (allocation points) of uninitialized values.
435 /// Thread-local shadow storage for function parameters.
436 GlobalVariable *ParamTLS;
438 /// Thread-local origin storage for function parameters.
439 GlobalVariable *ParamOriginTLS;
441 /// Thread-local shadow storage for function return value.
442 GlobalVariable *RetvalTLS;
444 /// Thread-local origin storage for function return value.
445 GlobalVariable *RetvalOriginTLS;
447 /// Thread-local shadow storage for in-register va_arg function
448 /// parameters (x86_64-specific).
449 GlobalVariable *VAArgTLS;
451 /// Thread-local shadow storage for va_arg overflow area
452 /// (x86_64-specific).
453 GlobalVariable *VAArgOverflowSizeTLS;
455 /// Thread-local space used to pass origin value to the UMR reporting
457 GlobalVariable *OriginTLS;
459 /// Are the instrumentation callbacks set up?
460 bool CallbacksInitialized = false;
462 /// The run-time callback to print a warning.
465 // These arrays are indexed by log2(AccessSize).
466 Value *MaybeWarningFn[kNumberOfAccessSizes];
467 Value *MaybeStoreOriginFn[kNumberOfAccessSizes];
469 /// Run-time helper that generates a new origin value for a stack
471 Value *MsanSetAllocaOrigin4Fn;
473 /// Run-time helper that poisons stack on function entry.
474 Value *MsanPoisonStackFn;
476 /// Run-time helper that records a store (or any event) of an
477 /// uninitialized value and returns an updated origin id encoding this info.
478 Value *MsanChainOriginFn;
480 /// MSan runtime replacements for memmove, memcpy and memset.
481 Value *MemmoveFn, *MemcpyFn, *MemsetFn;
483 /// Memory map parameters used in application-to-shadow calculation.
484 const MemoryMapParams *MapParams;
486 /// Custom memory map parameters used when -msan-shadow-base or
487 // -msan-origin-base is provided.
488 MemoryMapParams CustomMapParams;
490 MDNode *ColdCallWeights;
492 /// Branch weights for origin store.
493 MDNode *OriginStoreWeights;
495 /// An empty volatile inline asm that prevents callback merge.
498 Function *MsanCtorFunction;
501 } // end anonymous namespace
503 char MemorySanitizer::ID = 0;
505 INITIALIZE_PASS_BEGIN(
506 MemorySanitizer, "msan",
507 "MemorySanitizer: detects uninitialized reads.", false, false)
508 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
510 MemorySanitizer, "msan",
511 "MemorySanitizer: detects uninitialized reads.", false, false)
513 FunctionPass *llvm::createMemorySanitizerPass(int TrackOrigins, bool Recover) {
514 return new MemorySanitizer(TrackOrigins, Recover);
517 /// Create a non-const global initialized with the given string.
519 /// Creates a writable global for Str so that we can pass it to the
520 /// run-time lib. Runtime uses first 4 bytes of the string to store the
521 /// frame ID, so the string needs to be mutable.
522 static GlobalVariable *createPrivateNonConstGlobalForString(Module &M,
524 Constant *StrConst = ConstantDataArray::getString(M.getContext(), Str);
525 return new GlobalVariable(M, StrConst->getType(), /*isConstant=*/false,
526 GlobalValue::PrivateLinkage, StrConst, "");
529 /// Insert declarations for userspace-specific functions and globals.
530 void MemorySanitizer::createUserspaceApi(Module &M) {
532 // Create the callback.
533 // FIXME: this function should have "Cold" calling conv,
534 // which is not yet implemented.
535 StringRef WarningFnName = Recover ? "__msan_warning"
536 : "__msan_warning_noreturn";
537 WarningFn = M.getOrInsertFunction(WarningFnName, IRB.getVoidTy());
539 // Create the global TLS variables.
540 RetvalTLS = new GlobalVariable(
541 M, ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8), false,
542 GlobalVariable::ExternalLinkage, nullptr, "__msan_retval_tls", nullptr,
543 GlobalVariable::InitialExecTLSModel);
545 RetvalOriginTLS = new GlobalVariable(
546 M, OriginTy, false, GlobalVariable::ExternalLinkage, nullptr,
547 "__msan_retval_origin_tls", nullptr, GlobalVariable::InitialExecTLSModel);
549 ParamTLS = new GlobalVariable(
550 M, ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), false,
551 GlobalVariable::ExternalLinkage, nullptr, "__msan_param_tls", nullptr,
552 GlobalVariable::InitialExecTLSModel);
554 ParamOriginTLS = new GlobalVariable(
555 M, ArrayType::get(OriginTy, kParamTLSSize / 4), false,
556 GlobalVariable::ExternalLinkage, nullptr, "__msan_param_origin_tls",
557 nullptr, GlobalVariable::InitialExecTLSModel);
559 VAArgTLS = new GlobalVariable(
560 M, ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), false,
561 GlobalVariable::ExternalLinkage, nullptr, "__msan_va_arg_tls", nullptr,
562 GlobalVariable::InitialExecTLSModel);
563 VAArgOverflowSizeTLS = new GlobalVariable(
564 M, IRB.getInt64Ty(), false, GlobalVariable::ExternalLinkage, nullptr,
565 "__msan_va_arg_overflow_size_tls", nullptr,
566 GlobalVariable::InitialExecTLSModel);
567 OriginTLS = new GlobalVariable(
568 M, IRB.getInt32Ty(), false, GlobalVariable::ExternalLinkage, nullptr,
569 "__msan_origin_tls", nullptr, GlobalVariable::InitialExecTLSModel);
571 for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
573 unsigned AccessSize = 1 << AccessSizeIndex;
574 std::string FunctionName = "__msan_maybe_warning_" + itostr(AccessSize);
575 MaybeWarningFn[AccessSizeIndex] = M.getOrInsertFunction(
576 FunctionName, IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8),
579 FunctionName = "__msan_maybe_store_origin_" + itostr(AccessSize);
580 MaybeStoreOriginFn[AccessSizeIndex] = M.getOrInsertFunction(
581 FunctionName, IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8),
582 IRB.getInt8PtrTy(), IRB.getInt32Ty());
585 MsanSetAllocaOrigin4Fn = M.getOrInsertFunction(
586 "__msan_set_alloca_origin4", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy,
587 IRB.getInt8PtrTy(), IntptrTy);
589 M.getOrInsertFunction("__msan_poison_stack", IRB.getVoidTy(),
590 IRB.getInt8PtrTy(), IntptrTy);
593 /// Insert extern declaration of runtime-provided functions and globals.
594 void MemorySanitizer::initializeCallbacks(Module &M) {
595 // Only do this once.
596 if (CallbacksInitialized)
600 // Initialize callbacks that are common for kernel and userspace
602 MsanChainOriginFn = M.getOrInsertFunction(
603 "__msan_chain_origin", IRB.getInt32Ty(), IRB.getInt32Ty());
604 MemmoveFn = M.getOrInsertFunction(
605 "__msan_memmove", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
606 IRB.getInt8PtrTy(), IntptrTy);
607 MemcpyFn = M.getOrInsertFunction(
608 "__msan_memcpy", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
610 MemsetFn = M.getOrInsertFunction(
611 "__msan_memset", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt32Ty(),
613 // We insert an empty inline asm after __msan_report* to avoid callback merge.
614 EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false),
615 StringRef(""), StringRef(""),
616 /*hasSideEffects=*/true);
618 createUserspaceApi(M);
619 CallbacksInitialized = true;
622 /// Module-level initialization.
624 /// inserts a call to __msan_init to the module's constructor list.
625 bool MemorySanitizer::doInitialization(Module &M) {
626 auto &DL = M.getDataLayout();
628 bool ShadowPassed = ClShadowBase.getNumOccurrences() > 0;
629 bool OriginPassed = ClOriginBase.getNumOccurrences() > 0;
630 // Check the overrides first
631 if (ShadowPassed || OriginPassed) {
632 CustomMapParams.AndMask = ClAndMask;
633 CustomMapParams.XorMask = ClXorMask;
634 CustomMapParams.ShadowBase = ClShadowBase;
635 CustomMapParams.OriginBase = ClOriginBase;
636 MapParams = &CustomMapParams;
638 Triple TargetTriple(M.getTargetTriple());
639 switch (TargetTriple.getOS()) {
640 case Triple::FreeBSD:
641 switch (TargetTriple.getArch()) {
643 MapParams = FreeBSD_X86_MemoryMapParams.bits64;
646 MapParams = FreeBSD_X86_MemoryMapParams.bits32;
649 report_fatal_error("unsupported architecture");
653 switch (TargetTriple.getArch()) {
655 MapParams = NetBSD_X86_MemoryMapParams.bits64;
658 report_fatal_error("unsupported architecture");
662 switch (TargetTriple.getArch()) {
664 MapParams = Linux_X86_MemoryMapParams.bits64;
667 MapParams = Linux_X86_MemoryMapParams.bits32;
670 case Triple::mips64el:
671 MapParams = Linux_MIPS_MemoryMapParams.bits64;
674 case Triple::ppc64le:
675 MapParams = Linux_PowerPC_MemoryMapParams.bits64;
677 case Triple::aarch64:
678 case Triple::aarch64_be:
679 MapParams = Linux_ARM_MemoryMapParams.bits64;
682 report_fatal_error("unsupported architecture");
686 report_fatal_error("unsupported operating system");
690 C = &(M.getContext());
692 IntptrTy = IRB.getIntPtrTy(DL);
693 OriginTy = IRB.getInt32Ty();
695 ColdCallWeights = MDBuilder(*C).createBranchWeights(1, 1000);
696 OriginStoreWeights = MDBuilder(*C).createBranchWeights(1, 1000);
698 std::tie(MsanCtorFunction, std::ignore) =
699 createSanitizerCtorAndInitFunctions(M, kMsanModuleCtorName, kMsanInitName,
703 Comdat *MsanCtorComdat = M.getOrInsertComdat(kMsanModuleCtorName);
704 MsanCtorFunction->setComdat(MsanCtorComdat);
705 appendToGlobalCtors(M, MsanCtorFunction, 0, MsanCtorFunction);
707 appendToGlobalCtors(M, MsanCtorFunction, 0);
712 new GlobalVariable(M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
713 IRB.getInt32(TrackOrigins), "__msan_track_origins");
716 new GlobalVariable(M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
717 IRB.getInt32(Recover), "__msan_keep_going");
724 /// A helper class that handles instrumentation of VarArg
725 /// functions on a particular platform.
727 /// Implementations are expected to insert the instrumentation
728 /// necessary to propagate argument shadow through VarArg function
729 /// calls. Visit* methods are called during an InstVisitor pass over
730 /// the function, and should avoid creating new basic blocks. A new
731 /// instance of this class is created for each instrumented function.
732 struct VarArgHelper {
733 virtual ~VarArgHelper() = default;
735 /// Visit a CallSite.
736 virtual void visitCallSite(CallSite &CS, IRBuilder<> &IRB) = 0;
738 /// Visit a va_start call.
739 virtual void visitVAStartInst(VAStartInst &I) = 0;
741 /// Visit a va_copy call.
742 virtual void visitVACopyInst(VACopyInst &I) = 0;
744 /// Finalize function instrumentation.
746 /// This method is called after visiting all interesting (see above)
747 /// instructions in a function.
748 virtual void finalizeInstrumentation() = 0;
751 struct MemorySanitizerVisitor;
753 } // end anonymous namespace
755 static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
756 MemorySanitizerVisitor &Visitor);
758 static unsigned TypeSizeToSizeIndex(unsigned TypeSize) {
759 if (TypeSize <= 8) return 0;
760 return Log2_32_Ceil((TypeSize + 7) / 8);
765 /// This class does all the work for a given function. Store and Load
766 /// instructions store and load corresponding shadow and origin
767 /// values. Most instructions propagate shadow from arguments to their
768 /// return values. Certain instructions (most importantly, BranchInst)
769 /// test their argument shadow and print reports (with a runtime call) if it's
771 struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
774 SmallVector<PHINode *, 16> ShadowPHINodes, OriginPHINodes;
775 ValueMap<Value*, Value*> ShadowMap, OriginMap;
776 std::unique_ptr<VarArgHelper> VAHelper;
777 const TargetLibraryInfo *TLI;
778 BasicBlock *ActualFnStart;
780 // The following flags disable parts of MSan instrumentation based on
781 // blacklist contents and command-line options.
783 bool PropagateShadow;
786 bool CheckReturnValue;
788 struct ShadowOriginAndInsertPoint {
791 Instruction *OrigIns;
793 ShadowOriginAndInsertPoint(Value *S, Value *O, Instruction *I)
794 : Shadow(S), Origin(O), OrigIns(I) {}
796 SmallVector<ShadowOriginAndInsertPoint, 16> InstrumentationList;
797 SmallVector<StoreInst *, 16> StoreList;
799 MemorySanitizerVisitor(Function &F, MemorySanitizer &MS)
800 : F(F), MS(MS), VAHelper(CreateVarArgHelper(F, MS, *this)) {
801 bool SanitizeFunction = F.hasFnAttribute(Attribute::SanitizeMemory);
802 InsertChecks = SanitizeFunction;
803 PropagateShadow = SanitizeFunction;
804 PoisonStack = SanitizeFunction && ClPoisonStack;
805 PoisonUndef = SanitizeFunction && ClPoisonUndef;
806 // FIXME: Consider using SpecialCaseList to specify a list of functions that
807 // must always return fully initialized values. For now, we hardcode "main".
808 CheckReturnValue = SanitizeFunction && (F.getName() == "main");
809 TLI = &MS.getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
811 MS.initializeCallbacks(*F.getParent());
812 ActualFnStart = &F.getEntryBlock();
814 LLVM_DEBUG(if (!InsertChecks) dbgs()
815 << "MemorySanitizer is not inserting checks into '"
816 << F.getName() << "'\n");
819 Value *updateOrigin(Value *V, IRBuilder<> &IRB) {
820 if (MS.TrackOrigins <= 1) return V;
821 return IRB.CreateCall(MS.MsanChainOriginFn, V);
824 Value *originToIntptr(IRBuilder<> &IRB, Value *Origin) {
825 const DataLayout &DL = F.getParent()->getDataLayout();
826 unsigned IntptrSize = DL.getTypeStoreSize(MS.IntptrTy);
827 if (IntptrSize == kOriginSize) return Origin;
828 assert(IntptrSize == kOriginSize * 2);
829 Origin = IRB.CreateIntCast(Origin, MS.IntptrTy, /* isSigned */ false);
830 return IRB.CreateOr(Origin, IRB.CreateShl(Origin, kOriginSize * 8));
833 /// Fill memory range with the given origin value.
834 void paintOrigin(IRBuilder<> &IRB, Value *Origin, Value *OriginPtr,
835 unsigned Size, unsigned Alignment) {
836 const DataLayout &DL = F.getParent()->getDataLayout();
837 unsigned IntptrAlignment = DL.getABITypeAlignment(MS.IntptrTy);
838 unsigned IntptrSize = DL.getTypeStoreSize(MS.IntptrTy);
839 assert(IntptrAlignment >= kMinOriginAlignment);
840 assert(IntptrSize >= kOriginSize);
843 unsigned CurrentAlignment = Alignment;
844 if (Alignment >= IntptrAlignment && IntptrSize > kOriginSize) {
845 Value *IntptrOrigin = originToIntptr(IRB, Origin);
846 Value *IntptrOriginPtr =
847 IRB.CreatePointerCast(OriginPtr, PointerType::get(MS.IntptrTy, 0));
848 for (unsigned i = 0; i < Size / IntptrSize; ++i) {
849 Value *Ptr = i ? IRB.CreateConstGEP1_32(MS.IntptrTy, IntptrOriginPtr, i)
851 IRB.CreateAlignedStore(IntptrOrigin, Ptr, CurrentAlignment);
852 Ofs += IntptrSize / kOriginSize;
853 CurrentAlignment = IntptrAlignment;
857 for (unsigned i = Ofs; i < (Size + kOriginSize - 1) / kOriginSize; ++i) {
859 i ? IRB.CreateConstGEP1_32(nullptr, OriginPtr, i) : OriginPtr;
860 IRB.CreateAlignedStore(Origin, GEP, CurrentAlignment);
861 CurrentAlignment = kMinOriginAlignment;
865 void storeOrigin(IRBuilder<> &IRB, Value *Addr, Value *Shadow, Value *Origin,
866 Value *OriginPtr, unsigned Alignment, bool AsCall) {
867 const DataLayout &DL = F.getParent()->getDataLayout();
868 unsigned OriginAlignment = std::max(kMinOriginAlignment, Alignment);
869 unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType());
870 if (Shadow->getType()->isAggregateType()) {
871 paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
874 Value *ConvertedShadow = convertToShadowTyNoVec(Shadow, IRB);
875 Constant *ConstantShadow = dyn_cast_or_null<Constant>(ConvertedShadow);
876 if (ConstantShadow) {
877 if (ClCheckConstantShadow && !ConstantShadow->isZeroValue())
878 paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
883 unsigned TypeSizeInBits =
884 DL.getTypeSizeInBits(ConvertedShadow->getType());
885 unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
886 if (AsCall && SizeIndex < kNumberOfAccessSizes) {
887 Value *Fn = MS.MaybeStoreOriginFn[SizeIndex];
888 Value *ConvertedShadow2 = IRB.CreateZExt(
889 ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
890 IRB.CreateCall(Fn, {ConvertedShadow2,
891 IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()),
894 Value *Cmp = IRB.CreateICmpNE(
895 ConvertedShadow, getCleanShadow(ConvertedShadow), "_mscmp");
896 Instruction *CheckTerm = SplitBlockAndInsertIfThen(
897 Cmp, &*IRB.GetInsertPoint(), false, MS.OriginStoreWeights);
898 IRBuilder<> IRBNew(CheckTerm);
899 paintOrigin(IRBNew, updateOrigin(Origin, IRBNew), OriginPtr, StoreSize,
905 void materializeStores(bool InstrumentWithCalls) {
906 for (StoreInst *SI : StoreList) {
908 Value *Val = SI->getValueOperand();
909 Value *Addr = SI->getPointerOperand();
910 Value *Shadow = SI->isAtomic() ? getCleanShadow(Val) : getShadow(Val);
911 Value *ShadowPtr, *OriginPtr;
912 Type *ShadowTy = Shadow->getType();
913 unsigned Alignment = SI->getAlignment();
914 unsigned OriginAlignment = std::max(kMinOriginAlignment, Alignment);
915 std::tie(ShadowPtr, OriginPtr) =
916 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ true);
918 StoreInst *NewSI = IRB.CreateAlignedStore(Shadow, ShadowPtr, Alignment);
919 LLVM_DEBUG(dbgs() << " STORE: " << *NewSI << "\n");
923 SI->setOrdering(addReleaseOrdering(SI->getOrdering()));
925 if (MS.TrackOrigins && !SI->isAtomic())
926 storeOrigin(IRB, Addr, Shadow, getOrigin(Val), OriginPtr,
927 OriginAlignment, InstrumentWithCalls);
931 /// Helper function to insert a warning at IRB's current insert point.
932 void insertWarningFn(IRBuilder<> &IRB, Value *Origin) {
934 Origin = (Value *)IRB.getInt32(0);
935 if (MS.TrackOrigins) {
936 IRB.CreateStore(Origin, MS.OriginTLS);
938 IRB.CreateCall(MS.WarningFn, {});
939 IRB.CreateCall(MS.EmptyAsm, {});
940 // FIXME: Insert UnreachableInst if !MS.Recover?
941 // This may invalidate some of the following checks and needs to be done
945 void materializeOneCheck(Instruction *OrigIns, Value *Shadow, Value *Origin,
947 IRBuilder<> IRB(OrigIns);
948 LLVM_DEBUG(dbgs() << " SHAD0 : " << *Shadow << "\n");
949 Value *ConvertedShadow = convertToShadowTyNoVec(Shadow, IRB);
950 LLVM_DEBUG(dbgs() << " SHAD1 : " << *ConvertedShadow << "\n");
952 Constant *ConstantShadow = dyn_cast_or_null<Constant>(ConvertedShadow);
953 if (ConstantShadow) {
954 if (ClCheckConstantShadow && !ConstantShadow->isZeroValue()) {
955 insertWarningFn(IRB, Origin);
960 const DataLayout &DL = OrigIns->getModule()->getDataLayout();
962 unsigned TypeSizeInBits = DL.getTypeSizeInBits(ConvertedShadow->getType());
963 unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
964 if (AsCall && SizeIndex < kNumberOfAccessSizes) {
965 Value *Fn = MS.MaybeWarningFn[SizeIndex];
966 Value *ConvertedShadow2 =
967 IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
968 IRB.CreateCall(Fn, {ConvertedShadow2, MS.TrackOrigins && Origin
970 : (Value *)IRB.getInt32(0)});
972 Value *Cmp = IRB.CreateICmpNE(ConvertedShadow,
973 getCleanShadow(ConvertedShadow), "_mscmp");
974 Instruction *CheckTerm = SplitBlockAndInsertIfThen(
976 /* Unreachable */ !MS.Recover, MS.ColdCallWeights);
978 IRB.SetInsertPoint(CheckTerm);
979 insertWarningFn(IRB, Origin);
980 LLVM_DEBUG(dbgs() << " CHECK: " << *Cmp << "\n");
984 void materializeChecks(bool InstrumentWithCalls) {
985 for (const auto &ShadowData : InstrumentationList) {
986 Instruction *OrigIns = ShadowData.OrigIns;
987 Value *Shadow = ShadowData.Shadow;
988 Value *Origin = ShadowData.Origin;
989 materializeOneCheck(OrigIns, Shadow, Origin, InstrumentWithCalls);
991 LLVM_DEBUG(dbgs() << "DONE:\n" << F);
994 /// Add MemorySanitizer instrumentation to a function.
995 bool runOnFunction() {
996 // In the presence of unreachable blocks, we may see Phi nodes with
997 // incoming nodes from such blocks. Since InstVisitor skips unreachable
998 // blocks, such nodes will not have any shadow value associated with them.
999 // It's easier to remove unreachable blocks than deal with missing shadow.
1000 removeUnreachableBlocks(F);
1002 // Iterate all BBs in depth-first order and create shadow instructions
1003 // for all instructions (where applicable).
1004 // For PHI nodes we create dummy shadow PHIs which will be finalized later.
1005 for (BasicBlock *BB : depth_first(ActualFnStart))
1008 // Finalize PHI nodes.
1009 for (PHINode *PN : ShadowPHINodes) {
1010 PHINode *PNS = cast<PHINode>(getShadow(PN));
1011 PHINode *PNO = MS.TrackOrigins ? cast<PHINode>(getOrigin(PN)) : nullptr;
1012 size_t NumValues = PN->getNumIncomingValues();
1013 for (size_t v = 0; v < NumValues; v++) {
1014 PNS->addIncoming(getShadow(PN, v), PN->getIncomingBlock(v));
1015 if (PNO) PNO->addIncoming(getOrigin(PN, v), PN->getIncomingBlock(v));
1019 VAHelper->finalizeInstrumentation();
1021 bool InstrumentWithCalls = ClInstrumentationWithCallThreshold >= 0 &&
1022 InstrumentationList.size() + StoreList.size() >
1023 (unsigned)ClInstrumentationWithCallThreshold;
1025 // Insert shadow value checks.
1026 materializeChecks(InstrumentWithCalls);
1028 // Delayed instrumentation of StoreInst.
1029 // This may not add new address checks.
1030 materializeStores(InstrumentWithCalls);
1035 /// Compute the shadow type that corresponds to a given Value.
1036 Type *getShadowTy(Value *V) {
1037 return getShadowTy(V->getType());
1040 /// Compute the shadow type that corresponds to a given Type.
1041 Type *getShadowTy(Type *OrigTy) {
1042 if (!OrigTy->isSized()) {
1045 // For integer type, shadow is the same as the original type.
1046 // This may return weird-sized types like i1.
1047 if (IntegerType *IT = dyn_cast<IntegerType>(OrigTy))
1049 const DataLayout &DL = F.getParent()->getDataLayout();
1050 if (VectorType *VT = dyn_cast<VectorType>(OrigTy)) {
1051 uint32_t EltSize = DL.getTypeSizeInBits(VT->getElementType());
1052 return VectorType::get(IntegerType::get(*MS.C, EltSize),
1053 VT->getNumElements());
1055 if (ArrayType *AT = dyn_cast<ArrayType>(OrigTy)) {
1056 return ArrayType::get(getShadowTy(AT->getElementType()),
1057 AT->getNumElements());
1059 if (StructType *ST = dyn_cast<StructType>(OrigTy)) {
1060 SmallVector<Type*, 4> Elements;
1061 for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
1062 Elements.push_back(getShadowTy(ST->getElementType(i)));
1063 StructType *Res = StructType::get(*MS.C, Elements, ST->isPacked());
1064 LLVM_DEBUG(dbgs() << "getShadowTy: " << *ST << " ===> " << *Res << "\n");
1067 uint32_t TypeSize = DL.getTypeSizeInBits(OrigTy);
1068 return IntegerType::get(*MS.C, TypeSize);
1071 /// Flatten a vector type.
1072 Type *getShadowTyNoVec(Type *ty) {
1073 if (VectorType *vt = dyn_cast<VectorType>(ty))
1074 return IntegerType::get(*MS.C, vt->getBitWidth());
1078 /// Convert a shadow value to it's flattened variant.
1079 Value *convertToShadowTyNoVec(Value *V, IRBuilder<> &IRB) {
1080 Type *Ty = V->getType();
1081 Type *NoVecTy = getShadowTyNoVec(Ty);
1082 if (Ty == NoVecTy) return V;
1083 return IRB.CreateBitCast(V, NoVecTy);
1086 /// Compute the integer shadow offset that corresponds to a given
1087 /// application address.
1089 /// Offset = (Addr & ~AndMask) ^ XorMask
1090 Value *getShadowPtrOffset(Value *Addr, IRBuilder<> &IRB) {
1091 Value *OffsetLong = IRB.CreatePointerCast(Addr, MS.IntptrTy);
1093 uint64_t AndMask = MS.MapParams->AndMask;
1096 IRB.CreateAnd(OffsetLong, ConstantInt::get(MS.IntptrTy, ~AndMask));
1098 uint64_t XorMask = MS.MapParams->XorMask;
1101 IRB.CreateXor(OffsetLong, ConstantInt::get(MS.IntptrTy, XorMask));
1105 /// Compute the shadow and origin addresses corresponding to a given
1106 /// application address.
1108 /// Shadow = ShadowBase + Offset
1109 /// Origin = (OriginBase + Offset) & ~3ULL
1110 std::pair<Value *, Value *> getShadowOriginPtrUserspace(Value *Addr,
1113 unsigned Alignment) {
1114 Value *ShadowOffset = getShadowPtrOffset(Addr, IRB);
1115 Value *ShadowLong = ShadowOffset;
1116 uint64_t ShadowBase = MS.MapParams->ShadowBase;
1117 if (ShadowBase != 0) {
1119 IRB.CreateAdd(ShadowLong,
1120 ConstantInt::get(MS.IntptrTy, ShadowBase));
1123 IRB.CreateIntToPtr(ShadowLong, PointerType::get(ShadowTy, 0));
1124 Value *OriginPtr = nullptr;
1125 if (MS.TrackOrigins) {
1126 Value *OriginLong = ShadowOffset;
1127 uint64_t OriginBase = MS.MapParams->OriginBase;
1128 if (OriginBase != 0)
1129 OriginLong = IRB.CreateAdd(OriginLong,
1130 ConstantInt::get(MS.IntptrTy, OriginBase));
1131 if (Alignment < kMinOriginAlignment) {
1132 uint64_t Mask = kMinOriginAlignment - 1;
1134 IRB.CreateAnd(OriginLong, ConstantInt::get(MS.IntptrTy, ~Mask));
1137 IRB.CreateIntToPtr(OriginLong, PointerType::get(IRB.getInt32Ty(), 0));
1139 return std::make_pair(ShadowPtr, OriginPtr);
1142 std::pair<Value *, Value *> getShadowOriginPtr(Value *Addr, IRBuilder<> &IRB,
1146 std::pair<Value *, Value *> ret =
1147 getShadowOriginPtrUserspace(Addr, IRB, ShadowTy, Alignment);
1151 /// Compute the shadow address for a given function argument.
1153 /// Shadow = ParamTLS+ArgOffset.
1154 Value *getShadowPtrForArgument(Value *A, IRBuilder<> &IRB,
1156 Value *Base = IRB.CreatePointerCast(MS.ParamTLS, MS.IntptrTy);
1158 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
1159 return IRB.CreateIntToPtr(Base, PointerType::get(getShadowTy(A), 0),
1163 /// Compute the origin address for a given function argument.
1164 Value *getOriginPtrForArgument(Value *A, IRBuilder<> &IRB,
1166 if (!MS.TrackOrigins) return nullptr;
1167 Value *Base = IRB.CreatePointerCast(MS.ParamOriginTLS, MS.IntptrTy);
1169 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
1170 return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0),
1174 /// Compute the shadow address for a retval.
1175 Value *getShadowPtrForRetval(Value *A, IRBuilder<> &IRB) {
1176 return IRB.CreatePointerCast(MS.RetvalTLS,
1177 PointerType::get(getShadowTy(A), 0),
1181 /// Compute the origin address for a retval.
1182 Value *getOriginPtrForRetval(IRBuilder<> &IRB) {
1183 // We keep a single origin for the entire retval. Might be too optimistic.
1184 return MS.RetvalOriginTLS;
1187 /// Set SV to be the shadow value for V.
1188 void setShadow(Value *V, Value *SV) {
1189 assert(!ShadowMap.count(V) && "Values may only have one shadow");
1190 ShadowMap[V] = PropagateShadow ? SV : getCleanShadow(V);
1193 /// Set Origin to be the origin value for V.
1194 void setOrigin(Value *V, Value *Origin) {
1195 if (!MS.TrackOrigins) return;
1196 assert(!OriginMap.count(V) && "Values may only have one origin");
1197 LLVM_DEBUG(dbgs() << "ORIGIN: " << *V << " ==> " << *Origin << "\n");
1198 OriginMap[V] = Origin;
1201 Constant *getCleanShadow(Type *OrigTy) {
1202 Type *ShadowTy = getShadowTy(OrigTy);
1205 return Constant::getNullValue(ShadowTy);
1208 /// Create a clean shadow value for a given value.
1210 /// Clean shadow (all zeroes) means all bits of the value are defined
1212 Constant *getCleanShadow(Value *V) {
1213 return getCleanShadow(V->getType());
1216 /// Create a dirty shadow of a given shadow type.
1217 Constant *getPoisonedShadow(Type *ShadowTy) {
1219 if (isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy))
1220 return Constant::getAllOnesValue(ShadowTy);
1221 if (ArrayType *AT = dyn_cast<ArrayType>(ShadowTy)) {
1222 SmallVector<Constant *, 4> Vals(AT->getNumElements(),
1223 getPoisonedShadow(AT->getElementType()));
1224 return ConstantArray::get(AT, Vals);
1226 if (StructType *ST = dyn_cast<StructType>(ShadowTy)) {
1227 SmallVector<Constant *, 4> Vals;
1228 for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
1229 Vals.push_back(getPoisonedShadow(ST->getElementType(i)));
1230 return ConstantStruct::get(ST, Vals);
1232 llvm_unreachable("Unexpected shadow type");
1235 /// Create a dirty shadow for a given value.
1236 Constant *getPoisonedShadow(Value *V) {
1237 Type *ShadowTy = getShadowTy(V);
1240 return getPoisonedShadow(ShadowTy);
1243 /// Create a clean (zero) origin.
1244 Value *getCleanOrigin() {
1245 return Constant::getNullValue(MS.OriginTy);
1248 /// Get the shadow value for a given Value.
1250 /// This function either returns the value set earlier with setShadow,
1251 /// or extracts if from ParamTLS (for function arguments).
1252 Value *getShadow(Value *V) {
1253 if (!PropagateShadow) return getCleanShadow(V);
1254 if (Instruction *I = dyn_cast<Instruction>(V)) {
1255 if (I->getMetadata("nosanitize"))
1256 return getCleanShadow(V);
1257 // For instructions the shadow is already stored in the map.
1258 Value *Shadow = ShadowMap[V];
1260 LLVM_DEBUG(dbgs() << "No shadow: " << *V << "\n" << *(I->getParent()));
1262 assert(Shadow && "No shadow for a value");
1266 if (UndefValue *U = dyn_cast<UndefValue>(V)) {
1267 Value *AllOnes = PoisonUndef ? getPoisonedShadow(V) : getCleanShadow(V);
1268 LLVM_DEBUG(dbgs() << "Undef: " << *U << " ==> " << *AllOnes << "\n");
1272 if (Argument *A = dyn_cast<Argument>(V)) {
1273 // For arguments we compute the shadow on demand and store it in the map.
1274 Value **ShadowPtr = &ShadowMap[V];
1277 Function *F = A->getParent();
1278 IRBuilder<> EntryIRB(ActualFnStart->getFirstNonPHI());
1279 unsigned ArgOffset = 0;
1280 const DataLayout &DL = F->getParent()->getDataLayout();
1281 for (auto &FArg : F->args()) {
1282 if (!FArg.getType()->isSized()) {
1283 LLVM_DEBUG(dbgs() << "Arg is not sized\n");
1288 ? DL.getTypeAllocSize(FArg.getType()->getPointerElementType())
1289 : DL.getTypeAllocSize(FArg.getType());
1291 bool Overflow = ArgOffset + Size > kParamTLSSize;
1292 Value *Base = getShadowPtrForArgument(&FArg, EntryIRB, ArgOffset);
1293 if (FArg.hasByValAttr()) {
1294 // ByVal pointer itself has clean shadow. We copy the actual
1295 // argument shadow to the underlying memory.
1296 // Figure out maximal valid memcpy alignment.
1297 unsigned ArgAlign = FArg.getParamAlignment();
1298 if (ArgAlign == 0) {
1299 Type *EltType = A->getType()->getPointerElementType();
1300 ArgAlign = DL.getABITypeAlignment(EltType);
1302 Value *CpShadowPtr =
1303 getShadowOriginPtr(V, EntryIRB, EntryIRB.getInt8Ty(), ArgAlign,
1307 // ParamTLS overflow.
1308 EntryIRB.CreateMemSet(
1309 CpShadowPtr, Constant::getNullValue(EntryIRB.getInt8Ty()),
1312 unsigned CopyAlign = std::min(ArgAlign, kShadowTLSAlignment);
1313 Value *Cpy = EntryIRB.CreateMemCpy(CpShadowPtr, CopyAlign, Base,
1315 LLVM_DEBUG(dbgs() << " ByValCpy: " << *Cpy << "\n");
1318 *ShadowPtr = getCleanShadow(V);
1321 // ParamTLS overflow.
1322 *ShadowPtr = getCleanShadow(V);
1325 EntryIRB.CreateAlignedLoad(Base, kShadowTLSAlignment);
1329 << " ARG: " << FArg << " ==> " << **ShadowPtr << "\n");
1330 if (MS.TrackOrigins && !Overflow) {
1332 getOriginPtrForArgument(&FArg, EntryIRB, ArgOffset);
1333 setOrigin(A, EntryIRB.CreateLoad(OriginPtr));
1335 setOrigin(A, getCleanOrigin());
1338 ArgOffset += alignTo(Size, kShadowTLSAlignment);
1340 assert(*ShadowPtr && "Could not find shadow for an argument");
1343 // For everything else the shadow is zero.
1344 return getCleanShadow(V);
1347 /// Get the shadow for i-th argument of the instruction I.
1348 Value *getShadow(Instruction *I, int i) {
1349 return getShadow(I->getOperand(i));
1352 /// Get the origin for a value.
1353 Value *getOrigin(Value *V) {
1354 if (!MS.TrackOrigins) return nullptr;
1355 if (!PropagateShadow) return getCleanOrigin();
1356 if (isa<Constant>(V)) return getCleanOrigin();
1357 assert((isa<Instruction>(V) || isa<Argument>(V)) &&
1358 "Unexpected value type in getOrigin()");
1359 if (Instruction *I = dyn_cast<Instruction>(V)) {
1360 if (I->getMetadata("nosanitize"))
1361 return getCleanOrigin();
1363 Value *Origin = OriginMap[V];
1364 assert(Origin && "Missing origin");
1368 /// Get the origin for i-th argument of the instruction I.
1369 Value *getOrigin(Instruction *I, int i) {
1370 return getOrigin(I->getOperand(i));
1373 /// Remember the place where a shadow check should be inserted.
1375 /// This location will be later instrumented with a check that will print a
1376 /// UMR warning in runtime if the shadow value is not 0.
1377 void insertShadowCheck(Value *Shadow, Value *Origin, Instruction *OrigIns) {
1379 if (!InsertChecks) return;
1381 Type *ShadowTy = Shadow->getType();
1382 assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy)) &&
1383 "Can only insert checks for integer and vector shadow types");
1385 InstrumentationList.push_back(
1386 ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns));
1389 /// Remember the place where a shadow check should be inserted.
1391 /// This location will be later instrumented with a check that will print a
1392 /// UMR warning in runtime if the value is not fully defined.
1393 void insertShadowCheck(Value *Val, Instruction *OrigIns) {
1395 Value *Shadow, *Origin;
1396 if (ClCheckConstantShadow) {
1397 Shadow = getShadow(Val);
1398 if (!Shadow) return;
1399 Origin = getOrigin(Val);
1401 Shadow = dyn_cast_or_null<Instruction>(getShadow(Val));
1402 if (!Shadow) return;
1403 Origin = dyn_cast_or_null<Instruction>(getOrigin(Val));
1405 insertShadowCheck(Shadow, Origin, OrigIns);
1408 AtomicOrdering addReleaseOrdering(AtomicOrdering a) {
1410 case AtomicOrdering::NotAtomic:
1411 return AtomicOrdering::NotAtomic;
1412 case AtomicOrdering::Unordered:
1413 case AtomicOrdering::Monotonic:
1414 case AtomicOrdering::Release:
1415 return AtomicOrdering::Release;
1416 case AtomicOrdering::Acquire:
1417 case AtomicOrdering::AcquireRelease:
1418 return AtomicOrdering::AcquireRelease;
1419 case AtomicOrdering::SequentiallyConsistent:
1420 return AtomicOrdering::SequentiallyConsistent;
1422 llvm_unreachable("Unknown ordering");
1425 AtomicOrdering addAcquireOrdering(AtomicOrdering a) {
1427 case AtomicOrdering::NotAtomic:
1428 return AtomicOrdering::NotAtomic;
1429 case AtomicOrdering::Unordered:
1430 case AtomicOrdering::Monotonic:
1431 case AtomicOrdering::Acquire:
1432 return AtomicOrdering::Acquire;
1433 case AtomicOrdering::Release:
1434 case AtomicOrdering::AcquireRelease:
1435 return AtomicOrdering::AcquireRelease;
1436 case AtomicOrdering::SequentiallyConsistent:
1437 return AtomicOrdering::SequentiallyConsistent;
1439 llvm_unreachable("Unknown ordering");
1442 // ------------------- Visitors.
1443 using InstVisitor<MemorySanitizerVisitor>::visit;
1444 void visit(Instruction &I) {
1445 if (!I.getMetadata("nosanitize"))
1446 InstVisitor<MemorySanitizerVisitor>::visit(I);
1449 /// Instrument LoadInst
1451 /// Loads the corresponding shadow and (optionally) origin.
1452 /// Optionally, checks that the load address is fully defined.
1453 void visitLoadInst(LoadInst &I) {
1454 assert(I.getType()->isSized() && "Load type must have size");
1455 assert(!I.getMetadata("nosanitize"));
1456 IRBuilder<> IRB(I.getNextNode());
1457 Type *ShadowTy = getShadowTy(&I);
1458 Value *Addr = I.getPointerOperand();
1459 Value *ShadowPtr, *OriginPtr;
1460 unsigned Alignment = I.getAlignment();
1461 if (PropagateShadow) {
1462 std::tie(ShadowPtr, OriginPtr) =
1463 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
1464 setShadow(&I, IRB.CreateAlignedLoad(ShadowPtr, Alignment, "_msld"));
1466 setShadow(&I, getCleanShadow(&I));
1469 if (ClCheckAccessAddress)
1470 insertShadowCheck(I.getPointerOperand(), &I);
1473 I.setOrdering(addAcquireOrdering(I.getOrdering()));
1475 if (MS.TrackOrigins) {
1476 if (PropagateShadow) {
1477 unsigned OriginAlignment = std::max(kMinOriginAlignment, Alignment);
1478 setOrigin(&I, IRB.CreateAlignedLoad(OriginPtr, OriginAlignment));
1480 setOrigin(&I, getCleanOrigin());
1485 /// Instrument StoreInst
1487 /// Stores the corresponding shadow and (optionally) origin.
1488 /// Optionally, checks that the store address is fully defined.
1489 void visitStoreInst(StoreInst &I) {
1490 StoreList.push_back(&I);
1491 if (ClCheckAccessAddress)
1492 insertShadowCheck(I.getPointerOperand(), &I);
1495 void handleCASOrRMW(Instruction &I) {
1496 assert(isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I));
1498 IRBuilder<> IRB(&I);
1499 Value *Addr = I.getOperand(0);
1500 Value *ShadowPtr = getShadowOriginPtr(Addr, IRB, I.getType(),
1501 /*Alignment*/ 1, /*isStore*/ true)
1504 if (ClCheckAccessAddress)
1505 insertShadowCheck(Addr, &I);
1507 // Only test the conditional argument of cmpxchg instruction.
1508 // The other argument can potentially be uninitialized, but we can not
1509 // detect this situation reliably without possible false positives.
1510 if (isa<AtomicCmpXchgInst>(I))
1511 insertShadowCheck(I.getOperand(1), &I);
1513 IRB.CreateStore(getCleanShadow(&I), ShadowPtr);
1515 setShadow(&I, getCleanShadow(&I));
1516 setOrigin(&I, getCleanOrigin());
1519 void visitAtomicRMWInst(AtomicRMWInst &I) {
1521 I.setOrdering(addReleaseOrdering(I.getOrdering()));
1524 void visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
1526 I.setSuccessOrdering(addReleaseOrdering(I.getSuccessOrdering()));
1529 // Vector manipulation.
1530 void visitExtractElementInst(ExtractElementInst &I) {
1531 insertShadowCheck(I.getOperand(1), &I);
1532 IRBuilder<> IRB(&I);
1533 setShadow(&I, IRB.CreateExtractElement(getShadow(&I, 0), I.getOperand(1),
1535 setOrigin(&I, getOrigin(&I, 0));
1538 void visitInsertElementInst(InsertElementInst &I) {
1539 insertShadowCheck(I.getOperand(2), &I);
1540 IRBuilder<> IRB(&I);
1541 setShadow(&I, IRB.CreateInsertElement(getShadow(&I, 0), getShadow(&I, 1),
1542 I.getOperand(2), "_msprop"));
1543 setOriginForNaryOp(I);
1546 void visitShuffleVectorInst(ShuffleVectorInst &I) {
1547 insertShadowCheck(I.getOperand(2), &I);
1548 IRBuilder<> IRB(&I);
1549 setShadow(&I, IRB.CreateShuffleVector(getShadow(&I, 0), getShadow(&I, 1),
1550 I.getOperand(2), "_msprop"));
1551 setOriginForNaryOp(I);
1555 void visitSExtInst(SExtInst &I) {
1556 IRBuilder<> IRB(&I);
1557 setShadow(&I, IRB.CreateSExt(getShadow(&I, 0), I.getType(), "_msprop"));
1558 setOrigin(&I, getOrigin(&I, 0));
1561 void visitZExtInst(ZExtInst &I) {
1562 IRBuilder<> IRB(&I);
1563 setShadow(&I, IRB.CreateZExt(getShadow(&I, 0), I.getType(), "_msprop"));
1564 setOrigin(&I, getOrigin(&I, 0));
1567 void visitTruncInst(TruncInst &I) {
1568 IRBuilder<> IRB(&I);
1569 setShadow(&I, IRB.CreateTrunc(getShadow(&I, 0), I.getType(), "_msprop"));
1570 setOrigin(&I, getOrigin(&I, 0));
1573 void visitBitCastInst(BitCastInst &I) {
1574 // Special case: if this is the bitcast (there is exactly 1 allowed) between
1575 // a musttail call and a ret, don't instrument. New instructions are not
1576 // allowed after a musttail call.
1577 if (auto *CI = dyn_cast<CallInst>(I.getOperand(0)))
1578 if (CI->isMustTailCall())
1580 IRBuilder<> IRB(&I);
1581 setShadow(&I, IRB.CreateBitCast(getShadow(&I, 0), getShadowTy(&I)));
1582 setOrigin(&I, getOrigin(&I, 0));
1585 void visitPtrToIntInst(PtrToIntInst &I) {
1586 IRBuilder<> IRB(&I);
1587 setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false,
1588 "_msprop_ptrtoint"));
1589 setOrigin(&I, getOrigin(&I, 0));
1592 void visitIntToPtrInst(IntToPtrInst &I) {
1593 IRBuilder<> IRB(&I);
1594 setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false,
1595 "_msprop_inttoptr"));
1596 setOrigin(&I, getOrigin(&I, 0));
1599 void visitFPToSIInst(CastInst& I) { handleShadowOr(I); }
1600 void visitFPToUIInst(CastInst& I) { handleShadowOr(I); }
1601 void visitSIToFPInst(CastInst& I) { handleShadowOr(I); }
1602 void visitUIToFPInst(CastInst& I) { handleShadowOr(I); }
1603 void visitFPExtInst(CastInst& I) { handleShadowOr(I); }
1604 void visitFPTruncInst(CastInst& I) { handleShadowOr(I); }
1606 /// Propagate shadow for bitwise AND.
1608 /// This code is exact, i.e. if, for example, a bit in the left argument
1609 /// is defined and 0, then neither the value not definedness of the
1610 /// corresponding bit in B don't affect the resulting shadow.
1611 void visitAnd(BinaryOperator &I) {
1612 IRBuilder<> IRB(&I);
1613 // "And" of 0 and a poisoned value results in unpoisoned value.
1614 // 1&1 => 1; 0&1 => 0; p&1 => p;
1615 // 1&0 => 0; 0&0 => 0; p&0 => 0;
1616 // 1&p => p; 0&p => 0; p&p => p;
1617 // S = (S1 & S2) | (V1 & S2) | (S1 & V2)
1618 Value *S1 = getShadow(&I, 0);
1619 Value *S2 = getShadow(&I, 1);
1620 Value *V1 = I.getOperand(0);
1621 Value *V2 = I.getOperand(1);
1622 if (V1->getType() != S1->getType()) {
1623 V1 = IRB.CreateIntCast(V1, S1->getType(), false);
1624 V2 = IRB.CreateIntCast(V2, S2->getType(), false);
1626 Value *S1S2 = IRB.CreateAnd(S1, S2);
1627 Value *V1S2 = IRB.CreateAnd(V1, S2);
1628 Value *S1V2 = IRB.CreateAnd(S1, V2);
1629 setShadow(&I, IRB.CreateOr(S1S2, IRB.CreateOr(V1S2, S1V2)));
1630 setOriginForNaryOp(I);
1633 void visitOr(BinaryOperator &I) {
1634 IRBuilder<> IRB(&I);
1635 // "Or" of 1 and a poisoned value results in unpoisoned value.
1636 // 1|1 => 1; 0|1 => 1; p|1 => 1;
1637 // 1|0 => 1; 0|0 => 0; p|0 => p;
1638 // 1|p => 1; 0|p => p; p|p => p;
1639 // S = (S1 & S2) | (~V1 & S2) | (S1 & ~V2)
1640 Value *S1 = getShadow(&I, 0);
1641 Value *S2 = getShadow(&I, 1);
1642 Value *V1 = IRB.CreateNot(I.getOperand(0));
1643 Value *V2 = IRB.CreateNot(I.getOperand(1));
1644 if (V1->getType() != S1->getType()) {
1645 V1 = IRB.CreateIntCast(V1, S1->getType(), false);
1646 V2 = IRB.CreateIntCast(V2, S2->getType(), false);
1648 Value *S1S2 = IRB.CreateAnd(S1, S2);
1649 Value *V1S2 = IRB.CreateAnd(V1, S2);
1650 Value *S1V2 = IRB.CreateAnd(S1, V2);
1651 setShadow(&I, IRB.CreateOr(S1S2, IRB.CreateOr(V1S2, S1V2)));
1652 setOriginForNaryOp(I);
1655 /// Default propagation of shadow and/or origin.
1657 /// This class implements the general case of shadow propagation, used in all
1658 /// cases where we don't know and/or don't care about what the operation
1659 /// actually does. It converts all input shadow values to a common type
1660 /// (extending or truncating as necessary), and bitwise OR's them.
1662 /// This is much cheaper than inserting checks (i.e. requiring inputs to be
1663 /// fully initialized), and less prone to false positives.
1665 /// This class also implements the general case of origin propagation. For a
1666 /// Nary operation, result origin is set to the origin of an argument that is
1667 /// not entirely initialized. If there is more than one such arguments, the
1668 /// rightmost of them is picked. It does not matter which one is picked if all
1669 /// arguments are initialized.
1670 template <bool CombineShadow>
1672 Value *Shadow = nullptr;
1673 Value *Origin = nullptr;
1675 MemorySanitizerVisitor *MSV;
1678 Combiner(MemorySanitizerVisitor *MSV, IRBuilder<> &IRB)
1679 : IRB(IRB), MSV(MSV) {}
1681 /// Add a pair of shadow and origin values to the mix.
1682 Combiner &Add(Value *OpShadow, Value *OpOrigin) {
1683 if (CombineShadow) {
1688 OpShadow = MSV->CreateShadowCast(IRB, OpShadow, Shadow->getType());
1689 Shadow = IRB.CreateOr(Shadow, OpShadow, "_msprop");
1693 if (MSV->MS.TrackOrigins) {
1698 Constant *ConstOrigin = dyn_cast<Constant>(OpOrigin);
1699 // No point in adding something that might result in 0 origin value.
1700 if (!ConstOrigin || !ConstOrigin->isNullValue()) {
1701 Value *FlatShadow = MSV->convertToShadowTyNoVec(OpShadow, IRB);
1703 IRB.CreateICmpNE(FlatShadow, MSV->getCleanShadow(FlatShadow));
1704 Origin = IRB.CreateSelect(Cond, OpOrigin, Origin);
1711 /// Add an application value to the mix.
1712 Combiner &Add(Value *V) {
1713 Value *OpShadow = MSV->getShadow(V);
1714 Value *OpOrigin = MSV->MS.TrackOrigins ? MSV->getOrigin(V) : nullptr;
1715 return Add(OpShadow, OpOrigin);
1718 /// Set the current combined values as the given instruction's shadow
1720 void Done(Instruction *I) {
1721 if (CombineShadow) {
1723 Shadow = MSV->CreateShadowCast(IRB, Shadow, MSV->getShadowTy(I));
1724 MSV->setShadow(I, Shadow);
1726 if (MSV->MS.TrackOrigins) {
1728 MSV->setOrigin(I, Origin);
1733 using ShadowAndOriginCombiner = Combiner<true>;
1734 using OriginCombiner = Combiner<false>;
1736 /// Propagate origin for arbitrary operation.
1737 void setOriginForNaryOp(Instruction &I) {
1738 if (!MS.TrackOrigins) return;
1739 IRBuilder<> IRB(&I);
1740 OriginCombiner OC(this, IRB);
1741 for (Instruction::op_iterator OI = I.op_begin(); OI != I.op_end(); ++OI)
1746 size_t VectorOrPrimitiveTypeSizeInBits(Type *Ty) {
1747 assert(!(Ty->isVectorTy() && Ty->getScalarType()->isPointerTy()) &&
1748 "Vector of pointers is not a valid shadow type");
1749 return Ty->isVectorTy() ?
1750 Ty->getVectorNumElements() * Ty->getScalarSizeInBits() :
1751 Ty->getPrimitiveSizeInBits();
1754 /// Cast between two shadow types, extending or truncating as
1756 Value *CreateShadowCast(IRBuilder<> &IRB, Value *V, Type *dstTy,
1757 bool Signed = false) {
1758 Type *srcTy = V->getType();
1759 size_t srcSizeInBits = VectorOrPrimitiveTypeSizeInBits(srcTy);
1760 size_t dstSizeInBits = VectorOrPrimitiveTypeSizeInBits(dstTy);
1761 if (srcSizeInBits > 1 && dstSizeInBits == 1)
1762 return IRB.CreateICmpNE(V, getCleanShadow(V));
1764 if (dstTy->isIntegerTy() && srcTy->isIntegerTy())
1765 return IRB.CreateIntCast(V, dstTy, Signed);
1766 if (dstTy->isVectorTy() && srcTy->isVectorTy() &&
1767 dstTy->getVectorNumElements() == srcTy->getVectorNumElements())
1768 return IRB.CreateIntCast(V, dstTy, Signed);
1769 Value *V1 = IRB.CreateBitCast(V, Type::getIntNTy(*MS.C, srcSizeInBits));
1771 IRB.CreateIntCast(V1, Type::getIntNTy(*MS.C, dstSizeInBits), Signed);
1772 return IRB.CreateBitCast(V2, dstTy);
1773 // TODO: handle struct types.
1776 /// Cast an application value to the type of its own shadow.
1777 Value *CreateAppToShadowCast(IRBuilder<> &IRB, Value *V) {
1778 Type *ShadowTy = getShadowTy(V);
1779 if (V->getType() == ShadowTy)
1781 if (V->getType()->isPtrOrPtrVectorTy())
1782 return IRB.CreatePtrToInt(V, ShadowTy);
1784 return IRB.CreateBitCast(V, ShadowTy);
1787 /// Propagate shadow for arbitrary operation.
1788 void handleShadowOr(Instruction &I) {
1789 IRBuilder<> IRB(&I);
1790 ShadowAndOriginCombiner SC(this, IRB);
1791 for (Instruction::op_iterator OI = I.op_begin(); OI != I.op_end(); ++OI)
1796 // Handle multiplication by constant.
1798 // Handle a special case of multiplication by constant that may have one or
1799 // more zeros in the lower bits. This makes corresponding number of lower bits
1800 // of the result zero as well. We model it by shifting the other operand
1801 // shadow left by the required number of bits. Effectively, we transform
1802 // (X * (A * 2**B)) to ((X << B) * A) and instrument (X << B) as (Sx << B).
1803 // We use multiplication by 2**N instead of shift to cover the case of
1804 // multiplication by 0, which may occur in some elements of a vector operand.
1805 void handleMulByConstant(BinaryOperator &I, Constant *ConstArg,
1807 Constant *ShadowMul;
1808 Type *Ty = ConstArg->getType();
1809 if (Ty->isVectorTy()) {
1810 unsigned NumElements = Ty->getVectorNumElements();
1811 Type *EltTy = Ty->getSequentialElementType();
1812 SmallVector<Constant *, 16> Elements;
1813 for (unsigned Idx = 0; Idx < NumElements; ++Idx) {
1814 if (ConstantInt *Elt =
1815 dyn_cast<ConstantInt>(ConstArg->getAggregateElement(Idx))) {
1816 const APInt &V = Elt->getValue();
1817 APInt V2 = APInt(V.getBitWidth(), 1) << V.countTrailingZeros();
1818 Elements.push_back(ConstantInt::get(EltTy, V2));
1820 Elements.push_back(ConstantInt::get(EltTy, 1));
1823 ShadowMul = ConstantVector::get(Elements);
1825 if (ConstantInt *Elt = dyn_cast<ConstantInt>(ConstArg)) {
1826 const APInt &V = Elt->getValue();
1827 APInt V2 = APInt(V.getBitWidth(), 1) << V.countTrailingZeros();
1828 ShadowMul = ConstantInt::get(Ty, V2);
1830 ShadowMul = ConstantInt::get(Ty, 1);
1834 IRBuilder<> IRB(&I);
1836 IRB.CreateMul(getShadow(OtherArg), ShadowMul, "msprop_mul_cst"));
1837 setOrigin(&I, getOrigin(OtherArg));
1840 void visitMul(BinaryOperator &I) {
1841 Constant *constOp0 = dyn_cast<Constant>(I.getOperand(0));
1842 Constant *constOp1 = dyn_cast<Constant>(I.getOperand(1));
1843 if (constOp0 && !constOp1)
1844 handleMulByConstant(I, constOp0, I.getOperand(1));
1845 else if (constOp1 && !constOp0)
1846 handleMulByConstant(I, constOp1, I.getOperand(0));
1851 void visitFAdd(BinaryOperator &I) { handleShadowOr(I); }
1852 void visitFSub(BinaryOperator &I) { handleShadowOr(I); }
1853 void visitFMul(BinaryOperator &I) { handleShadowOr(I); }
1854 void visitAdd(BinaryOperator &I) { handleShadowOr(I); }
1855 void visitSub(BinaryOperator &I) { handleShadowOr(I); }
1856 void visitXor(BinaryOperator &I) { handleShadowOr(I); }
1858 void handleIntegerDiv(Instruction &I) {
1859 IRBuilder<> IRB(&I);
1860 // Strict on the second argument.
1861 insertShadowCheck(I.getOperand(1), &I);
1862 setShadow(&I, getShadow(&I, 0));
1863 setOrigin(&I, getOrigin(&I, 0));
1866 void visitUDiv(BinaryOperator &I) { handleIntegerDiv(I); }
1867 void visitSDiv(BinaryOperator &I) { handleIntegerDiv(I); }
1868 void visitURem(BinaryOperator &I) { handleIntegerDiv(I); }
1869 void visitSRem(BinaryOperator &I) { handleIntegerDiv(I); }
1871 // Floating point division is side-effect free. We can not require that the
1872 // divisor is fully initialized and must propagate shadow. See PR37523.
1873 void visitFDiv(BinaryOperator &I) { handleShadowOr(I); }
1874 void visitFRem(BinaryOperator &I) { handleShadowOr(I); }
1876 /// Instrument == and != comparisons.
1878 /// Sometimes the comparison result is known even if some of the bits of the
1879 /// arguments are not.
1880 void handleEqualityComparison(ICmpInst &I) {
1881 IRBuilder<> IRB(&I);
1882 Value *A = I.getOperand(0);
1883 Value *B = I.getOperand(1);
1884 Value *Sa = getShadow(A);
1885 Value *Sb = getShadow(B);
1887 // Get rid of pointers and vectors of pointers.
1888 // For ints (and vectors of ints), types of A and Sa match,
1889 // and this is a no-op.
1890 A = IRB.CreatePointerCast(A, Sa->getType());
1891 B = IRB.CreatePointerCast(B, Sb->getType());
1893 // A == B <==> (C = A^B) == 0
1894 // A != B <==> (C = A^B) != 0
1896 Value *C = IRB.CreateXor(A, B);
1897 Value *Sc = IRB.CreateOr(Sa, Sb);
1898 // Now dealing with i = (C == 0) comparison (or C != 0, does not matter now)
1899 // Result is defined if one of the following is true
1900 // * there is a defined 1 bit in C
1901 // * C is fully defined
1902 // Si = !(C & ~Sc) && Sc
1903 Value *Zero = Constant::getNullValue(Sc->getType());
1904 Value *MinusOne = Constant::getAllOnesValue(Sc->getType());
1906 IRB.CreateAnd(IRB.CreateICmpNE(Sc, Zero),
1908 IRB.CreateAnd(IRB.CreateXor(Sc, MinusOne), C), Zero));
1909 Si->setName("_msprop_icmp");
1911 setOriginForNaryOp(I);
1914 /// Build the lowest possible value of V, taking into account V's
1915 /// uninitialized bits.
1916 Value *getLowestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
1919 // Split shadow into sign bit and other bits.
1920 Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1);
1921 Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits);
1922 // Maximise the undefined shadow bit, minimize other undefined bits.
1924 IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaOtherBits)), SaSignBit);
1926 // Minimize undefined bits.
1927 return IRB.CreateAnd(A, IRB.CreateNot(Sa));
1931 /// Build the highest possible value of V, taking into account V's
1932 /// uninitialized bits.
1933 Value *getHighestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
1936 // Split shadow into sign bit and other bits.
1937 Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1);
1938 Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits);
1939 // Minimise the undefined shadow bit, maximise other undefined bits.
1941 IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaSignBit)), SaOtherBits);
1943 // Maximize undefined bits.
1944 return IRB.CreateOr(A, Sa);
1948 /// Instrument relational comparisons.
1950 /// This function does exact shadow propagation for all relational
1951 /// comparisons of integers, pointers and vectors of those.
1952 /// FIXME: output seems suboptimal when one of the operands is a constant
1953 void handleRelationalComparisonExact(ICmpInst &I) {
1954 IRBuilder<> IRB(&I);
1955 Value *A = I.getOperand(0);
1956 Value *B = I.getOperand(1);
1957 Value *Sa = getShadow(A);
1958 Value *Sb = getShadow(B);
1960 // Get rid of pointers and vectors of pointers.
1961 // For ints (and vectors of ints), types of A and Sa match,
1962 // and this is a no-op.
1963 A = IRB.CreatePointerCast(A, Sa->getType());
1964 B = IRB.CreatePointerCast(B, Sb->getType());
1966 // Let [a0, a1] be the interval of possible values of A, taking into account
1967 // its undefined bits. Let [b0, b1] be the interval of possible values of B.
1968 // Then (A cmp B) is defined iff (a0 cmp b1) == (a1 cmp b0).
1969 bool IsSigned = I.isSigned();
1970 Value *S1 = IRB.CreateICmp(I.getPredicate(),
1971 getLowestPossibleValue(IRB, A, Sa, IsSigned),
1972 getHighestPossibleValue(IRB, B, Sb, IsSigned));
1973 Value *S2 = IRB.CreateICmp(I.getPredicate(),
1974 getHighestPossibleValue(IRB, A, Sa, IsSigned),
1975 getLowestPossibleValue(IRB, B, Sb, IsSigned));
1976 Value *Si = IRB.CreateXor(S1, S2);
1978 setOriginForNaryOp(I);
1981 /// Instrument signed relational comparisons.
1983 /// Handle sign bit tests: x<0, x>=0, x<=-1, x>-1 by propagating the highest
1984 /// bit of the shadow. Everything else is delegated to handleShadowOr().
1985 void handleSignedRelationalComparison(ICmpInst &I) {
1987 Value *op = nullptr;
1988 CmpInst::Predicate pre;
1989 if ((constOp = dyn_cast<Constant>(I.getOperand(1)))) {
1990 op = I.getOperand(0);
1991 pre = I.getPredicate();
1992 } else if ((constOp = dyn_cast<Constant>(I.getOperand(0)))) {
1993 op = I.getOperand(1);
1994 pre = I.getSwappedPredicate();
2000 if ((constOp->isNullValue() &&
2001 (pre == CmpInst::ICMP_SLT || pre == CmpInst::ICMP_SGE)) ||
2002 (constOp->isAllOnesValue() &&
2003 (pre == CmpInst::ICMP_SGT || pre == CmpInst::ICMP_SLE))) {
2004 IRBuilder<> IRB(&I);
2005 Value *Shadow = IRB.CreateICmpSLT(getShadow(op), getCleanShadow(op),
2007 setShadow(&I, Shadow);
2008 setOrigin(&I, getOrigin(op));
2014 void visitICmpInst(ICmpInst &I) {
2015 if (!ClHandleICmp) {
2019 if (I.isEquality()) {
2020 handleEqualityComparison(I);
2024 assert(I.isRelational());
2025 if (ClHandleICmpExact) {
2026 handleRelationalComparisonExact(I);
2030 handleSignedRelationalComparison(I);
2034 assert(I.isUnsigned());
2035 if ((isa<Constant>(I.getOperand(0)) || isa<Constant>(I.getOperand(1)))) {
2036 handleRelationalComparisonExact(I);
2043 void visitFCmpInst(FCmpInst &I) {
2047 void handleShift(BinaryOperator &I) {
2048 IRBuilder<> IRB(&I);
2049 // If any of the S2 bits are poisoned, the whole thing is poisoned.
2050 // Otherwise perform the same shift on S1.
2051 Value *S1 = getShadow(&I, 0);
2052 Value *S2 = getShadow(&I, 1);
2053 Value *S2Conv = IRB.CreateSExt(IRB.CreateICmpNE(S2, getCleanShadow(S2)),
2055 Value *V2 = I.getOperand(1);
2056 Value *Shift = IRB.CreateBinOp(I.getOpcode(), S1, V2);
2057 setShadow(&I, IRB.CreateOr(Shift, S2Conv));
2058 setOriginForNaryOp(I);
2061 void visitShl(BinaryOperator &I) { handleShift(I); }
2062 void visitAShr(BinaryOperator &I) { handleShift(I); }
2063 void visitLShr(BinaryOperator &I) { handleShift(I); }
2065 /// Instrument llvm.memmove
2067 /// At this point we don't know if llvm.memmove will be inlined or not.
2068 /// If we don't instrument it and it gets inlined,
2069 /// our interceptor will not kick in and we will lose the memmove.
2070 /// If we instrument the call here, but it does not get inlined,
2071 /// we will memove the shadow twice: which is bad in case
2072 /// of overlapping regions. So, we simply lower the intrinsic to a call.
2074 /// Similar situation exists for memcpy and memset.
2075 void visitMemMoveInst(MemMoveInst &I) {
2076 IRBuilder<> IRB(&I);
2079 {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2080 IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
2081 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2082 I.eraseFromParent();
2085 // Similar to memmove: avoid copying shadow twice.
2086 // This is somewhat unfortunate as it may slowdown small constant memcpys.
2087 // FIXME: consider doing manual inline for small constant sizes and proper
2089 void visitMemCpyInst(MemCpyInst &I) {
2090 IRBuilder<> IRB(&I);
2093 {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2094 IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
2095 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2096 I.eraseFromParent();
2100 void visitMemSetInst(MemSetInst &I) {
2101 IRBuilder<> IRB(&I);
2104 {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2105 IRB.CreateIntCast(I.getArgOperand(1), IRB.getInt32Ty(), false),
2106 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2107 I.eraseFromParent();
2110 void visitVAStartInst(VAStartInst &I) {
2111 VAHelper->visitVAStartInst(I);
2114 void visitVACopyInst(VACopyInst &I) {
2115 VAHelper->visitVACopyInst(I);
2118 /// Handle vector store-like intrinsics.
2120 /// Instrument intrinsics that look like a simple SIMD store: writes memory,
2121 /// has 1 pointer argument and 1 vector argument, returns void.
2122 bool handleVectorStoreIntrinsic(IntrinsicInst &I) {
2123 IRBuilder<> IRB(&I);
2124 Value* Addr = I.getArgOperand(0);
2125 Value *Shadow = getShadow(&I, 1);
2126 Value *ShadowPtr, *OriginPtr;
2128 // We don't know the pointer alignment (could be unaligned SSE store!).
2129 // Have to assume to worst case.
2130 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
2131 Addr, IRB, Shadow->getType(), /*Alignment*/ 1, /*isStore*/ true);
2132 IRB.CreateAlignedStore(Shadow, ShadowPtr, 1);
2134 if (ClCheckAccessAddress)
2135 insertShadowCheck(Addr, &I);
2137 // FIXME: factor out common code from materializeStores
2138 if (MS.TrackOrigins) IRB.CreateStore(getOrigin(&I, 1), OriginPtr);
2142 /// Handle vector load-like intrinsics.
2144 /// Instrument intrinsics that look like a simple SIMD load: reads memory,
2145 /// has 1 pointer argument, returns a vector.
2146 bool handleVectorLoadIntrinsic(IntrinsicInst &I) {
2147 IRBuilder<> IRB(&I);
2148 Value *Addr = I.getArgOperand(0);
2150 Type *ShadowTy = getShadowTy(&I);
2151 Value *ShadowPtr, *OriginPtr;
2152 if (PropagateShadow) {
2153 // We don't know the pointer alignment (could be unaligned SSE load!).
2154 // Have to assume to worst case.
2155 unsigned Alignment = 1;
2156 std::tie(ShadowPtr, OriginPtr) =
2157 getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
2158 setShadow(&I, IRB.CreateAlignedLoad(ShadowPtr, Alignment, "_msld"));
2160 setShadow(&I, getCleanShadow(&I));
2163 if (ClCheckAccessAddress)
2164 insertShadowCheck(Addr, &I);
2166 if (MS.TrackOrigins) {
2167 if (PropagateShadow)
2168 setOrigin(&I, IRB.CreateLoad(OriginPtr));
2170 setOrigin(&I, getCleanOrigin());
2175 /// Handle (SIMD arithmetic)-like intrinsics.
2177 /// Instrument intrinsics with any number of arguments of the same type,
2178 /// equal to the return type. The type should be simple (no aggregates or
2179 /// pointers; vectors are fine).
2180 /// Caller guarantees that this intrinsic does not access memory.
2181 bool maybeHandleSimpleNomemIntrinsic(IntrinsicInst &I) {
2182 Type *RetTy = I.getType();
2183 if (!(RetTy->isIntOrIntVectorTy() ||
2184 RetTy->isFPOrFPVectorTy() ||
2185 RetTy->isX86_MMXTy()))
2188 unsigned NumArgOperands = I.getNumArgOperands();
2190 for (unsigned i = 0; i < NumArgOperands; ++i) {
2191 Type *Ty = I.getArgOperand(i)->getType();
2196 IRBuilder<> IRB(&I);
2197 ShadowAndOriginCombiner SC(this, IRB);
2198 for (unsigned i = 0; i < NumArgOperands; ++i)
2199 SC.Add(I.getArgOperand(i));
2205 /// Heuristically instrument unknown intrinsics.
2207 /// The main purpose of this code is to do something reasonable with all
2208 /// random intrinsics we might encounter, most importantly - SIMD intrinsics.
2209 /// We recognize several classes of intrinsics by their argument types and
2210 /// ModRefBehaviour and apply special intrumentation when we are reasonably
2211 /// sure that we know what the intrinsic does.
2213 /// We special-case intrinsics where this approach fails. See llvm.bswap
2214 /// handling as an example of that.
2215 bool handleUnknownIntrinsic(IntrinsicInst &I) {
2216 unsigned NumArgOperands = I.getNumArgOperands();
2217 if (NumArgOperands == 0)
2220 if (NumArgOperands == 2 &&
2221 I.getArgOperand(0)->getType()->isPointerTy() &&
2222 I.getArgOperand(1)->getType()->isVectorTy() &&
2223 I.getType()->isVoidTy() &&
2224 !I.onlyReadsMemory()) {
2225 // This looks like a vector store.
2226 return handleVectorStoreIntrinsic(I);
2229 if (NumArgOperands == 1 &&
2230 I.getArgOperand(0)->getType()->isPointerTy() &&
2231 I.getType()->isVectorTy() &&
2232 I.onlyReadsMemory()) {
2233 // This looks like a vector load.
2234 return handleVectorLoadIntrinsic(I);
2237 if (I.doesNotAccessMemory())
2238 if (maybeHandleSimpleNomemIntrinsic(I))
2241 // FIXME: detect and handle SSE maskstore/maskload
2245 void handleBswap(IntrinsicInst &I) {
2246 IRBuilder<> IRB(&I);
2247 Value *Op = I.getArgOperand(0);
2248 Type *OpType = Op->getType();
2249 Function *BswapFunc = Intrinsic::getDeclaration(
2250 F.getParent(), Intrinsic::bswap, makeArrayRef(&OpType, 1));
2251 setShadow(&I, IRB.CreateCall(BswapFunc, getShadow(Op)));
2252 setOrigin(&I, getOrigin(Op));
2255 // Instrument vector convert instrinsic.
2257 // This function instruments intrinsics like cvtsi2ss:
2258 // %Out = int_xxx_cvtyyy(%ConvertOp)
2260 // %Out = int_xxx_cvtyyy(%CopyOp, %ConvertOp)
2261 // Intrinsic converts \p NumUsedElements elements of \p ConvertOp to the same
2262 // number \p Out elements, and (if has 2 arguments) copies the rest of the
2263 // elements from \p CopyOp.
2264 // In most cases conversion involves floating-point value which may trigger a
2265 // hardware exception when not fully initialized. For this reason we require
2266 // \p ConvertOp[0:NumUsedElements] to be fully initialized and trap otherwise.
2267 // We copy the shadow of \p CopyOp[NumUsedElements:] to \p
2268 // Out[NumUsedElements:]. This means that intrinsics without \p CopyOp always
2269 // return a fully initialized value.
2270 void handleVectorConvertIntrinsic(IntrinsicInst &I, int NumUsedElements) {
2271 IRBuilder<> IRB(&I);
2272 Value *CopyOp, *ConvertOp;
2274 switch (I.getNumArgOperands()) {
2276 assert(isa<ConstantInt>(I.getArgOperand(2)) && "Invalid rounding mode");
2279 CopyOp = I.getArgOperand(0);
2280 ConvertOp = I.getArgOperand(1);
2283 ConvertOp = I.getArgOperand(0);
2287 llvm_unreachable("Cvt intrinsic with unsupported number of arguments.");
2290 // The first *NumUsedElements* elements of ConvertOp are converted to the
2291 // same number of output elements. The rest of the output is copied from
2292 // CopyOp, or (if not available) filled with zeroes.
2293 // Combine shadow for elements of ConvertOp that are used in this operation,
2294 // and insert a check.
2295 // FIXME: consider propagating shadow of ConvertOp, at least in the case of
2296 // int->any conversion.
2297 Value *ConvertShadow = getShadow(ConvertOp);
2298 Value *AggShadow = nullptr;
2299 if (ConvertOp->getType()->isVectorTy()) {
2300 AggShadow = IRB.CreateExtractElement(
2301 ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), 0));
2302 for (int i = 1; i < NumUsedElements; ++i) {
2303 Value *MoreShadow = IRB.CreateExtractElement(
2304 ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), i));
2305 AggShadow = IRB.CreateOr(AggShadow, MoreShadow);
2308 AggShadow = ConvertShadow;
2310 assert(AggShadow->getType()->isIntegerTy());
2311 insertShadowCheck(AggShadow, getOrigin(ConvertOp), &I);
2313 // Build result shadow by zero-filling parts of CopyOp shadow that come from
2316 assert(CopyOp->getType() == I.getType());
2317 assert(CopyOp->getType()->isVectorTy());
2318 Value *ResultShadow = getShadow(CopyOp);
2319 Type *EltTy = ResultShadow->getType()->getVectorElementType();
2320 for (int i = 0; i < NumUsedElements; ++i) {
2321 ResultShadow = IRB.CreateInsertElement(
2322 ResultShadow, ConstantInt::getNullValue(EltTy),
2323 ConstantInt::get(IRB.getInt32Ty(), i));
2325 setShadow(&I, ResultShadow);
2326 setOrigin(&I, getOrigin(CopyOp));
2328 setShadow(&I, getCleanShadow(&I));
2329 setOrigin(&I, getCleanOrigin());
2333 // Given a scalar or vector, extract lower 64 bits (or less), and return all
2334 // zeroes if it is zero, and all ones otherwise.
2335 Value *Lower64ShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
2336 if (S->getType()->isVectorTy())
2337 S = CreateShadowCast(IRB, S, IRB.getInt64Ty(), /* Signed */ true);
2338 assert(S->getType()->getPrimitiveSizeInBits() <= 64);
2339 Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S));
2340 return CreateShadowCast(IRB, S2, T, /* Signed */ true);
2343 // Given a vector, extract its first element, and return all
2344 // zeroes if it is zero, and all ones otherwise.
2345 Value *LowerElementShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
2346 Value *S1 = IRB.CreateExtractElement(S, (uint64_t)0);
2347 Value *S2 = IRB.CreateICmpNE(S1, getCleanShadow(S1));
2348 return CreateShadowCast(IRB, S2, T, /* Signed */ true);
2351 Value *VariableShadowExtend(IRBuilder<> &IRB, Value *S) {
2352 Type *T = S->getType();
2353 assert(T->isVectorTy());
2354 Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S));
2355 return IRB.CreateSExt(S2, T);
2358 // Instrument vector shift instrinsic.
2360 // This function instruments intrinsics like int_x86_avx2_psll_w.
2361 // Intrinsic shifts %In by %ShiftSize bits.
2362 // %ShiftSize may be a vector. In that case the lower 64 bits determine shift
2363 // size, and the rest is ignored. Behavior is defined even if shift size is
2364 // greater than register (or field) width.
2365 void handleVectorShiftIntrinsic(IntrinsicInst &I, bool Variable) {
2366 assert(I.getNumArgOperands() == 2);
2367 IRBuilder<> IRB(&I);
2368 // If any of the S2 bits are poisoned, the whole thing is poisoned.
2369 // Otherwise perform the same shift on S1.
2370 Value *S1 = getShadow(&I, 0);
2371 Value *S2 = getShadow(&I, 1);
2372 Value *S2Conv = Variable ? VariableShadowExtend(IRB, S2)
2373 : Lower64ShadowExtend(IRB, S2, getShadowTy(&I));
2374 Value *V1 = I.getOperand(0);
2375 Value *V2 = I.getOperand(1);
2376 Value *Shift = IRB.CreateCall(I.getCalledValue(),
2377 {IRB.CreateBitCast(S1, V1->getType()), V2});
2378 Shift = IRB.CreateBitCast(Shift, getShadowTy(&I));
2379 setShadow(&I, IRB.CreateOr(Shift, S2Conv));
2380 setOriginForNaryOp(I);
2383 // Get an X86_MMX-sized vector type.
2384 Type *getMMXVectorTy(unsigned EltSizeInBits) {
2385 const unsigned X86_MMXSizeInBits = 64;
2386 return VectorType::get(IntegerType::get(*MS.C, EltSizeInBits),
2387 X86_MMXSizeInBits / EltSizeInBits);
2390 // Returns a signed counterpart for an (un)signed-saturate-and-pack
2392 Intrinsic::ID getSignedPackIntrinsic(Intrinsic::ID id) {
2394 case Intrinsic::x86_sse2_packsswb_128:
2395 case Intrinsic::x86_sse2_packuswb_128:
2396 return Intrinsic::x86_sse2_packsswb_128;
2398 case Intrinsic::x86_sse2_packssdw_128:
2399 case Intrinsic::x86_sse41_packusdw:
2400 return Intrinsic::x86_sse2_packssdw_128;
2402 case Intrinsic::x86_avx2_packsswb:
2403 case Intrinsic::x86_avx2_packuswb:
2404 return Intrinsic::x86_avx2_packsswb;
2406 case Intrinsic::x86_avx2_packssdw:
2407 case Intrinsic::x86_avx2_packusdw:
2408 return Intrinsic::x86_avx2_packssdw;
2410 case Intrinsic::x86_mmx_packsswb:
2411 case Intrinsic::x86_mmx_packuswb:
2412 return Intrinsic::x86_mmx_packsswb;
2414 case Intrinsic::x86_mmx_packssdw:
2415 return Intrinsic::x86_mmx_packssdw;
2417 llvm_unreachable("unexpected intrinsic id");
2421 // Instrument vector pack instrinsic.
2423 // This function instruments intrinsics like x86_mmx_packsswb, that
2424 // packs elements of 2 input vectors into half as many bits with saturation.
2425 // Shadow is propagated with the signed variant of the same intrinsic applied
2426 // to sext(Sa != zeroinitializer), sext(Sb != zeroinitializer).
2427 // EltSizeInBits is used only for x86mmx arguments.
2428 void handleVectorPackIntrinsic(IntrinsicInst &I, unsigned EltSizeInBits = 0) {
2429 assert(I.getNumArgOperands() == 2);
2430 bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
2431 IRBuilder<> IRB(&I);
2432 Value *S1 = getShadow(&I, 0);
2433 Value *S2 = getShadow(&I, 1);
2434 assert(isX86_MMX || S1->getType()->isVectorTy());
2436 // SExt and ICmpNE below must apply to individual elements of input vectors.
2437 // In case of x86mmx arguments, cast them to appropriate vector types and
2439 Type *T = isX86_MMX ? getMMXVectorTy(EltSizeInBits) : S1->getType();
2441 S1 = IRB.CreateBitCast(S1, T);
2442 S2 = IRB.CreateBitCast(S2, T);
2444 Value *S1_ext = IRB.CreateSExt(
2445 IRB.CreateICmpNE(S1, Constant::getNullValue(T)), T);
2446 Value *S2_ext = IRB.CreateSExt(
2447 IRB.CreateICmpNE(S2, Constant::getNullValue(T)), T);
2449 Type *X86_MMXTy = Type::getX86_MMXTy(*MS.C);
2450 S1_ext = IRB.CreateBitCast(S1_ext, X86_MMXTy);
2451 S2_ext = IRB.CreateBitCast(S2_ext, X86_MMXTy);
2454 Function *ShadowFn = Intrinsic::getDeclaration(
2455 F.getParent(), getSignedPackIntrinsic(I.getIntrinsicID()));
2458 IRB.CreateCall(ShadowFn, {S1_ext, S2_ext}, "_msprop_vector_pack");
2459 if (isX86_MMX) S = IRB.CreateBitCast(S, getShadowTy(&I));
2461 setOriginForNaryOp(I);
2464 // Instrument sum-of-absolute-differencies intrinsic.
2465 void handleVectorSadIntrinsic(IntrinsicInst &I) {
2466 const unsigned SignificantBitsPerResultElement = 16;
2467 bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
2468 Type *ResTy = isX86_MMX ? IntegerType::get(*MS.C, 64) : I.getType();
2469 unsigned ZeroBitsPerResultElement =
2470 ResTy->getScalarSizeInBits() - SignificantBitsPerResultElement;
2472 IRBuilder<> IRB(&I);
2473 Value *S = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2474 S = IRB.CreateBitCast(S, ResTy);
2475 S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)),
2477 S = IRB.CreateLShr(S, ZeroBitsPerResultElement);
2478 S = IRB.CreateBitCast(S, getShadowTy(&I));
2480 setOriginForNaryOp(I);
2483 // Instrument multiply-add intrinsic.
2484 void handleVectorPmaddIntrinsic(IntrinsicInst &I,
2485 unsigned EltSizeInBits = 0) {
2486 bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
2487 Type *ResTy = isX86_MMX ? getMMXVectorTy(EltSizeInBits * 2) : I.getType();
2488 IRBuilder<> IRB(&I);
2489 Value *S = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2490 S = IRB.CreateBitCast(S, ResTy);
2491 S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)),
2493 S = IRB.CreateBitCast(S, getShadowTy(&I));
2495 setOriginForNaryOp(I);
2498 // Instrument compare-packed intrinsic.
2499 // Basically, an or followed by sext(icmp ne 0) to end up with all-zeros or
2501 void handleVectorComparePackedIntrinsic(IntrinsicInst &I) {
2502 IRBuilder<> IRB(&I);
2503 Type *ResTy = getShadowTy(&I);
2504 Value *S0 = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2505 Value *S = IRB.CreateSExt(
2506 IRB.CreateICmpNE(S0, Constant::getNullValue(ResTy)), ResTy);
2508 setOriginForNaryOp(I);
2511 // Instrument compare-scalar intrinsic.
2512 // This handles both cmp* intrinsics which return the result in the first
2513 // element of a vector, and comi* which return the result as i32.
2514 void handleVectorCompareScalarIntrinsic(IntrinsicInst &I) {
2515 IRBuilder<> IRB(&I);
2516 Value *S0 = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2517 Value *S = LowerElementShadowExtend(IRB, S0, getShadowTy(&I));
2519 setOriginForNaryOp(I);
2522 void handleStmxcsr(IntrinsicInst &I) {
2523 IRBuilder<> IRB(&I);
2524 Value* Addr = I.getArgOperand(0);
2525 Type *Ty = IRB.getInt32Ty();
2527 getShadowOriginPtr(Addr, IRB, Ty, /*Alignment*/ 1, /*isStore*/ true)
2530 IRB.CreateStore(getCleanShadow(Ty),
2531 IRB.CreatePointerCast(ShadowPtr, Ty->getPointerTo()));
2533 if (ClCheckAccessAddress)
2534 insertShadowCheck(Addr, &I);
2537 void handleLdmxcsr(IntrinsicInst &I) {
2538 if (!InsertChecks) return;
2540 IRBuilder<> IRB(&I);
2541 Value *Addr = I.getArgOperand(0);
2542 Type *Ty = IRB.getInt32Ty();
2543 unsigned Alignment = 1;
2544 Value *ShadowPtr, *OriginPtr;
2545 std::tie(ShadowPtr, OriginPtr) =
2546 getShadowOriginPtr(Addr, IRB, Ty, Alignment, /*isStore*/ false);
2548 if (ClCheckAccessAddress)
2549 insertShadowCheck(Addr, &I);
2551 Value *Shadow = IRB.CreateAlignedLoad(ShadowPtr, Alignment, "_ldmxcsr");
2553 MS.TrackOrigins ? IRB.CreateLoad(OriginPtr) : getCleanOrigin();
2554 insertShadowCheck(Shadow, Origin, &I);
2557 void handleMaskedStore(IntrinsicInst &I) {
2558 IRBuilder<> IRB(&I);
2559 Value *V = I.getArgOperand(0);
2560 Value *Addr = I.getArgOperand(1);
2561 unsigned Align = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
2562 Value *Mask = I.getArgOperand(3);
2563 Value *Shadow = getShadow(V);
2567 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
2568 Addr, IRB, Shadow->getType(), Align, /*isStore*/ true);
2570 if (ClCheckAccessAddress) {
2571 insertShadowCheck(Addr, &I);
2572 // Uninitialized mask is kind of like uninitialized address, but not as
2574 insertShadowCheck(Mask, &I);
2577 IRB.CreateMaskedStore(Shadow, ShadowPtr, Align, Mask);
2579 if (MS.TrackOrigins) {
2580 auto &DL = F.getParent()->getDataLayout();
2581 paintOrigin(IRB, getOrigin(V), OriginPtr,
2582 DL.getTypeStoreSize(Shadow->getType()),
2583 std::max(Align, kMinOriginAlignment));
2587 bool handleMaskedLoad(IntrinsicInst &I) {
2588 IRBuilder<> IRB(&I);
2589 Value *Addr = I.getArgOperand(0);
2590 unsigned Align = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
2591 Value *Mask = I.getArgOperand(2);
2592 Value *PassThru = I.getArgOperand(3);
2594 Type *ShadowTy = getShadowTy(&I);
2595 Value *ShadowPtr, *OriginPtr;
2596 if (PropagateShadow) {
2597 std::tie(ShadowPtr, OriginPtr) =
2598 getShadowOriginPtr(Addr, IRB, ShadowTy, Align, /*isStore*/ false);
2599 setShadow(&I, IRB.CreateMaskedLoad(ShadowPtr, Align, Mask,
2600 getShadow(PassThru), "_msmaskedld"));
2602 setShadow(&I, getCleanShadow(&I));
2605 if (ClCheckAccessAddress) {
2606 insertShadowCheck(Addr, &I);
2607 insertShadowCheck(Mask, &I);
2610 if (MS.TrackOrigins) {
2611 if (PropagateShadow) {
2612 // Choose between PassThru's and the loaded value's origins.
2613 Value *MaskedPassThruShadow = IRB.CreateAnd(
2614 getShadow(PassThru), IRB.CreateSExt(IRB.CreateNeg(Mask), ShadowTy));
2616 Value *Acc = IRB.CreateExtractElement(
2617 MaskedPassThruShadow, ConstantInt::get(IRB.getInt32Ty(), 0));
2618 for (int i = 1, N = PassThru->getType()->getVectorNumElements(); i < N;
2620 Value *More = IRB.CreateExtractElement(
2621 MaskedPassThruShadow, ConstantInt::get(IRB.getInt32Ty(), i));
2622 Acc = IRB.CreateOr(Acc, More);
2625 Value *Origin = IRB.CreateSelect(
2626 IRB.CreateICmpNE(Acc, Constant::getNullValue(Acc->getType())),
2627 getOrigin(PassThru), IRB.CreateLoad(OriginPtr));
2629 setOrigin(&I, Origin);
2631 setOrigin(&I, getCleanOrigin());
2638 void visitIntrinsicInst(IntrinsicInst &I) {
2639 switch (I.getIntrinsicID()) {
2640 case Intrinsic::bswap:
2643 case Intrinsic::masked_store:
2644 handleMaskedStore(I);
2646 case Intrinsic::masked_load:
2647 handleMaskedLoad(I);
2649 case Intrinsic::x86_sse_stmxcsr:
2652 case Intrinsic::x86_sse_ldmxcsr:
2655 case Intrinsic::x86_avx512_vcvtsd2usi64:
2656 case Intrinsic::x86_avx512_vcvtsd2usi32:
2657 case Intrinsic::x86_avx512_vcvtss2usi64:
2658 case Intrinsic::x86_avx512_vcvtss2usi32:
2659 case Intrinsic::x86_avx512_cvttss2usi64:
2660 case Intrinsic::x86_avx512_cvttss2usi:
2661 case Intrinsic::x86_avx512_cvttsd2usi64:
2662 case Intrinsic::x86_avx512_cvttsd2usi:
2663 case Intrinsic::x86_avx512_cvtusi2ss:
2664 case Intrinsic::x86_avx512_cvtusi642sd:
2665 case Intrinsic::x86_avx512_cvtusi642ss:
2666 case Intrinsic::x86_sse2_cvtsd2si64:
2667 case Intrinsic::x86_sse2_cvtsd2si:
2668 case Intrinsic::x86_sse2_cvtsd2ss:
2669 case Intrinsic::x86_sse2_cvttsd2si64:
2670 case Intrinsic::x86_sse2_cvttsd2si:
2671 case Intrinsic::x86_sse_cvtss2si64:
2672 case Intrinsic::x86_sse_cvtss2si:
2673 case Intrinsic::x86_sse_cvttss2si64:
2674 case Intrinsic::x86_sse_cvttss2si:
2675 handleVectorConvertIntrinsic(I, 1);
2677 case Intrinsic::x86_sse_cvtps2pi:
2678 case Intrinsic::x86_sse_cvttps2pi:
2679 handleVectorConvertIntrinsic(I, 2);
2682 case Intrinsic::x86_avx512_psll_w_512:
2683 case Intrinsic::x86_avx512_psll_d_512:
2684 case Intrinsic::x86_avx512_psll_q_512:
2685 case Intrinsic::x86_avx512_pslli_w_512:
2686 case Intrinsic::x86_avx512_pslli_d_512:
2687 case Intrinsic::x86_avx512_pslli_q_512:
2688 case Intrinsic::x86_avx512_psrl_w_512:
2689 case Intrinsic::x86_avx512_psrl_d_512:
2690 case Intrinsic::x86_avx512_psrl_q_512:
2691 case Intrinsic::x86_avx512_psra_w_512:
2692 case Intrinsic::x86_avx512_psra_d_512:
2693 case Intrinsic::x86_avx512_psra_q_512:
2694 case Intrinsic::x86_avx512_psrli_w_512:
2695 case Intrinsic::x86_avx512_psrli_d_512:
2696 case Intrinsic::x86_avx512_psrli_q_512:
2697 case Intrinsic::x86_avx512_psrai_w_512:
2698 case Intrinsic::x86_avx512_psrai_d_512:
2699 case Intrinsic::x86_avx512_psrai_q_512:
2700 case Intrinsic::x86_avx512_psra_q_256:
2701 case Intrinsic::x86_avx512_psra_q_128:
2702 case Intrinsic::x86_avx512_psrai_q_256:
2703 case Intrinsic::x86_avx512_psrai_q_128:
2704 case Intrinsic::x86_avx2_psll_w:
2705 case Intrinsic::x86_avx2_psll_d:
2706 case Intrinsic::x86_avx2_psll_q:
2707 case Intrinsic::x86_avx2_pslli_w:
2708 case Intrinsic::x86_avx2_pslli_d:
2709 case Intrinsic::x86_avx2_pslli_q:
2710 case Intrinsic::x86_avx2_psrl_w:
2711 case Intrinsic::x86_avx2_psrl_d:
2712 case Intrinsic::x86_avx2_psrl_q:
2713 case Intrinsic::x86_avx2_psra_w:
2714 case Intrinsic::x86_avx2_psra_d:
2715 case Intrinsic::x86_avx2_psrli_w:
2716 case Intrinsic::x86_avx2_psrli_d:
2717 case Intrinsic::x86_avx2_psrli_q:
2718 case Intrinsic::x86_avx2_psrai_w:
2719 case Intrinsic::x86_avx2_psrai_d:
2720 case Intrinsic::x86_sse2_psll_w:
2721 case Intrinsic::x86_sse2_psll_d:
2722 case Intrinsic::x86_sse2_psll_q:
2723 case Intrinsic::x86_sse2_pslli_w:
2724 case Intrinsic::x86_sse2_pslli_d:
2725 case Intrinsic::x86_sse2_pslli_q:
2726 case Intrinsic::x86_sse2_psrl_w:
2727 case Intrinsic::x86_sse2_psrl_d:
2728 case Intrinsic::x86_sse2_psrl_q:
2729 case Intrinsic::x86_sse2_psra_w:
2730 case Intrinsic::x86_sse2_psra_d:
2731 case Intrinsic::x86_sse2_psrli_w:
2732 case Intrinsic::x86_sse2_psrli_d:
2733 case Intrinsic::x86_sse2_psrli_q:
2734 case Intrinsic::x86_sse2_psrai_w:
2735 case Intrinsic::x86_sse2_psrai_d:
2736 case Intrinsic::x86_mmx_psll_w:
2737 case Intrinsic::x86_mmx_psll_d:
2738 case Intrinsic::x86_mmx_psll_q:
2739 case Intrinsic::x86_mmx_pslli_w:
2740 case Intrinsic::x86_mmx_pslli_d:
2741 case Intrinsic::x86_mmx_pslli_q:
2742 case Intrinsic::x86_mmx_psrl_w:
2743 case Intrinsic::x86_mmx_psrl_d:
2744 case Intrinsic::x86_mmx_psrl_q:
2745 case Intrinsic::x86_mmx_psra_w:
2746 case Intrinsic::x86_mmx_psra_d:
2747 case Intrinsic::x86_mmx_psrli_w:
2748 case Intrinsic::x86_mmx_psrli_d:
2749 case Intrinsic::x86_mmx_psrli_q:
2750 case Intrinsic::x86_mmx_psrai_w:
2751 case Intrinsic::x86_mmx_psrai_d:
2752 handleVectorShiftIntrinsic(I, /* Variable */ false);
2754 case Intrinsic::x86_avx2_psllv_d:
2755 case Intrinsic::x86_avx2_psllv_d_256:
2756 case Intrinsic::x86_avx512_psllv_d_512:
2757 case Intrinsic::x86_avx2_psllv_q:
2758 case Intrinsic::x86_avx2_psllv_q_256:
2759 case Intrinsic::x86_avx512_psllv_q_512:
2760 case Intrinsic::x86_avx2_psrlv_d:
2761 case Intrinsic::x86_avx2_psrlv_d_256:
2762 case Intrinsic::x86_avx512_psrlv_d_512:
2763 case Intrinsic::x86_avx2_psrlv_q:
2764 case Intrinsic::x86_avx2_psrlv_q_256:
2765 case Intrinsic::x86_avx512_psrlv_q_512:
2766 case Intrinsic::x86_avx2_psrav_d:
2767 case Intrinsic::x86_avx2_psrav_d_256:
2768 case Intrinsic::x86_avx512_psrav_d_512:
2769 case Intrinsic::x86_avx512_psrav_q_128:
2770 case Intrinsic::x86_avx512_psrav_q_256:
2771 case Intrinsic::x86_avx512_psrav_q_512:
2772 handleVectorShiftIntrinsic(I, /* Variable */ true);
2775 case Intrinsic::x86_sse2_packsswb_128:
2776 case Intrinsic::x86_sse2_packssdw_128:
2777 case Intrinsic::x86_sse2_packuswb_128:
2778 case Intrinsic::x86_sse41_packusdw:
2779 case Intrinsic::x86_avx2_packsswb:
2780 case Intrinsic::x86_avx2_packssdw:
2781 case Intrinsic::x86_avx2_packuswb:
2782 case Intrinsic::x86_avx2_packusdw:
2783 handleVectorPackIntrinsic(I);
2786 case Intrinsic::x86_mmx_packsswb:
2787 case Intrinsic::x86_mmx_packuswb:
2788 handleVectorPackIntrinsic(I, 16);
2791 case Intrinsic::x86_mmx_packssdw:
2792 handleVectorPackIntrinsic(I, 32);
2795 case Intrinsic::x86_mmx_psad_bw:
2796 case Intrinsic::x86_sse2_psad_bw:
2797 case Intrinsic::x86_avx2_psad_bw:
2798 handleVectorSadIntrinsic(I);
2801 case Intrinsic::x86_sse2_pmadd_wd:
2802 case Intrinsic::x86_avx2_pmadd_wd:
2803 case Intrinsic::x86_ssse3_pmadd_ub_sw_128:
2804 case Intrinsic::x86_avx2_pmadd_ub_sw:
2805 handleVectorPmaddIntrinsic(I);
2808 case Intrinsic::x86_ssse3_pmadd_ub_sw:
2809 handleVectorPmaddIntrinsic(I, 8);
2812 case Intrinsic::x86_mmx_pmadd_wd:
2813 handleVectorPmaddIntrinsic(I, 16);
2816 case Intrinsic::x86_sse_cmp_ss:
2817 case Intrinsic::x86_sse2_cmp_sd:
2818 case Intrinsic::x86_sse_comieq_ss:
2819 case Intrinsic::x86_sse_comilt_ss:
2820 case Intrinsic::x86_sse_comile_ss:
2821 case Intrinsic::x86_sse_comigt_ss:
2822 case Intrinsic::x86_sse_comige_ss:
2823 case Intrinsic::x86_sse_comineq_ss:
2824 case Intrinsic::x86_sse_ucomieq_ss:
2825 case Intrinsic::x86_sse_ucomilt_ss:
2826 case Intrinsic::x86_sse_ucomile_ss:
2827 case Intrinsic::x86_sse_ucomigt_ss:
2828 case Intrinsic::x86_sse_ucomige_ss:
2829 case Intrinsic::x86_sse_ucomineq_ss:
2830 case Intrinsic::x86_sse2_comieq_sd:
2831 case Intrinsic::x86_sse2_comilt_sd:
2832 case Intrinsic::x86_sse2_comile_sd:
2833 case Intrinsic::x86_sse2_comigt_sd:
2834 case Intrinsic::x86_sse2_comige_sd:
2835 case Intrinsic::x86_sse2_comineq_sd:
2836 case Intrinsic::x86_sse2_ucomieq_sd:
2837 case Intrinsic::x86_sse2_ucomilt_sd:
2838 case Intrinsic::x86_sse2_ucomile_sd:
2839 case Intrinsic::x86_sse2_ucomigt_sd:
2840 case Intrinsic::x86_sse2_ucomige_sd:
2841 case Intrinsic::x86_sse2_ucomineq_sd:
2842 handleVectorCompareScalarIntrinsic(I);
2845 case Intrinsic::x86_sse_cmp_ps:
2846 case Intrinsic::x86_sse2_cmp_pd:
2847 // FIXME: For x86_avx_cmp_pd_256 and x86_avx_cmp_ps_256 this function
2848 // generates reasonably looking IR that fails in the backend with "Do not
2849 // know how to split the result of this operator!".
2850 handleVectorComparePackedIntrinsic(I);
2854 if (!handleUnknownIntrinsic(I))
2855 visitInstruction(I);
2860 void visitCallSite(CallSite CS) {
2861 Instruction &I = *CS.getInstruction();
2862 assert(!I.getMetadata("nosanitize"));
2863 assert((CS.isCall() || CS.isInvoke()) && "Unknown type of CallSite");
2865 CallInst *Call = cast<CallInst>(&I);
2867 // For inline asm, do the usual thing: check argument shadow and mark all
2868 // outputs as clean. Note that any side effects of the inline asm that are
2869 // not immediately visible in its constraints are not handled.
2870 if (Call->isInlineAsm()) {
2871 if (ClHandleAsmConservative)
2872 visitAsmInstruction(I);
2874 visitInstruction(I);
2878 assert(!isa<IntrinsicInst>(&I) && "intrinsics are handled elsewhere");
2880 // We are going to insert code that relies on the fact that the callee
2881 // will become a non-readonly function after it is instrumented by us. To
2882 // prevent this code from being optimized out, mark that function
2883 // non-readonly in advance.
2884 if (Function *Func = Call->getCalledFunction()) {
2885 // Clear out readonly/readnone attributes.
2887 B.addAttribute(Attribute::ReadOnly)
2888 .addAttribute(Attribute::ReadNone);
2889 Func->removeAttributes(AttributeList::FunctionIndex, B);
2892 maybeMarkSanitizerLibraryCallNoBuiltin(Call, TLI);
2894 IRBuilder<> IRB(&I);
2896 unsigned ArgOffset = 0;
2897 LLVM_DEBUG(dbgs() << " CallSite: " << I << "\n");
2898 for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end();
2899 ArgIt != End; ++ArgIt) {
2901 unsigned i = ArgIt - CS.arg_begin();
2902 if (!A->getType()->isSized()) {
2903 LLVM_DEBUG(dbgs() << "Arg " << i << " is not sized: " << I << "\n");
2907 Value *Store = nullptr;
2908 // Compute the Shadow for arg even if it is ByVal, because
2909 // in that case getShadow() will copy the actual arg shadow to
2910 // __msan_param_tls.
2911 Value *ArgShadow = getShadow(A);
2912 Value *ArgShadowBase = getShadowPtrForArgument(A, IRB, ArgOffset);
2913 LLVM_DEBUG(dbgs() << " Arg#" << i << ": " << *A
2914 << " Shadow: " << *ArgShadow << "\n");
2915 bool ArgIsInitialized = false;
2916 const DataLayout &DL = F.getParent()->getDataLayout();
2917 if (CS.paramHasAttr(i, Attribute::ByVal)) {
2918 assert(A->getType()->isPointerTy() &&
2919 "ByVal argument is not a pointer!");
2920 Size = DL.getTypeAllocSize(A->getType()->getPointerElementType());
2921 if (ArgOffset + Size > kParamTLSSize) break;
2922 unsigned ParamAlignment = CS.getParamAlignment(i);
2923 unsigned Alignment = std::min(ParamAlignment, kShadowTLSAlignment);
2924 Value *AShadowPtr = getShadowOriginPtr(A, IRB, IRB.getInt8Ty(),
2925 Alignment, /*isStore*/ false)
2928 Store = IRB.CreateMemCpy(ArgShadowBase, Alignment, AShadowPtr,
2931 Size = DL.getTypeAllocSize(A->getType());
2932 if (ArgOffset + Size > kParamTLSSize) break;
2933 Store = IRB.CreateAlignedStore(ArgShadow, ArgShadowBase,
2934 kShadowTLSAlignment);
2935 Constant *Cst = dyn_cast<Constant>(ArgShadow);
2936 if (Cst && Cst->isNullValue()) ArgIsInitialized = true;
2938 if (MS.TrackOrigins && !ArgIsInitialized)
2939 IRB.CreateStore(getOrigin(A),
2940 getOriginPtrForArgument(A, IRB, ArgOffset));
2942 assert(Size != 0 && Store != nullptr);
2943 LLVM_DEBUG(dbgs() << " Param:" << *Store << "\n");
2944 ArgOffset += alignTo(Size, 8);
2946 LLVM_DEBUG(dbgs() << " done with call args\n");
2949 cast<FunctionType>(CS.getCalledValue()->getType()->getContainedType(0));
2950 if (FT->isVarArg()) {
2951 VAHelper->visitCallSite(CS, IRB);
2954 // Now, get the shadow for the RetVal.
2955 if (!I.getType()->isSized()) return;
2956 // Don't emit the epilogue for musttail call returns.
2957 if (CS.isCall() && cast<CallInst>(&I)->isMustTailCall()) return;
2958 IRBuilder<> IRBBefore(&I);
2959 // Until we have full dynamic coverage, make sure the retval shadow is 0.
2960 Value *Base = getShadowPtrForRetval(&I, IRBBefore);
2961 IRBBefore.CreateAlignedStore(getCleanShadow(&I), Base, kShadowTLSAlignment);
2962 BasicBlock::iterator NextInsn;
2964 NextInsn = ++I.getIterator();
2965 assert(NextInsn != I.getParent()->end());
2967 BasicBlock *NormalDest = cast<InvokeInst>(&I)->getNormalDest();
2968 if (!NormalDest->getSinglePredecessor()) {
2969 // FIXME: this case is tricky, so we are just conservative here.
2970 // Perhaps we need to split the edge between this BB and NormalDest,
2971 // but a naive attempt to use SplitEdge leads to a crash.
2972 setShadow(&I, getCleanShadow(&I));
2973 setOrigin(&I, getCleanOrigin());
2976 // FIXME: NextInsn is likely in a basic block that has not been visited yet.
2977 // Anything inserted there will be instrumented by MSan later!
2978 NextInsn = NormalDest->getFirstInsertionPt();
2979 assert(NextInsn != NormalDest->end() &&
2980 "Could not find insertion point for retval shadow load");
2982 IRBuilder<> IRBAfter(&*NextInsn);
2983 Value *RetvalShadow =
2984 IRBAfter.CreateAlignedLoad(getShadowPtrForRetval(&I, IRBAfter),
2985 kShadowTLSAlignment, "_msret");
2986 setShadow(&I, RetvalShadow);
2987 if (MS.TrackOrigins)
2988 setOrigin(&I, IRBAfter.CreateLoad(getOriginPtrForRetval(IRBAfter)));
2991 bool isAMustTailRetVal(Value *RetVal) {
2992 if (auto *I = dyn_cast<BitCastInst>(RetVal)) {
2993 RetVal = I->getOperand(0);
2995 if (auto *I = dyn_cast<CallInst>(RetVal)) {
2996 return I->isMustTailCall();
3001 void visitReturnInst(ReturnInst &I) {
3002 IRBuilder<> IRB(&I);
3003 Value *RetVal = I.getReturnValue();
3004 if (!RetVal) return;
3005 // Don't emit the epilogue for musttail call returns.
3006 if (isAMustTailRetVal(RetVal)) return;
3007 Value *ShadowPtr = getShadowPtrForRetval(RetVal, IRB);
3008 if (CheckReturnValue) {
3009 insertShadowCheck(RetVal, &I);
3010 Value *Shadow = getCleanShadow(RetVal);
3011 IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment);
3013 Value *Shadow = getShadow(RetVal);
3014 IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment);
3015 if (MS.TrackOrigins)
3016 IRB.CreateStore(getOrigin(RetVal), getOriginPtrForRetval(IRB));
3020 void visitPHINode(PHINode &I) {
3021 IRBuilder<> IRB(&I);
3022 if (!PropagateShadow) {
3023 setShadow(&I, getCleanShadow(&I));
3024 setOrigin(&I, getCleanOrigin());
3028 ShadowPHINodes.push_back(&I);
3029 setShadow(&I, IRB.CreatePHI(getShadowTy(&I), I.getNumIncomingValues(),
3031 if (MS.TrackOrigins)
3032 setOrigin(&I, IRB.CreatePHI(MS.OriginTy, I.getNumIncomingValues(),
3036 void visitAllocaInst(AllocaInst &I) {
3037 setShadow(&I, getCleanShadow(&I));
3038 setOrigin(&I, getCleanOrigin());
3039 IRBuilder<> IRB(I.getNextNode());
3040 const DataLayout &DL = F.getParent()->getDataLayout();
3041 uint64_t TypeSize = DL.getTypeAllocSize(I.getAllocatedType());
3042 Value *Len = ConstantInt::get(MS.IntptrTy, TypeSize);
3043 if (I.isArrayAllocation())
3044 Len = IRB.CreateMul(Len, I.getArraySize());
3045 if (PoisonStack && ClPoisonStackWithCall) {
3046 IRB.CreateCall(MS.MsanPoisonStackFn,
3047 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len});
3049 Value *ShadowBase = getShadowOriginPtr(&I, IRB, IRB.getInt8Ty(),
3050 I.getAlignment(), /*isStore*/ true)
3053 Value *PoisonValue = IRB.getInt8(PoisonStack ? ClPoisonStackPattern : 0);
3054 IRB.CreateMemSet(ShadowBase, PoisonValue, Len, I.getAlignment());
3057 if (PoisonStack && MS.TrackOrigins) {
3058 SmallString<2048> StackDescriptionStorage;
3059 raw_svector_ostream StackDescription(StackDescriptionStorage);
3060 // We create a string with a description of the stack allocation and
3061 // pass it into __msan_set_alloca_origin.
3062 // It will be printed by the run-time if stack-originated UMR is found.
3063 // The first 4 bytes of the string are set to '----' and will be replaced
3064 // by __msan_va_arg_overflow_size_tls at the first call.
3065 StackDescription << "----" << I.getName() << "@" << F.getName();
3067 createPrivateNonConstGlobalForString(*F.getParent(),
3068 StackDescription.str());
3070 IRB.CreateCall(MS.MsanSetAllocaOrigin4Fn,
3071 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len,
3072 IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy()),
3073 IRB.CreatePointerCast(&F, MS.IntptrTy)});
3077 void visitSelectInst(SelectInst& I) {
3078 IRBuilder<> IRB(&I);
3079 // a = select b, c, d
3080 Value *B = I.getCondition();
3081 Value *C = I.getTrueValue();
3082 Value *D = I.getFalseValue();
3083 Value *Sb = getShadow(B);
3084 Value *Sc = getShadow(C);
3085 Value *Sd = getShadow(D);
3087 // Result shadow if condition shadow is 0.
3088 Value *Sa0 = IRB.CreateSelect(B, Sc, Sd);
3090 if (I.getType()->isAggregateType()) {
3091 // To avoid "sign extending" i1 to an arbitrary aggregate type, we just do
3092 // an extra "select". This results in much more compact IR.
3093 // Sa = select Sb, poisoned, (select b, Sc, Sd)
3094 Sa1 = getPoisonedShadow(getShadowTy(I.getType()));
3096 // Sa = select Sb, [ (c^d) | Sc | Sd ], [ b ? Sc : Sd ]
3097 // If Sb (condition is poisoned), look for bits in c and d that are equal
3098 // and both unpoisoned.
3099 // If !Sb (condition is unpoisoned), simply pick one of Sc and Sd.
3101 // Cast arguments to shadow-compatible type.
3102 C = CreateAppToShadowCast(IRB, C);
3103 D = CreateAppToShadowCast(IRB, D);
3105 // Result shadow if condition shadow is 1.
3106 Sa1 = IRB.CreateOr(IRB.CreateXor(C, D), IRB.CreateOr(Sc, Sd));
3108 Value *Sa = IRB.CreateSelect(Sb, Sa1, Sa0, "_msprop_select");
3110 if (MS.TrackOrigins) {
3111 // Origins are always i32, so any vector conditions must be flattened.
3112 // FIXME: consider tracking vector origins for app vectors?
3113 if (B->getType()->isVectorTy()) {
3114 Type *FlatTy = getShadowTyNoVec(B->getType());
3115 B = IRB.CreateICmpNE(IRB.CreateBitCast(B, FlatTy),
3116 ConstantInt::getNullValue(FlatTy));
3117 Sb = IRB.CreateICmpNE(IRB.CreateBitCast(Sb, FlatTy),
3118 ConstantInt::getNullValue(FlatTy));
3120 // a = select b, c, d
3121 // Oa = Sb ? Ob : (b ? Oc : Od)
3123 &I, IRB.CreateSelect(Sb, getOrigin(I.getCondition()),
3124 IRB.CreateSelect(B, getOrigin(I.getTrueValue()),
3125 getOrigin(I.getFalseValue()))));
3129 void visitLandingPadInst(LandingPadInst &I) {
3131 // See https://github.com/google/sanitizers/issues/504
3132 setShadow(&I, getCleanShadow(&I));
3133 setOrigin(&I, getCleanOrigin());
3136 void visitCatchSwitchInst(CatchSwitchInst &I) {
3137 setShadow(&I, getCleanShadow(&I));
3138 setOrigin(&I, getCleanOrigin());
3141 void visitFuncletPadInst(FuncletPadInst &I) {
3142 setShadow(&I, getCleanShadow(&I));
3143 setOrigin(&I, getCleanOrigin());
3146 void visitGetElementPtrInst(GetElementPtrInst &I) {
3150 void visitExtractValueInst(ExtractValueInst &I) {
3151 IRBuilder<> IRB(&I);
3152 Value *Agg = I.getAggregateOperand();
3153 LLVM_DEBUG(dbgs() << "ExtractValue: " << I << "\n");
3154 Value *AggShadow = getShadow(Agg);
3155 LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n");
3156 Value *ResShadow = IRB.CreateExtractValue(AggShadow, I.getIndices());
3157 LLVM_DEBUG(dbgs() << " ResShadow: " << *ResShadow << "\n");
3158 setShadow(&I, ResShadow);
3159 setOriginForNaryOp(I);
3162 void visitInsertValueInst(InsertValueInst &I) {
3163 IRBuilder<> IRB(&I);
3164 LLVM_DEBUG(dbgs() << "InsertValue: " << I << "\n");
3165 Value *AggShadow = getShadow(I.getAggregateOperand());
3166 Value *InsShadow = getShadow(I.getInsertedValueOperand());
3167 LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n");
3168 LLVM_DEBUG(dbgs() << " InsShadow: " << *InsShadow << "\n");
3169 Value *Res = IRB.CreateInsertValue(AggShadow, InsShadow, I.getIndices());
3170 LLVM_DEBUG(dbgs() << " Res: " << *Res << "\n");
3172 setOriginForNaryOp(I);
3175 void dumpInst(Instruction &I) {
3176 if (CallInst *CI = dyn_cast<CallInst>(&I)) {
3177 errs() << "ZZZ call " << CI->getCalledFunction()->getName() << "\n";
3179 errs() << "ZZZ " << I.getOpcodeName() << "\n";
3181 errs() << "QQQ " << I << "\n";
3184 void visitResumeInst(ResumeInst &I) {
3185 LLVM_DEBUG(dbgs() << "Resume: " << I << "\n");
3186 // Nothing to do here.
3189 void visitCleanupReturnInst(CleanupReturnInst &CRI) {
3190 LLVM_DEBUG(dbgs() << "CleanupReturn: " << CRI << "\n");
3191 // Nothing to do here.
3194 void visitCatchReturnInst(CatchReturnInst &CRI) {
3195 LLVM_DEBUG(dbgs() << "CatchReturn: " << CRI << "\n");
3196 // Nothing to do here.
3199 void visitAsmInstruction(Instruction &I) {
3200 // Conservative inline assembly handling: check for poisoned shadow of
3201 // asm() arguments, then unpoison the result and all the memory locations
3202 // pointed to by those arguments.
3203 CallInst *CI = dyn_cast<CallInst>(&I);
3205 for (size_t i = 0, n = CI->getNumOperands(); i < n; i++) {
3206 Value *Operand = CI->getOperand(i);
3207 if (Operand->getType()->isSized())
3208 insertShadowCheck(Operand, &I);
3210 setShadow(&I, getCleanShadow(&I));
3211 setOrigin(&I, getCleanOrigin());
3212 IRBuilder<> IRB(&I);
3213 IRB.SetInsertPoint(I.getNextNode());
3214 for (size_t i = 0, n = CI->getNumOperands(); i < n; i++) {
3215 Value *Operand = CI->getOperand(i);
3216 Type *OpType = Operand->getType();
3217 if (!OpType->isPointerTy())
3219 Type *ElType = OpType->getPointerElementType();
3220 if (!ElType->isSized())
3222 Value *ShadowPtr, *OriginPtr;
3223 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
3224 Operand, IRB, ElType, /*Alignment*/ 1, /*isStore*/ true);
3225 Value *CShadow = getCleanShadow(ElType);
3228 IRB.CreatePointerCast(ShadowPtr, CShadow->getType()->getPointerTo()));
3232 void visitInstruction(Instruction &I) {
3233 // Everything else: stop propagating and check for poisoned shadow.
3234 if (ClDumpStrictInstructions)
3236 LLVM_DEBUG(dbgs() << "DEFAULT: " << I << "\n");
3237 for (size_t i = 0, n = I.getNumOperands(); i < n; i++) {
3238 Value *Operand = I.getOperand(i);
3239 if (Operand->getType()->isSized())
3240 insertShadowCheck(Operand, &I);
3242 setShadow(&I, getCleanShadow(&I));
3243 setOrigin(&I, getCleanOrigin());
3247 /// AMD64-specific implementation of VarArgHelper.
3248 struct VarArgAMD64Helper : public VarArgHelper {
3249 // An unfortunate workaround for asymmetric lowering of va_arg stuff.
3250 // See a comment in visitCallSite for more details.
3251 static const unsigned AMD64GpEndOffset = 48; // AMD64 ABI Draft 0.99.6 p3.5.7
3252 static const unsigned AMD64FpEndOffset = 176;
3255 MemorySanitizer &MS;
3256 MemorySanitizerVisitor &MSV;
3257 Value *VAArgTLSCopy = nullptr;
3258 Value *VAArgOverflowSize = nullptr;
3260 SmallVector<CallInst*, 16> VAStartInstrumentationList;
3262 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
3264 VarArgAMD64Helper(Function &F, MemorySanitizer &MS,
3265 MemorySanitizerVisitor &MSV) : F(F), MS(MS), MSV(MSV) {}
3267 ArgKind classifyArgument(Value* arg) {
3268 // A very rough approximation of X86_64 argument classification rules.
3269 Type *T = arg->getType();
3270 if (T->isFPOrFPVectorTy() || T->isX86_MMXTy())
3271 return AK_FloatingPoint;
3272 if (T->isIntegerTy() && T->getPrimitiveSizeInBits() <= 64)
3273 return AK_GeneralPurpose;
3274 if (T->isPointerTy())
3275 return AK_GeneralPurpose;
3279 // For VarArg functions, store the argument shadow in an ABI-specific format
3280 // that corresponds to va_list layout.
3281 // We do this because Clang lowers va_arg in the frontend, and this pass
3282 // only sees the low level code that deals with va_list internals.
3283 // A much easier alternative (provided that Clang emits va_arg instructions)
3284 // would have been to associate each live instance of va_list with a copy of
3285 // MSanParamTLS, and extract shadow on va_arg() call in the argument list
3287 void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {
3288 unsigned GpOffset = 0;
3289 unsigned FpOffset = AMD64GpEndOffset;
3290 unsigned OverflowOffset = AMD64FpEndOffset;
3291 const DataLayout &DL = F.getParent()->getDataLayout();
3292 for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end();
3293 ArgIt != End; ++ArgIt) {
3295 unsigned ArgNo = CS.getArgumentNo(ArgIt);
3296 bool IsFixed = ArgNo < CS.getFunctionType()->getNumParams();
3297 bool IsByVal = CS.paramHasAttr(ArgNo, Attribute::ByVal);
3299 // ByVal arguments always go to the overflow area.
3300 // Fixed arguments passed through the overflow area will be stepped
3301 // over by va_start, so don't count them towards the offset.
3304 assert(A->getType()->isPointerTy());
3305 Type *RealTy = A->getType()->getPointerElementType();
3306 uint64_t ArgSize = DL.getTypeAllocSize(RealTy);
3308 getShadowPtrForVAArgument(RealTy, IRB, OverflowOffset);
3309 OverflowOffset += alignTo(ArgSize, 8);
3310 Value *ShadowPtr, *OriginPtr;
3311 std::tie(ShadowPtr, OriginPtr) =
3312 MSV.getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), kShadowTLSAlignment,
3315 IRB.CreateMemCpy(ShadowBase, kShadowTLSAlignment, ShadowPtr,
3316 kShadowTLSAlignment, ArgSize);
3318 ArgKind AK = classifyArgument(A);
3319 if (AK == AK_GeneralPurpose && GpOffset >= AMD64GpEndOffset)
3321 if (AK == AK_FloatingPoint && FpOffset >= AMD64FpEndOffset)
3325 case AK_GeneralPurpose:
3326 ShadowBase = getShadowPtrForVAArgument(A->getType(), IRB, GpOffset);
3329 case AK_FloatingPoint:
3330 ShadowBase = getShadowPtrForVAArgument(A->getType(), IRB, FpOffset);
3336 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
3338 getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset);
3339 OverflowOffset += alignTo(ArgSize, 8);
3341 // Take fixed arguments into account for GpOffset and FpOffset,
3342 // but don't actually store shadows for them.
3345 IRB.CreateAlignedStore(MSV.getShadow(A), ShadowBase,
3346 kShadowTLSAlignment);
3349 Constant *OverflowSize =
3350 ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AMD64FpEndOffset);
3351 IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
3354 /// Compute the shadow address for a given va_arg.
3355 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
3357 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
3358 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
3359 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
3363 void unpoisonVAListTagForInst(IntrinsicInst &I) {
3364 IRBuilder<> IRB(&I);
3365 Value *VAListTag = I.getArgOperand(0);
3366 Value *ShadowPtr, *OriginPtr;
3367 unsigned Alignment = 8;
3368 std::tie(ShadowPtr, OriginPtr) =
3369 MSV.getShadowOriginPtr(VAListTag, IRB, IRB.getInt8Ty(), Alignment,
3372 // Unpoison the whole __va_list_tag.
3373 // FIXME: magic ABI constants.
3374 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
3375 /* size */ 24, Alignment, false);
3376 // We shouldn't need to zero out the origins, as they're only checked for
3380 void visitVAStartInst(VAStartInst &I) override {
3381 if (F.getCallingConv() == CallingConv::Win64)
3383 VAStartInstrumentationList.push_back(&I);
3384 unpoisonVAListTagForInst(I);
3387 void visitVACopyInst(VACopyInst &I) override {
3388 if (F.getCallingConv() == CallingConv::Win64) return;
3389 unpoisonVAListTagForInst(I);
3392 void finalizeInstrumentation() override {
3393 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
3394 "finalizeInstrumentation called twice");
3395 if (!VAStartInstrumentationList.empty()) {
3396 // If there is a va_start in this function, make a backup copy of
3397 // va_arg_tls somewhere in the function entry block.
3398 IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
3399 VAArgOverflowSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS);
3401 IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset),
3403 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
3404 IRB.CreateMemCpy(VAArgTLSCopy, 8, MS.VAArgTLS, 8, CopySize);
3407 // Instrument va_start.
3408 // Copy va_list shadow from the backup copy of the TLS contents.
3409 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
3410 CallInst *OrigInst = VAStartInstrumentationList[i];
3411 IRBuilder<> IRB(OrigInst->getNextNode());
3412 Value *VAListTag = OrigInst->getArgOperand(0);
3414 Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr(
3415 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
3416 ConstantInt::get(MS.IntptrTy, 16)),
3417 PointerType::get(Type::getInt64PtrTy(*MS.C), 0));
3418 Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrPtr);
3419 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
3420 unsigned Alignment = 16;
3421 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
3422 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
3423 Alignment, /*isStore*/ true);
3424 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
3426 Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr(
3427 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
3428 ConstantInt::get(MS.IntptrTy, 8)),
3429 PointerType::get(Type::getInt64PtrTy(*MS.C), 0));
3430 Value *OverflowArgAreaPtr = IRB.CreateLoad(OverflowArgAreaPtrPtr);
3431 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
3432 std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) =
3433 MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.getInt8Ty(),
3434 Alignment, /*isStore*/ true);
3435 Value *SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSCopy,
3437 IRB.CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
3443 /// MIPS64-specific implementation of VarArgHelper.
3444 struct VarArgMIPS64Helper : public VarArgHelper {
3446 MemorySanitizer &MS;
3447 MemorySanitizerVisitor &MSV;
3448 Value *VAArgTLSCopy = nullptr;
3449 Value *VAArgSize = nullptr;
3451 SmallVector<CallInst*, 16> VAStartInstrumentationList;
3453 VarArgMIPS64Helper(Function &F, MemorySanitizer &MS,
3454 MemorySanitizerVisitor &MSV) : F(F), MS(MS), MSV(MSV) {}
3456 void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {
3457 unsigned VAArgOffset = 0;
3458 const DataLayout &DL = F.getParent()->getDataLayout();
3459 for (CallSite::arg_iterator ArgIt = CS.arg_begin() +
3460 CS.getFunctionType()->getNumParams(), End = CS.arg_end();
3461 ArgIt != End; ++ArgIt) {
3462 Triple TargetTriple(F.getParent()->getTargetTriple());
3465 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
3466 if (TargetTriple.getArch() == Triple::mips64) {
3467 // Adjusting the shadow for argument with size < 8 to match the placement
3468 // of bits in big endian system
3470 VAArgOffset += (8 - ArgSize);
3472 Base = getShadowPtrForVAArgument(A->getType(), IRB, VAArgOffset);
3473 VAArgOffset += ArgSize;
3474 VAArgOffset = alignTo(VAArgOffset, 8);
3475 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
3478 Constant *TotalVAArgSize = ConstantInt::get(IRB.getInt64Ty(), VAArgOffset);
3479 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
3480 // a new class member i.e. it is the total size of all VarArgs.
3481 IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
3484 /// Compute the shadow address for a given va_arg.
3485 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
3487 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
3488 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
3489 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
3493 void visitVAStartInst(VAStartInst &I) override {
3494 IRBuilder<> IRB(&I);
3495 VAStartInstrumentationList.push_back(&I);
3496 Value *VAListTag = I.getArgOperand(0);
3497 Value *ShadowPtr, *OriginPtr;
3498 unsigned Alignment = 8;
3499 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
3500 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
3501 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
3502 /* size */ 8, Alignment, false);
3505 void visitVACopyInst(VACopyInst &I) override {
3506 IRBuilder<> IRB(&I);
3507 VAStartInstrumentationList.push_back(&I);
3508 Value *VAListTag = I.getArgOperand(0);
3509 Value *ShadowPtr, *OriginPtr;
3510 unsigned Alignment = 8;
3511 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
3512 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
3513 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
3514 /* size */ 8, Alignment, false);
3517 void finalizeInstrumentation() override {
3518 assert(!VAArgSize && !VAArgTLSCopy &&
3519 "finalizeInstrumentation called twice");
3520 IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
3521 VAArgSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS);
3522 Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0),
3525 if (!VAStartInstrumentationList.empty()) {
3526 // If there is a va_start in this function, make a backup copy of
3527 // va_arg_tls somewhere in the function entry block.
3528 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
3529 IRB.CreateMemCpy(VAArgTLSCopy, 8, MS.VAArgTLS, 8, CopySize);
3532 // Instrument va_start.
3533 // Copy va_list shadow from the backup copy of the TLS contents.
3534 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
3535 CallInst *OrigInst = VAStartInstrumentationList[i];
3536 IRBuilder<> IRB(OrigInst->getNextNode());
3537 Value *VAListTag = OrigInst->getArgOperand(0);
3538 Value *RegSaveAreaPtrPtr =
3539 IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
3540 PointerType::get(Type::getInt64PtrTy(*MS.C), 0));
3541 Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrPtr);
3542 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
3543 unsigned Alignment = 8;
3544 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
3545 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
3546 Alignment, /*isStore*/ true);
3547 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
3553 /// AArch64-specific implementation of VarArgHelper.
3554 struct VarArgAArch64Helper : public VarArgHelper {
3555 static const unsigned kAArch64GrArgSize = 64;
3556 static const unsigned kAArch64VrArgSize = 128;
3558 static const unsigned AArch64GrBegOffset = 0;
3559 static const unsigned AArch64GrEndOffset = kAArch64GrArgSize;
3560 // Make VR space aligned to 16 bytes.
3561 static const unsigned AArch64VrBegOffset = AArch64GrEndOffset;
3562 static const unsigned AArch64VrEndOffset = AArch64VrBegOffset
3563 + kAArch64VrArgSize;
3564 static const unsigned AArch64VAEndOffset = AArch64VrEndOffset;
3567 MemorySanitizer &MS;
3568 MemorySanitizerVisitor &MSV;
3569 Value *VAArgTLSCopy = nullptr;
3570 Value *VAArgOverflowSize = nullptr;
3572 SmallVector<CallInst*, 16> VAStartInstrumentationList;
3574 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
3576 VarArgAArch64Helper(Function &F, MemorySanitizer &MS,
3577 MemorySanitizerVisitor &MSV) : F(F), MS(MS), MSV(MSV) {}
3579 ArgKind classifyArgument(Value* arg) {
3580 Type *T = arg->getType();
3581 if (T->isFPOrFPVectorTy())
3582 return AK_FloatingPoint;
3583 if ((T->isIntegerTy() && T->getPrimitiveSizeInBits() <= 64)
3584 || (T->isPointerTy()))
3585 return AK_GeneralPurpose;
3589 // The instrumentation stores the argument shadow in a non ABI-specific
3590 // format because it does not know which argument is named (since Clang,
3591 // like x86_64 case, lowers the va_args in the frontend and this pass only
3592 // sees the low level code that deals with va_list internals).
3593 // The first seven GR registers are saved in the first 56 bytes of the
3594 // va_arg tls arra, followers by the first 8 FP/SIMD registers, and then
3595 // the remaining arguments.
3596 // Using constant offset within the va_arg TLS array allows fast copy
3597 // in the finalize instrumentation.
3598 void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {
3599 unsigned GrOffset = AArch64GrBegOffset;
3600 unsigned VrOffset = AArch64VrBegOffset;
3601 unsigned OverflowOffset = AArch64VAEndOffset;
3603 const DataLayout &DL = F.getParent()->getDataLayout();
3604 for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end();
3605 ArgIt != End; ++ArgIt) {
3607 unsigned ArgNo = CS.getArgumentNo(ArgIt);
3608 bool IsFixed = ArgNo < CS.getFunctionType()->getNumParams();
3609 ArgKind AK = classifyArgument(A);
3610 if (AK == AK_GeneralPurpose && GrOffset >= AArch64GrEndOffset)
3612 if (AK == AK_FloatingPoint && VrOffset >= AArch64VrEndOffset)
3616 case AK_GeneralPurpose:
3617 Base = getShadowPtrForVAArgument(A->getType(), IRB, GrOffset);
3620 case AK_FloatingPoint:
3621 Base = getShadowPtrForVAArgument(A->getType(), IRB, VrOffset);
3625 // Don't count fixed arguments in the overflow area - va_start will
3626 // skip right over them.
3629 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
3630 Base = getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset);
3631 OverflowOffset += alignTo(ArgSize, 8);
3634 // Count Gp/Vr fixed arguments to their respective offsets, but don't
3635 // bother to actually store a shadow.
3638 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
3640 Constant *OverflowSize =
3641 ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AArch64VAEndOffset);
3642 IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
3645 /// Compute the shadow address for a given va_arg.
3646 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
3648 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
3649 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
3650 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
3654 void visitVAStartInst(VAStartInst &I) override {
3655 IRBuilder<> IRB(&I);
3656 VAStartInstrumentationList.push_back(&I);
3657 Value *VAListTag = I.getArgOperand(0);
3658 Value *ShadowPtr, *OriginPtr;
3659 unsigned Alignment = 8;
3660 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
3661 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
3662 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
3663 /* size */ 32, Alignment, false);
3666 void visitVACopyInst(VACopyInst &I) override {
3667 IRBuilder<> IRB(&I);
3668 VAStartInstrumentationList.push_back(&I);
3669 Value *VAListTag = I.getArgOperand(0);
3670 Value *ShadowPtr, *OriginPtr;
3671 unsigned Alignment = 8;
3672 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
3673 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
3674 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
3675 /* size */ 32, Alignment, false);
3678 // Retrieve a va_list field of 'void*' size.
3679 Value* getVAField64(IRBuilder<> &IRB, Value *VAListTag, int offset) {
3680 Value *SaveAreaPtrPtr =
3682 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
3683 ConstantInt::get(MS.IntptrTy, offset)),
3684 Type::getInt64PtrTy(*MS.C));
3685 return IRB.CreateLoad(SaveAreaPtrPtr);
3688 // Retrieve a va_list field of 'int' size.
3689 Value* getVAField32(IRBuilder<> &IRB, Value *VAListTag, int offset) {
3690 Value *SaveAreaPtr =
3692 IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
3693 ConstantInt::get(MS.IntptrTy, offset)),
3694 Type::getInt32PtrTy(*MS.C));
3695 Value *SaveArea32 = IRB.CreateLoad(SaveAreaPtr);
3696 return IRB.CreateSExt(SaveArea32, MS.IntptrTy);
3699 void finalizeInstrumentation() override {
3700 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
3701 "finalizeInstrumentation called twice");
3702 if (!VAStartInstrumentationList.empty()) {
3703 // If there is a va_start in this function, make a backup copy of
3704 // va_arg_tls somewhere in the function entry block.
3705 IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
3706 VAArgOverflowSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS);
3708 IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AArch64VAEndOffset),
3710 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
3711 IRB.CreateMemCpy(VAArgTLSCopy, 8, MS.VAArgTLS, 8, CopySize);
3714 Value *GrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64GrArgSize);
3715 Value *VrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64VrArgSize);
3717 // Instrument va_start, copy va_list shadow from the backup copy of
3718 // the TLS contents.
3719 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
3720 CallInst *OrigInst = VAStartInstrumentationList[i];
3721 IRBuilder<> IRB(OrigInst->getNextNode());
3723 Value *VAListTag = OrigInst->getArgOperand(0);
3725 // The variadic ABI for AArch64 creates two areas to save the incoming
3726 // argument registers (one for 64-bit general register xn-x7 and another
3727 // for 128-bit FP/SIMD vn-v7).
3728 // We need then to propagate the shadow arguments on both regions
3729 // 'va::__gr_top + va::__gr_offs' and 'va::__vr_top + va::__vr_offs'.
3730 // The remaning arguments are saved on shadow for 'va::stack'.
3731 // One caveat is it requires only to propagate the non-named arguments,
3732 // however on the call site instrumentation 'all' the arguments are
3733 // saved. So to copy the shadow values from the va_arg TLS array
3734 // we need to adjust the offset for both GR and VR fields based on
3735 // the __{gr,vr}_offs value (since they are stores based on incoming
3736 // named arguments).
3738 // Read the stack pointer from the va_list.
3739 Value *StackSaveAreaPtr = getVAField64(IRB, VAListTag, 0);
3741 // Read both the __gr_top and __gr_off and add them up.
3742 Value *GrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 8);
3743 Value *GrOffSaveArea = getVAField32(IRB, VAListTag, 24);
3745 Value *GrRegSaveAreaPtr = IRB.CreateAdd(GrTopSaveAreaPtr, GrOffSaveArea);
3747 // Read both the __vr_top and __vr_off and add them up.
3748 Value *VrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 16);
3749 Value *VrOffSaveArea = getVAField32(IRB, VAListTag, 28);
3751 Value *VrRegSaveAreaPtr = IRB.CreateAdd(VrTopSaveAreaPtr, VrOffSaveArea);
3753 // It does not know how many named arguments is being used and, on the
3754 // callsite all the arguments were saved. Since __gr_off is defined as
3755 // '0 - ((8 - named_gr) * 8)', the idea is to just propagate the variadic
3756 // argument by ignoring the bytes of shadow from named arguments.
3757 Value *GrRegSaveAreaShadowPtrOff =
3758 IRB.CreateAdd(GrArgSize, GrOffSaveArea);
3760 Value *GrRegSaveAreaShadowPtr =
3761 MSV.getShadowOriginPtr(GrRegSaveAreaPtr, IRB, IRB.getInt8Ty(),
3762 /*Alignment*/ 8, /*isStore*/ true)
3765 Value *GrSrcPtr = IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
3766 GrRegSaveAreaShadowPtrOff);
3767 Value *GrCopySize = IRB.CreateSub(GrArgSize, GrRegSaveAreaShadowPtrOff);
3769 IRB.CreateMemCpy(GrRegSaveAreaShadowPtr, 8, GrSrcPtr, 8, GrCopySize);
3771 // Again, but for FP/SIMD values.
3772 Value *VrRegSaveAreaShadowPtrOff =
3773 IRB.CreateAdd(VrArgSize, VrOffSaveArea);
3775 Value *VrRegSaveAreaShadowPtr =
3776 MSV.getShadowOriginPtr(VrRegSaveAreaPtr, IRB, IRB.getInt8Ty(),
3777 /*Alignment*/ 8, /*isStore*/ true)
3780 Value *VrSrcPtr = IRB.CreateInBoundsGEP(
3782 IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
3783 IRB.getInt32(AArch64VrBegOffset)),
3784 VrRegSaveAreaShadowPtrOff);
3785 Value *VrCopySize = IRB.CreateSub(VrArgSize, VrRegSaveAreaShadowPtrOff);
3787 IRB.CreateMemCpy(VrRegSaveAreaShadowPtr, 8, VrSrcPtr, 8, VrCopySize);
3789 // And finally for remaining arguments.
3790 Value *StackSaveAreaShadowPtr =
3791 MSV.getShadowOriginPtr(StackSaveAreaPtr, IRB, IRB.getInt8Ty(),
3792 /*Alignment*/ 16, /*isStore*/ true)
3795 Value *StackSrcPtr =
3796 IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
3797 IRB.getInt32(AArch64VAEndOffset));
3799 IRB.CreateMemCpy(StackSaveAreaShadowPtr, 16, StackSrcPtr, 16,
3805 /// PowerPC64-specific implementation of VarArgHelper.
3806 struct VarArgPowerPC64Helper : public VarArgHelper {
3808 MemorySanitizer &MS;
3809 MemorySanitizerVisitor &MSV;
3810 Value *VAArgTLSCopy = nullptr;
3811 Value *VAArgSize = nullptr;
3813 SmallVector<CallInst*, 16> VAStartInstrumentationList;
3815 VarArgPowerPC64Helper(Function &F, MemorySanitizer &MS,
3816 MemorySanitizerVisitor &MSV) : F(F), MS(MS), MSV(MSV) {}
3818 void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {
3819 // For PowerPC, we need to deal with alignment of stack arguments -
3820 // they are mostly aligned to 8 bytes, but vectors and i128 arrays
3821 // are aligned to 16 bytes, byvals can be aligned to 8 or 16 bytes,
3822 // and QPX vectors are aligned to 32 bytes. For that reason, we
3823 // compute current offset from stack pointer (which is always properly
3824 // aligned), and offset for the first vararg, then subtract them.
3826 Triple TargetTriple(F.getParent()->getTargetTriple());
3827 // Parameter save area starts at 48 bytes from frame pointer for ABIv1,
3828 // and 32 bytes for ABIv2. This is usually determined by target
3829 // endianness, but in theory could be overriden by function attribute.
3830 // For simplicity, we ignore it here (it'd only matter for QPX vectors).
3831 if (TargetTriple.getArch() == Triple::ppc64)
3835 unsigned VAArgOffset = VAArgBase;
3836 const DataLayout &DL = F.getParent()->getDataLayout();
3837 for (CallSite::arg_iterator ArgIt = CS.arg_begin(), End = CS.arg_end();
3838 ArgIt != End; ++ArgIt) {
3840 unsigned ArgNo = CS.getArgumentNo(ArgIt);
3841 bool IsFixed = ArgNo < CS.getFunctionType()->getNumParams();
3842 bool IsByVal = CS.paramHasAttr(ArgNo, Attribute::ByVal);
3844 assert(A->getType()->isPointerTy());
3845 Type *RealTy = A->getType()->getPointerElementType();
3846 uint64_t ArgSize = DL.getTypeAllocSize(RealTy);
3847 uint64_t ArgAlign = CS.getParamAlignment(ArgNo);
3850 VAArgOffset = alignTo(VAArgOffset, ArgAlign);
3852 Value *Base = getShadowPtrForVAArgument(RealTy, IRB,
3853 VAArgOffset - VAArgBase);
3854 Value *AShadowPtr, *AOriginPtr;
3855 std::tie(AShadowPtr, AOriginPtr) = MSV.getShadowOriginPtr(
3856 A, IRB, IRB.getInt8Ty(), kShadowTLSAlignment, /*isStore*/ false);
3858 IRB.CreateMemCpy(Base, kShadowTLSAlignment, AShadowPtr,
3859 kShadowTLSAlignment, ArgSize);
3861 VAArgOffset += alignTo(ArgSize, 8);
3864 uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
3865 uint64_t ArgAlign = 8;
3866 if (A->getType()->isArrayTy()) {
3867 // Arrays are aligned to element size, except for long double
3868 // arrays, which are aligned to 8 bytes.
3869 Type *ElementTy = A->getType()->getArrayElementType();
3870 if (!ElementTy->isPPC_FP128Ty())
3871 ArgAlign = DL.getTypeAllocSize(ElementTy);
3872 } else if (A->getType()->isVectorTy()) {
3873 // Vectors are naturally aligned.
3874 ArgAlign = DL.getTypeAllocSize(A->getType());
3878 VAArgOffset = alignTo(VAArgOffset, ArgAlign);
3879 if (DL.isBigEndian()) {
3880 // Adjusting the shadow for argument with size < 8 to match the placement
3881 // of bits in big endian system
3883 VAArgOffset += (8 - ArgSize);
3886 Base = getShadowPtrForVAArgument(A->getType(), IRB,
3887 VAArgOffset - VAArgBase);
3888 IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
3890 VAArgOffset += ArgSize;
3891 VAArgOffset = alignTo(VAArgOffset, 8);
3894 VAArgBase = VAArgOffset;
3897 Constant *TotalVAArgSize = ConstantInt::get(IRB.getInt64Ty(),
3898 VAArgOffset - VAArgBase);
3899 // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
3900 // a new class member i.e. it is the total size of all VarArgs.
3901 IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
3904 /// Compute the shadow address for a given va_arg.
3905 Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
3907 Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
3908 Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
3909 return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
3913 void visitVAStartInst(VAStartInst &I) override {
3914 IRBuilder<> IRB(&I);
3915 VAStartInstrumentationList.push_back(&I);
3916 Value *VAListTag = I.getArgOperand(0);
3917 Value *ShadowPtr, *OriginPtr;
3918 unsigned Alignment = 8;
3919 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
3920 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
3921 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
3922 /* size */ 8, Alignment, false);
3925 void visitVACopyInst(VACopyInst &I) override {
3926 IRBuilder<> IRB(&I);
3927 Value *VAListTag = I.getArgOperand(0);
3928 Value *ShadowPtr, *OriginPtr;
3929 unsigned Alignment = 8;
3930 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
3931 VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
3932 // Unpoison the whole __va_list_tag.
3933 // FIXME: magic ABI constants.
3934 IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
3935 /* size */ 8, Alignment, false);
3938 void finalizeInstrumentation() override {
3939 assert(!VAArgSize && !VAArgTLSCopy &&
3940 "finalizeInstrumentation called twice");
3941 IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
3942 VAArgSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS);
3943 Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0),
3946 if (!VAStartInstrumentationList.empty()) {
3947 // If there is a va_start in this function, make a backup copy of
3948 // va_arg_tls somewhere in the function entry block.
3949 VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
3950 IRB.CreateMemCpy(VAArgTLSCopy, 8, MS.VAArgTLS, 8, CopySize);
3953 // Instrument va_start.
3954 // Copy va_list shadow from the backup copy of the TLS contents.
3955 for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
3956 CallInst *OrigInst = VAStartInstrumentationList[i];
3957 IRBuilder<> IRB(OrigInst->getNextNode());
3958 Value *VAListTag = OrigInst->getArgOperand(0);
3959 Value *RegSaveAreaPtrPtr =
3960 IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
3961 PointerType::get(Type::getInt64PtrTy(*MS.C), 0));
3962 Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrPtr);
3963 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
3964 unsigned Alignment = 8;
3965 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
3966 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
3967 Alignment, /*isStore*/ true);
3968 IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
3974 /// A no-op implementation of VarArgHelper.
3975 struct VarArgNoOpHelper : public VarArgHelper {
3976 VarArgNoOpHelper(Function &F, MemorySanitizer &MS,
3977 MemorySanitizerVisitor &MSV) {}
3979 void visitCallSite(CallSite &CS, IRBuilder<> &IRB) override {}
3981 void visitVAStartInst(VAStartInst &I) override {}
3983 void visitVACopyInst(VACopyInst &I) override {}
3985 void finalizeInstrumentation() override {}
3988 } // end anonymous namespace
3990 static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
3991 MemorySanitizerVisitor &Visitor) {
3992 // VarArg handling is only implemented on AMD64. False positives are possible
3993 // on other platforms.
3994 Triple TargetTriple(Func.getParent()->getTargetTriple());
3995 if (TargetTriple.getArch() == Triple::x86_64)
3996 return new VarArgAMD64Helper(Func, Msan, Visitor);
3997 else if (TargetTriple.isMIPS64())
3998 return new VarArgMIPS64Helper(Func, Msan, Visitor);
3999 else if (TargetTriple.getArch() == Triple::aarch64)
4000 return new VarArgAArch64Helper(Func, Msan, Visitor);
4001 else if (TargetTriple.getArch() == Triple::ppc64 ||
4002 TargetTriple.getArch() == Triple::ppc64le)
4003 return new VarArgPowerPC64Helper(Func, Msan, Visitor);
4005 return new VarArgNoOpHelper(Func, Msan, Visitor);
4008 bool MemorySanitizer::runOnFunction(Function &F) {
4009 if (&F == MsanCtorFunction)
4011 MemorySanitizerVisitor Visitor(F, *this);
4013 // Clear out readonly/readnone attributes.
4015 B.addAttribute(Attribute::ReadOnly)
4016 .addAttribute(Attribute::ReadNone);
4017 F.removeAttributes(AttributeList::FunctionIndex, B);
4019 return Visitor.runOnFunction();