1 //===-ThinLTOCodeGenerator.cpp - LLVM Link Time Optimizer -----------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file implements the Thin Link Time Optimization library. This library is
11 // intended to be used by linker to optimize code at link time.
13 //===----------------------------------------------------------------------===//
15 #include "llvm/LTO/legacy/ThinLTOCodeGenerator.h"
17 #include "llvm/ADT/Statistic.h"
18 #include "llvm/ADT/StringExtras.h"
19 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
20 #include "llvm/Analysis/ProfileSummaryInfo.h"
21 #include "llvm/Analysis/TargetLibraryInfo.h"
22 #include "llvm/Analysis/TargetTransformInfo.h"
23 #include "llvm/Bitcode/BitcodeReader.h"
24 #include "llvm/Bitcode/BitcodeWriter.h"
25 #include "llvm/Bitcode/BitcodeWriterPass.h"
26 #include "llvm/ExecutionEngine/ObjectMemoryBuffer.h"
27 #include "llvm/IR/DiagnosticPrinter.h"
28 #include "llvm/IR/DebugInfo.h"
29 #include "llvm/IR/LLVMContext.h"
30 #include "llvm/IR/LegacyPassManager.h"
31 #include "llvm/IR/Mangler.h"
32 #include "llvm/IR/Verifier.h"
33 #include "llvm/IRReader/IRReader.h"
34 #include "llvm/LTO/LTO.h"
35 #include "llvm/Linker/Linker.h"
36 #include "llvm/MC/SubtargetFeature.h"
37 #include "llvm/Object/IRObjectFile.h"
38 #include "llvm/Support/CachePruning.h"
39 #include "llvm/Support/Debug.h"
40 #include "llvm/Support/Error.h"
41 #include "llvm/Support/Path.h"
42 #include "llvm/Support/SHA1.h"
43 #include "llvm/Support/TargetRegistry.h"
44 #include "llvm/Support/ThreadPool.h"
45 #include "llvm/Support/Threading.h"
46 #include "llvm/Support/ToolOutputFile.h"
47 #include "llvm/Support/VCSRevision.h"
48 #include "llvm/Target/TargetMachine.h"
49 #include "llvm/Transforms/IPO.h"
50 #include "llvm/Transforms/IPO/FunctionImport.h"
51 #include "llvm/Transforms/IPO/Internalize.h"
52 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
53 #include "llvm/Transforms/ObjCARC.h"
54 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
60 #define DEBUG_TYPE "thinlto"
63 // Flags -discard-value-names, defined in LTOCodeGenerator.cpp
64 extern cl::opt<bool> LTODiscardValueNames;
65 extern cl::opt<std::string> LTORemarksFilename;
66 extern cl::opt<bool> LTOPassRemarksWithHotness;
67 extern cl::opt<bool> LTOStripInvalidDebugInfo;
73 ThreadCount("threads", cl::init(llvm::heavyweight_hardware_concurrency()));
75 // Simple helper to save temporary files for debug.
76 static void saveTempBitcode(const Module &TheModule, StringRef TempDir,
77 unsigned count, StringRef Suffix) {
80 // User asked to save temps, let dump the bitcode file after import.
81 std::string SaveTempPath = (TempDir + llvm::utostr(count) + Suffix).str();
83 raw_fd_ostream OS(SaveTempPath, EC, sys::fs::F_None);
85 report_fatal_error(Twine("Failed to open ") + SaveTempPath +
86 " to save optimized bitcode\n");
87 WriteBitcodeToFile(&TheModule, OS, /* ShouldPreserveUseListOrder */ true);
90 static const GlobalValueSummary *
91 getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
92 // If there is any strong definition anywhere, get it.
93 auto StrongDefForLinker = llvm::find_if(
94 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
95 auto Linkage = Summary->linkage();
96 return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
97 !GlobalValue::isWeakForLinker(Linkage);
99 if (StrongDefForLinker != GVSummaryList.end())
100 return StrongDefForLinker->get();
101 // Get the first *linker visible* definition for this global in the summary
103 auto FirstDefForLinker = llvm::find_if(
104 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
105 auto Linkage = Summary->linkage();
106 return !GlobalValue::isAvailableExternallyLinkage(Linkage);
108 // Extern templates can be emitted as available_externally.
109 if (FirstDefForLinker == GVSummaryList.end())
111 return FirstDefForLinker->get();
114 // Populate map of GUID to the prevailing copy for any multiply defined
115 // symbols. Currently assume first copy is prevailing, or any strong
116 // definition. Can be refined with Linker information in the future.
117 static void computePrevailingCopies(
118 const ModuleSummaryIndex &Index,
119 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy) {
120 auto HasMultipleCopies = [&](const GlobalValueSummaryList &GVSummaryList) {
121 return GVSummaryList.size() > 1;
124 for (auto &I : Index) {
125 if (HasMultipleCopies(I.second.SummaryList))
126 PrevailingCopy[I.first] =
127 getFirstDefinitionForLinker(I.second.SummaryList);
131 static StringMap<MemoryBufferRef>
132 generateModuleMap(const std::vector<ThinLTOBuffer> &Modules) {
133 StringMap<MemoryBufferRef> ModuleMap;
134 for (auto &ModuleBuffer : Modules) {
135 assert(ModuleMap.find(ModuleBuffer.getBufferIdentifier()) ==
137 "Expect unique Buffer Identifier");
138 ModuleMap[ModuleBuffer.getBufferIdentifier()] = ModuleBuffer.getMemBuffer();
143 static void promoteModule(Module &TheModule, const ModuleSummaryIndex &Index) {
144 if (renameModuleForThinLTO(TheModule, Index))
145 report_fatal_error("renameModuleForThinLTO failed");
149 class ThinLTODiagnosticInfo : public DiagnosticInfo {
152 ThinLTODiagnosticInfo(const Twine &DiagMsg,
153 DiagnosticSeverity Severity = DS_Error)
154 : DiagnosticInfo(DK_Linker, Severity), Msg(DiagMsg) {}
155 void print(DiagnosticPrinter &DP) const override { DP << Msg; }
159 /// Verify the module and strip broken debug info.
160 static void verifyLoadedModule(Module &TheModule) {
161 bool BrokenDebugInfo = false;
162 if (verifyModule(TheModule, &dbgs(),
163 LTOStripInvalidDebugInfo ? &BrokenDebugInfo : nullptr))
164 report_fatal_error("Broken module found, compilation aborted!");
165 if (BrokenDebugInfo) {
166 TheModule.getContext().diagnose(ThinLTODiagnosticInfo(
167 "Invalid debug info found, debug info will be stripped", DS_Warning));
168 StripDebugInfo(TheModule);
172 static std::unique_ptr<Module>
173 loadModuleFromBuffer(const MemoryBufferRef &Buffer, LLVMContext &Context,
174 bool Lazy, bool IsImporting) {
176 Expected<std::unique_ptr<Module>> ModuleOrErr =
178 ? getLazyBitcodeModule(Buffer, Context,
179 /* ShouldLazyLoadMetadata */ true, IsImporting)
180 : parseBitcodeFile(Buffer, Context);
182 handleAllErrors(ModuleOrErr.takeError(), [&](ErrorInfoBase &EIB) {
183 SMDiagnostic Err = SMDiagnostic(Buffer.getBufferIdentifier(),
184 SourceMgr::DK_Error, EIB.message());
185 Err.print("ThinLTO", errs());
187 report_fatal_error("Can't load module, abort.");
190 verifyLoadedModule(*ModuleOrErr.get());
191 return std::move(ModuleOrErr.get());
195 crossImportIntoModule(Module &TheModule, const ModuleSummaryIndex &Index,
196 StringMap<MemoryBufferRef> &ModuleMap,
197 const FunctionImporter::ImportMapTy &ImportList) {
198 auto Loader = [&](StringRef Identifier) {
199 return loadModuleFromBuffer(ModuleMap[Identifier], TheModule.getContext(),
200 /*Lazy=*/true, /*IsImporting*/ true);
203 FunctionImporter Importer(Index, Loader);
204 Expected<bool> Result = Importer.importFunctions(TheModule, ImportList);
206 handleAllErrors(Result.takeError(), [&](ErrorInfoBase &EIB) {
207 SMDiagnostic Err = SMDiagnostic(TheModule.getModuleIdentifier(),
208 SourceMgr::DK_Error, EIB.message());
209 Err.print("ThinLTO", errs());
211 report_fatal_error("importFunctions failed");
213 // Verify again after cross-importing.
214 verifyLoadedModule(TheModule);
217 static void optimizeModule(Module &TheModule, TargetMachine &TM,
218 unsigned OptLevel, bool Freestanding) {
219 // Populate the PassManager
220 PassManagerBuilder PMB;
221 PMB.LibraryInfo = new TargetLibraryInfoImpl(TM.getTargetTriple());
223 PMB.LibraryInfo->disableAllFunctions();
224 PMB.Inliner = createFunctionInliningPass();
225 // FIXME: should get it from the bitcode?
226 PMB.OptLevel = OptLevel;
227 PMB.LoopVectorize = true;
228 PMB.SLPVectorize = true;
229 // Already did this in verifyLoadedModule().
230 PMB.VerifyInput = false;
231 PMB.VerifyOutput = false;
233 legacy::PassManager PM;
235 // Add the TTI (required to inform the vectorizer about register size for
237 PM.add(createTargetTransformInfoWrapperPass(TM.getTargetIRAnalysis()));
240 PMB.populateThinLTOPassManager(PM);
245 // Convert the PreservedSymbols map from "Name" based to "GUID" based.
246 static DenseSet<GlobalValue::GUID>
247 computeGUIDPreservedSymbols(const StringSet<> &PreservedSymbols,
248 const Triple &TheTriple) {
249 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols(PreservedSymbols.size());
250 for (auto &Entry : PreservedSymbols) {
251 StringRef Name = Entry.first();
252 if (TheTriple.isOSBinFormatMachO() && Name.size() > 0 && Name[0] == '_')
253 Name = Name.drop_front();
254 GUIDPreservedSymbols.insert(GlobalValue::getGUID(Name));
256 return GUIDPreservedSymbols;
259 std::unique_ptr<MemoryBuffer> codegenModule(Module &TheModule,
261 SmallVector<char, 128> OutputBuffer;
265 raw_svector_ostream OS(OutputBuffer);
266 legacy::PassManager PM;
268 // If the bitcode files contain ARC code and were compiled with optimization,
269 // the ObjCARCContractPass must be run, so do it unconditionally here.
270 PM.add(createObjCARCContractPass());
272 // Setup the codegen now.
273 if (TM.addPassesToEmitFile(PM, OS, TargetMachine::CGFT_ObjectFile,
274 /* DisableVerify */ true))
275 report_fatal_error("Failed to setup codegen");
277 // Run codegen now. resulting binary is in OutputBuffer.
280 return make_unique<ObjectMemoryBuffer>(std::move(OutputBuffer));
283 /// Manage caching for a single Module.
284 class ModuleCacheEntry {
285 SmallString<128> EntryPath;
288 // Create a cache entry. This compute a unique hash for the Module considering
289 // the current list of export/import, and offer an interface to query to
290 // access the content in the cache.
292 StringRef CachePath, const ModuleSummaryIndex &Index, StringRef ModuleID,
293 const FunctionImporter::ImportMapTy &ImportList,
294 const FunctionImporter::ExportSetTy &ExportList,
295 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
296 const GVSummaryMapTy &DefinedFunctions,
297 const DenseSet<GlobalValue::GUID> &PreservedSymbols, unsigned OptLevel,
298 bool Freestanding, const TargetMachineBuilder &TMBuilder) {
299 if (CachePath.empty())
302 if (!Index.modulePaths().count(ModuleID))
303 // The module does not have an entry, it can't have a hash at all
306 // Compute the unique hash for this entry
307 // This is based on the current compiler version, the module itself, the
308 // export list, the hash for every single module in the import list, the
309 // list of ResolvedODR for the module, and the list of preserved symbols.
311 // Include the hash for the current module
312 auto ModHash = Index.getModuleHash(ModuleID);
314 if (all_of(ModHash, [](uint32_t V) { return V == 0; }))
315 // No hash entry, no caching!
320 // Include the parts of the LTO configuration that affect code generation.
321 auto AddString = [&](StringRef Str) {
323 Hasher.update(ArrayRef<uint8_t>{0});
325 auto AddUnsigned = [&](unsigned I) {
331 Hasher.update(ArrayRef<uint8_t>{Data, 4});
334 // Start with the compiler revision
335 Hasher.update(LLVM_VERSION_STRING);
337 Hasher.update(LLVM_REVISION);
340 // Hash the optimization level and the target machine settings.
341 AddString(TMBuilder.MCpu);
342 // FIXME: Hash more of Options. For now all clients initialize Options from
343 // command-line flags (which is unsupported in production), but may set
344 // RelaxELFRelocations. The clang driver can also pass FunctionSections,
345 // DataSections and DebuggerTuning via command line flags.
346 AddUnsigned(TMBuilder.Options.RelaxELFRelocations);
347 AddUnsigned(TMBuilder.Options.FunctionSections);
348 AddUnsigned(TMBuilder.Options.DataSections);
349 AddUnsigned((unsigned)TMBuilder.Options.DebuggerTuning);
350 AddString(TMBuilder.MAttr);
351 if (TMBuilder.RelocModel)
352 AddUnsigned(*TMBuilder.RelocModel);
353 AddUnsigned(TMBuilder.CGOptLevel);
354 AddUnsigned(OptLevel);
355 AddUnsigned(Freestanding);
357 Hasher.update(ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));
358 for (auto F : ExportList)
359 // The export list can impact the internalization, be conservative here
360 Hasher.update(ArrayRef<uint8_t>((uint8_t *)&F, sizeof(F)));
362 // Include the hash for every module we import functions from
363 for (auto &Entry : ImportList) {
364 auto ModHash = Index.getModuleHash(Entry.first());
365 Hasher.update(ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));
368 // Include the hash for the resolved ODR.
369 for (auto &Entry : ResolvedODR) {
370 Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&Entry.first,
371 sizeof(GlobalValue::GUID)));
372 Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&Entry.second,
373 sizeof(GlobalValue::LinkageTypes)));
376 // Include the hash for the preserved symbols.
377 for (auto &Entry : PreservedSymbols) {
378 if (DefinedFunctions.count(Entry))
380 ArrayRef<uint8_t>((const uint8_t *)&Entry, sizeof(GlobalValue::GUID)));
383 // This choice of file name allows the cache to be pruned (see pruneCache()
384 // in include/llvm/Support/CachePruning.h).
385 sys::path::append(EntryPath, CachePath,
386 "llvmcache-" + toHex(Hasher.result()));
389 // Access the path to this entry in the cache.
390 StringRef getEntryPath() { return EntryPath; }
392 // Try loading the buffer for this cache entry.
393 ErrorOr<std::unique_ptr<MemoryBuffer>> tryLoadingBuffer() {
394 if (EntryPath.empty())
395 return std::error_code();
396 return MemoryBuffer::getFile(EntryPath);
399 // Cache the Produced object file
400 void write(const MemoryBuffer &OutputBuffer) {
401 if (EntryPath.empty())
404 // Write to a temporary to avoid race condition
405 SmallString<128> TempFilename;
408 sys::fs::createTemporaryFile("Thin", "tmp.o", TempFD, TempFilename);
410 errs() << "Error: " << EC.message() << "\n";
411 report_fatal_error("ThinLTO: Can't get a temporary file");
414 raw_fd_ostream OS(TempFD, /* ShouldClose */ true);
415 OS << OutputBuffer.getBuffer();
417 // Rename to final destination (hopefully race condition won't matter here)
418 EC = sys::fs::rename(TempFilename, EntryPath);
420 sys::fs::remove(TempFilename);
421 raw_fd_ostream OS(EntryPath, EC, sys::fs::F_None);
423 report_fatal_error(Twine("Failed to open ") + EntryPath +
424 " to save cached entry\n");
425 OS << OutputBuffer.getBuffer();
430 static std::unique_ptr<MemoryBuffer>
431 ProcessThinLTOModule(Module &TheModule, ModuleSummaryIndex &Index,
432 StringMap<MemoryBufferRef> &ModuleMap, TargetMachine &TM,
433 const FunctionImporter::ImportMapTy &ImportList,
434 const FunctionImporter::ExportSetTy &ExportList,
435 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
436 const GVSummaryMapTy &DefinedGlobals,
437 const ThinLTOCodeGenerator::CachingOptions &CacheOptions,
438 bool DisableCodeGen, StringRef SaveTempsDir,
439 bool Freestanding, unsigned OptLevel, unsigned count) {
441 // "Benchmark"-like optimization: single-source case
442 bool SingleModule = (ModuleMap.size() == 1);
445 promoteModule(TheModule, Index);
447 // Apply summary-based LinkOnce/Weak resolution decisions.
448 thinLTOResolveWeakForLinkerModule(TheModule, DefinedGlobals);
450 // Save temps: after promotion.
451 saveTempBitcode(TheModule, SaveTempsDir, count, ".1.promoted.bc");
454 // Be friendly and don't nuke totally the module when the client didn't
455 // supply anything to preserve.
456 if (!ExportList.empty() || !GUIDPreservedSymbols.empty()) {
457 // Apply summary-based internalization decisions.
458 thinLTOInternalizeModule(TheModule, DefinedGlobals);
461 // Save internalized bitcode
462 saveTempBitcode(TheModule, SaveTempsDir, count, ".2.internalized.bc");
465 crossImportIntoModule(TheModule, Index, ModuleMap, ImportList);
467 // Save temps: after cross-module import.
468 saveTempBitcode(TheModule, SaveTempsDir, count, ".3.imported.bc");
471 optimizeModule(TheModule, TM, OptLevel, Freestanding);
473 saveTempBitcode(TheModule, SaveTempsDir, count, ".4.opt.bc");
475 if (DisableCodeGen) {
476 // Configured to stop before CodeGen, serialize the bitcode and return.
477 SmallVector<char, 128> OutputBuffer;
479 raw_svector_ostream OS(OutputBuffer);
480 ProfileSummaryInfo PSI(TheModule);
481 auto Index = buildModuleSummaryIndex(TheModule, nullptr, &PSI);
482 WriteBitcodeToFile(&TheModule, OS, true, &Index);
484 return make_unique<ObjectMemoryBuffer>(std::move(OutputBuffer));
487 return codegenModule(TheModule, TM);
490 /// Resolve LinkOnce/Weak symbols. Record resolutions in the \p ResolvedODR map
491 /// for caching, and in the \p Index for application during the ThinLTO
492 /// backends. This is needed for correctness for exported symbols (ensure
493 /// at least one copy kept) and a compile-time optimization (to drop duplicate
494 /// copies when possible).
495 static void resolveWeakForLinkerInIndex(
496 ModuleSummaryIndex &Index,
497 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>>
500 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
501 computePrevailingCopies(Index, PrevailingCopy);
503 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
504 const auto &Prevailing = PrevailingCopy.find(GUID);
505 // Not in map means that there was only one copy, which must be prevailing.
506 if (Prevailing == PrevailingCopy.end())
508 return Prevailing->second == S;
511 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
512 GlobalValue::GUID GUID,
513 GlobalValue::LinkageTypes NewLinkage) {
514 ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
517 thinLTOResolveWeakForLinkerInIndex(Index, isPrevailing, recordNewLinkage);
520 // Initialize the TargetMachine builder for a given Triple
521 static void initTMBuilder(TargetMachineBuilder &TMBuilder,
522 const Triple &TheTriple) {
523 // Set a default CPU for Darwin triples (copied from LTOCodeGenerator).
524 // FIXME this looks pretty terrible...
525 if (TMBuilder.MCpu.empty() && TheTriple.isOSDarwin()) {
526 if (TheTriple.getArch() == llvm::Triple::x86_64)
527 TMBuilder.MCpu = "core2";
528 else if (TheTriple.getArch() == llvm::Triple::x86)
529 TMBuilder.MCpu = "yonah";
530 else if (TheTriple.getArch() == llvm::Triple::aarch64)
531 TMBuilder.MCpu = "cyclone";
533 TMBuilder.TheTriple = std::move(TheTriple);
536 } // end anonymous namespace
538 void ThinLTOCodeGenerator::addModule(StringRef Identifier, StringRef Data) {
539 ThinLTOBuffer Buffer(Data, Identifier);
542 ErrorOr<std::string> TripleOrErr = expectedToErrorOrAndEmitErrors(
543 Context, getBitcodeTargetTriple(Buffer.getMemBuffer()));
546 TripleStr = *TripleOrErr;
548 Triple TheTriple(TripleStr);
551 initTMBuilder(TMBuilder, Triple(TheTriple));
552 else if (TMBuilder.TheTriple != TheTriple) {
553 if (!TMBuilder.TheTriple.isCompatibleWith(TheTriple))
554 report_fatal_error("ThinLTO modules with incompatible triples not "
556 initTMBuilder(TMBuilder, Triple(TMBuilder.TheTriple.merge(TheTriple)));
559 Modules.push_back(Buffer);
562 void ThinLTOCodeGenerator::preserveSymbol(StringRef Name) {
563 PreservedSymbols.insert(Name);
566 void ThinLTOCodeGenerator::crossReferenceSymbol(StringRef Name) {
567 // FIXME: At the moment, we don't take advantage of this extra information,
568 // we're conservatively considering cross-references as preserved.
569 // CrossReferencedSymbols.insert(Name);
570 PreservedSymbols.insert(Name);
573 // TargetMachine factory
574 std::unique_ptr<TargetMachine> TargetMachineBuilder::create() const {
576 const Target *TheTarget =
577 TargetRegistry::lookupTarget(TheTriple.str(), ErrMsg);
579 report_fatal_error("Can't load target for this Triple: " + ErrMsg);
582 // Use MAttr as the default set of features.
583 SubtargetFeatures Features(MAttr);
584 Features.getDefaultSubtargetFeatures(TheTriple);
585 std::string FeatureStr = Features.getString();
587 return std::unique_ptr<TargetMachine>(TheTarget->createTargetMachine(
588 TheTriple.str(), MCpu, FeatureStr, Options, RelocModel,
589 CodeModel::Default, CGOptLevel));
593 * Produce the combined summary index from all the bitcode files:
596 std::unique_ptr<ModuleSummaryIndex> ThinLTOCodeGenerator::linkCombinedIndex() {
597 std::unique_ptr<ModuleSummaryIndex> CombinedIndex =
598 llvm::make_unique<ModuleSummaryIndex>();
599 uint64_t NextModuleId = 0;
600 for (auto &ModuleBuffer : Modules) {
601 if (Error Err = readModuleSummaryIndex(ModuleBuffer.getMemBuffer(),
602 *CombinedIndex, NextModuleId++)) {
604 logAllUnhandledErrors(
605 std::move(Err), errs(),
606 "error: can't create module summary index for buffer: ");
610 return CombinedIndex;
614 * Perform promotion and renaming of exported internal functions.
615 * Index is updated to reflect linkage changes from weak resolution.
617 void ThinLTOCodeGenerator::promote(Module &TheModule,
618 ModuleSummaryIndex &Index) {
619 auto ModuleCount = Index.modulePaths().size();
620 auto ModuleIdentifier = TheModule.getModuleIdentifier();
622 // Collect for each module the list of function it defines (GUID -> Summary).
623 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
624 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
626 // Convert the preserved symbols set from string to GUID
627 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
628 PreservedSymbols, Triple(TheModule.getTargetTriple()));
630 // Compute "dead" symbols, we don't want to import/export these!
631 auto DeadSymbols = computeDeadSymbols(Index, GUIDPreservedSymbols);
633 // Generate import/export list
634 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
635 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
636 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
637 ExportLists, &DeadSymbols);
639 // Resolve LinkOnce/Weak symbols.
640 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
641 resolveWeakForLinkerInIndex(Index, ResolvedODR);
643 thinLTOResolveWeakForLinkerModule(
644 TheModule, ModuleToDefinedGVSummaries[ModuleIdentifier]);
646 // Promote the exported values in the index, so that they are promoted
648 auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) {
649 const auto &ExportList = ExportLists.find(ModuleIdentifier);
650 return (ExportList != ExportLists.end() &&
651 ExportList->second.count(GUID)) ||
652 GUIDPreservedSymbols.count(GUID);
654 thinLTOInternalizeAndPromoteInIndex(Index, isExported);
656 promoteModule(TheModule, Index);
660 * Perform cross-module importing for the module identified by ModuleIdentifier.
662 void ThinLTOCodeGenerator::crossModuleImport(Module &TheModule,
663 ModuleSummaryIndex &Index) {
664 auto ModuleMap = generateModuleMap(Modules);
665 auto ModuleCount = Index.modulePaths().size();
667 // Collect for each module the list of function it defines (GUID -> Summary).
668 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
669 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
671 // Convert the preserved symbols set from string to GUID
672 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
673 PreservedSymbols, Triple(TheModule.getTargetTriple()));
675 // Compute "dead" symbols, we don't want to import/export these!
676 auto DeadSymbols = computeDeadSymbols(Index, GUIDPreservedSymbols);
678 // Generate import/export list
679 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
680 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
681 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
682 ExportLists, &DeadSymbols);
683 auto &ImportList = ImportLists[TheModule.getModuleIdentifier()];
685 crossImportIntoModule(TheModule, Index, ModuleMap, ImportList);
689 * Compute the list of summaries needed for importing into module.
691 void ThinLTOCodeGenerator::gatherImportedSummariesForModule(
692 StringRef ModulePath, ModuleSummaryIndex &Index,
693 std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex) {
694 auto ModuleCount = Index.modulePaths().size();
696 // Collect for each module the list of function it defines (GUID -> Summary).
697 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
698 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
700 // Generate import/export list
701 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
702 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
703 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
706 llvm::gatherImportedSummariesForModule(ModulePath, ModuleToDefinedGVSummaries,
707 ImportLists[ModulePath],
708 ModuleToSummariesForIndex);
712 * Emit the list of files needed for importing into module.
714 void ThinLTOCodeGenerator::emitImports(StringRef ModulePath,
715 StringRef OutputName,
716 ModuleSummaryIndex &Index) {
717 auto ModuleCount = Index.modulePaths().size();
719 // Collect for each module the list of function it defines (GUID -> Summary).
720 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
721 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
723 // Generate import/export list
724 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
725 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
726 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
730 if ((EC = EmitImportsFiles(ModulePath, OutputName, ImportLists[ModulePath])))
731 report_fatal_error(Twine("Failed to open ") + OutputName +
732 " to save imports lists\n");
736 * Perform internalization. Index is updated to reflect linkage changes.
738 void ThinLTOCodeGenerator::internalize(Module &TheModule,
739 ModuleSummaryIndex &Index) {
740 initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
741 auto ModuleCount = Index.modulePaths().size();
742 auto ModuleIdentifier = TheModule.getModuleIdentifier();
744 // Convert the preserved symbols set from string to GUID
745 auto GUIDPreservedSymbols =
746 computeGUIDPreservedSymbols(PreservedSymbols, TMBuilder.TheTriple);
748 // Collect for each module the list of function it defines (GUID -> Summary).
749 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
750 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
752 // Compute "dead" symbols, we don't want to import/export these!
753 auto DeadSymbols = computeDeadSymbols(Index, GUIDPreservedSymbols);
755 // Generate import/export list
756 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
757 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
758 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
759 ExportLists, &DeadSymbols);
760 auto &ExportList = ExportLists[ModuleIdentifier];
762 // Be friendly and don't nuke totally the module when the client didn't
763 // supply anything to preserve.
764 if (ExportList.empty() && GUIDPreservedSymbols.empty())
768 auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) {
769 const auto &ExportList = ExportLists.find(ModuleIdentifier);
770 return (ExportList != ExportLists.end() &&
771 ExportList->second.count(GUID)) ||
772 GUIDPreservedSymbols.count(GUID);
774 thinLTOInternalizeAndPromoteInIndex(Index, isExported);
775 thinLTOInternalizeModule(TheModule,
776 ModuleToDefinedGVSummaries[ModuleIdentifier]);
780 * Perform post-importing ThinLTO optimizations.
782 void ThinLTOCodeGenerator::optimize(Module &TheModule) {
783 initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
786 optimizeModule(TheModule, *TMBuilder.create(), OptLevel, Freestanding);
790 * Perform ThinLTO CodeGen.
792 std::unique_ptr<MemoryBuffer> ThinLTOCodeGenerator::codegen(Module &TheModule) {
793 initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
794 return codegenModule(TheModule, *TMBuilder.create());
797 /// Write out the generated object file, either from CacheEntryPath or from
798 /// OutputBuffer, preferring hard-link when possible.
799 /// Returns the path to the generated file in SavedObjectsDirectoryPath.
800 static std::string writeGeneratedObject(int count, StringRef CacheEntryPath,
801 StringRef SavedObjectsDirectoryPath,
802 const MemoryBuffer &OutputBuffer) {
803 SmallString<128> OutputPath(SavedObjectsDirectoryPath);
804 llvm::sys::path::append(OutputPath, Twine(count) + ".thinlto.o");
805 OutputPath.c_str(); // Ensure the string is null terminated.
806 if (sys::fs::exists(OutputPath))
807 sys::fs::remove(OutputPath);
809 // We don't return a memory buffer to the linker, just a list of files.
810 if (!CacheEntryPath.empty()) {
811 // Cache is enabled, hard-link the entry (or copy if hard-link fails).
812 auto Err = sys::fs::create_hard_link(CacheEntryPath, OutputPath);
814 return OutputPath.str();
815 // Hard linking failed, try to copy.
816 Err = sys::fs::copy_file(CacheEntryPath, OutputPath);
818 return OutputPath.str();
819 // Copy failed (could be because the CacheEntry was removed from the cache
820 // in the meantime by another process), fall back and try to write down the
821 // buffer to the output.
822 errs() << "error: can't link or copy from cached entry '" << CacheEntryPath
823 << "' to '" << OutputPath << "'\n";
825 // No cache entry, just write out the buffer.
827 raw_fd_ostream OS(OutputPath, Err, sys::fs::F_None);
829 report_fatal_error("Can't open output '" + OutputPath + "'\n");
830 OS << OutputBuffer.getBuffer();
831 return OutputPath.str();
834 // Main entry point for the ThinLTO processing
835 void ThinLTOCodeGenerator::run() {
836 // Prepare the resulting object vector
837 assert(ProducedBinaries.empty() && "The generator should not be reused");
838 if (SavedObjectsDirectoryPath.empty())
839 ProducedBinaries.resize(Modules.size());
841 sys::fs::create_directories(SavedObjectsDirectoryPath);
843 sys::fs::is_directory(SavedObjectsDirectoryPath, IsDir);
845 report_fatal_error("Unexistent dir: '" + SavedObjectsDirectoryPath + "'");
846 ProducedBinaryFiles.resize(Modules.size());
850 // Perform only parallel codegen and return.
853 for (auto &ModuleBuffer : Modules) {
854 Pool.async([&](int count) {
856 Context.setDiscardValueNames(LTODiscardValueNames);
860 loadModuleFromBuffer(ModuleBuffer.getMemBuffer(), Context, false,
861 /*IsImporting*/ false);
864 auto OutputBuffer = codegen(*TheModule);
865 if (SavedObjectsDirectoryPath.empty())
866 ProducedBinaries[count] = std::move(OutputBuffer);
868 ProducedBinaryFiles[count] = writeGeneratedObject(
869 count, "", SavedObjectsDirectoryPath, *OutputBuffer);
876 // Sequential linking phase
877 auto Index = linkCombinedIndex();
879 // Save temps: index.
880 if (!SaveTempsDir.empty()) {
881 auto SaveTempPath = SaveTempsDir + "index.bc";
883 raw_fd_ostream OS(SaveTempPath, EC, sys::fs::F_None);
885 report_fatal_error(Twine("Failed to open ") + SaveTempPath +
886 " to save optimized bitcode\n");
887 WriteIndexToFile(*Index, OS);
891 // Prepare the module map.
892 auto ModuleMap = generateModuleMap(Modules);
893 auto ModuleCount = Modules.size();
895 // Collect for each module the list of function it defines (GUID -> Summary).
896 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
897 Index->collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
899 // Convert the preserved symbols set from string to GUID, this is needed for
900 // computing the caching hash and the internalization.
901 auto GUIDPreservedSymbols =
902 computeGUIDPreservedSymbols(PreservedSymbols, TMBuilder.TheTriple);
904 // Compute "dead" symbols, we don't want to import/export these!
905 auto DeadSymbols = computeDeadSymbols(*Index, GUIDPreservedSymbols);
907 // Collect the import/export lists for all modules from the call-graph in the
909 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
910 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
911 ComputeCrossModuleImport(*Index, ModuleToDefinedGVSummaries, ImportLists,
912 ExportLists, &DeadSymbols);
914 // We use a std::map here to be able to have a defined ordering when
915 // producing a hash for the cache entry.
916 // FIXME: we should be able to compute the caching hash for the entry based
917 // on the index, and nuke this map.
918 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
920 // Resolve LinkOnce/Weak symbols, this has to be computed early because it
921 // impacts the caching.
922 resolveWeakForLinkerInIndex(*Index, ResolvedODR);
924 auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) {
925 const auto &ExportList = ExportLists.find(ModuleIdentifier);
926 return (ExportList != ExportLists.end() &&
927 ExportList->second.count(GUID)) ||
928 GUIDPreservedSymbols.count(GUID);
931 // Use global summary-based analysis to identify symbols that can be
932 // internalized (because they aren't exported or preserved as per callback).
933 // Changes are made in the index, consumed in the ThinLTO backends.
934 thinLTOInternalizeAndPromoteInIndex(*Index, isExported);
936 // Make sure that every module has an entry in the ExportLists and
937 // ResolvedODR maps to enable threaded access to these maps below.
938 for (auto &DefinedGVSummaries : ModuleToDefinedGVSummaries) {
939 ExportLists[DefinedGVSummaries.first()];
940 ResolvedODR[DefinedGVSummaries.first()];
943 // Compute the ordering we will process the inputs: the rough heuristic here
944 // is to sort them per size so that the largest module get schedule as soon as
945 // possible. This is purely a compile-time optimization.
946 std::vector<int> ModulesOrdering;
947 ModulesOrdering.resize(Modules.size());
948 std::iota(ModulesOrdering.begin(), ModulesOrdering.end(), 0);
949 std::sort(ModulesOrdering.begin(), ModulesOrdering.end(),
950 [&](int LeftIndex, int RightIndex) {
951 auto LSize = Modules[LeftIndex].getBuffer().size();
952 auto RSize = Modules[RightIndex].getBuffer().size();
953 return LSize > RSize;
956 // Parallel optimizer + codegen
958 ThreadPool Pool(ThreadCount);
959 for (auto IndexCount : ModulesOrdering) {
960 auto &ModuleBuffer = Modules[IndexCount];
961 Pool.async([&](int count) {
962 auto ModuleIdentifier = ModuleBuffer.getBufferIdentifier();
963 auto &ExportList = ExportLists[ModuleIdentifier];
965 auto &DefinedFunctions = ModuleToDefinedGVSummaries[ModuleIdentifier];
967 // The module may be cached, this helps handling it.
968 ModuleCacheEntry CacheEntry(CacheOptions.Path, *Index, ModuleIdentifier,
969 ImportLists[ModuleIdentifier], ExportList,
970 ResolvedODR[ModuleIdentifier],
971 DefinedFunctions, GUIDPreservedSymbols,
972 OptLevel, Freestanding, TMBuilder);
973 auto CacheEntryPath = CacheEntry.getEntryPath();
976 auto ErrOrBuffer = CacheEntry.tryLoadingBuffer();
977 DEBUG(dbgs() << "Cache " << (ErrOrBuffer ? "hit" : "miss") << " '"
978 << CacheEntryPath << "' for buffer " << count << " "
979 << ModuleIdentifier << "\n");
983 if (SavedObjectsDirectoryPath.empty())
984 ProducedBinaries[count] = std::move(ErrOrBuffer.get());
986 ProducedBinaryFiles[count] = writeGeneratedObject(
987 count, CacheEntryPath, SavedObjectsDirectoryPath,
994 Context.setDiscardValueNames(LTODiscardValueNames);
995 Context.enableDebugTypeODRUniquing();
996 auto DiagFileOrErr = lto::setupOptimizationRemarks(
997 Context, LTORemarksFilename, LTOPassRemarksWithHotness, count);
998 if (!DiagFileOrErr) {
999 errs() << "Error: " << toString(DiagFileOrErr.takeError()) << "\n";
1000 report_fatal_error("ThinLTO: Can't get an output file for the "
1006 loadModuleFromBuffer(ModuleBuffer.getMemBuffer(), Context, false,
1007 /*IsImporting*/ false);
1009 // Save temps: original file.
1010 saveTempBitcode(*TheModule, SaveTempsDir, count, ".0.original.bc");
1012 auto &ImportList = ImportLists[ModuleIdentifier];
1013 // Run the main process now, and generates a binary
1014 auto OutputBuffer = ProcessThinLTOModule(
1015 *TheModule, *Index, ModuleMap, *TMBuilder.create(), ImportList,
1016 ExportList, GUIDPreservedSymbols,
1017 ModuleToDefinedGVSummaries[ModuleIdentifier], CacheOptions,
1018 DisableCodeGen, SaveTempsDir, Freestanding, OptLevel, count);
1020 // Commit to the cache (if enabled)
1021 CacheEntry.write(*OutputBuffer);
1023 if (SavedObjectsDirectoryPath.empty()) {
1024 // We need to generated a memory buffer for the linker.
1025 if (!CacheEntryPath.empty()) {
1026 // Cache is enabled, reload from the cache
1027 // We do this to lower memory pressuree: the buffer is on the heap
1028 // and releasing it frees memory that can be used for the next input
1029 // file. The final binary link will read from the VFS cache
1030 // (hopefully!) or from disk if the memory pressure wasn't too high.
1031 auto ReloadedBufferOrErr = CacheEntry.tryLoadingBuffer();
1032 if (auto EC = ReloadedBufferOrErr.getError()) {
1033 // On error, keeping the preexisting buffer and printing a
1034 // diagnostic is more friendly than just crashing.
1035 errs() << "error: can't reload cached file '" << CacheEntryPath
1036 << "': " << EC.message() << "\n";
1038 OutputBuffer = std::move(*ReloadedBufferOrErr);
1041 ProducedBinaries[count] = std::move(OutputBuffer);
1044 ProducedBinaryFiles[count] = writeGeneratedObject(
1045 count, CacheEntryPath, SavedObjectsDirectoryPath, *OutputBuffer);
1050 pruneCache(CacheOptions.Path, CacheOptions.Policy);
1052 // If statistics were requested, print them out now.
1053 if (llvm::AreStatisticsEnabled())
1054 llvm::PrintStatistics();
1055 reportAndResetTimings();