1 //===-ThinLTOCodeGenerator.cpp - LLVM Link Time Optimizer -----------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file implements the Thin Link Time Optimization library. This library is
11 // intended to be used by linker to optimize code at link time.
13 //===----------------------------------------------------------------------===//
15 #include "llvm/LTO/legacy/ThinLTOCodeGenerator.h"
17 #ifdef HAVE_LLVM_REVISION
18 #include "LLVMLTORevision.h"
21 #include "llvm/ADT/Statistic.h"
22 #include "llvm/ADT/StringExtras.h"
23 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
24 #include "llvm/Analysis/ProfileSummaryInfo.h"
25 #include "llvm/Analysis/TargetLibraryInfo.h"
26 #include "llvm/Analysis/TargetTransformInfo.h"
27 #include "llvm/Bitcode/BitcodeReader.h"
28 #include "llvm/Bitcode/BitcodeWriter.h"
29 #include "llvm/Bitcode/BitcodeWriterPass.h"
30 #include "llvm/ExecutionEngine/ObjectMemoryBuffer.h"
31 #include "llvm/IR/DiagnosticPrinter.h"
32 #include "llvm/IR/LLVMContext.h"
33 #include "llvm/IR/LegacyPassManager.h"
34 #include "llvm/IR/Mangler.h"
35 #include "llvm/IRReader/IRReader.h"
36 #include "llvm/LTO/LTO.h"
37 #include "llvm/Linker/Linker.h"
38 #include "llvm/MC/SubtargetFeature.h"
39 #include "llvm/Object/IRObjectFile.h"
40 #include "llvm/Object/ModuleSummaryIndexObjectFile.h"
41 #include "llvm/Support/CachePruning.h"
42 #include "llvm/Support/Debug.h"
43 #include "llvm/Support/Error.h"
44 #include "llvm/Support/Path.h"
45 #include "llvm/Support/SHA1.h"
46 #include "llvm/Support/TargetRegistry.h"
47 #include "llvm/Support/ThreadPool.h"
48 #include "llvm/Support/Threading.h"
49 #include "llvm/Support/ToolOutputFile.h"
50 #include "llvm/Target/TargetMachine.h"
51 #include "llvm/Transforms/IPO.h"
52 #include "llvm/Transforms/IPO/FunctionImport.h"
53 #include "llvm/Transforms/IPO/Internalize.h"
54 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
55 #include "llvm/Transforms/ObjCARC.h"
56 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
62 #define DEBUG_TYPE "thinlto"
65 // Flags -discard-value-names, defined in LTOCodeGenerator.cpp
66 extern cl::opt<bool> LTODiscardValueNames;
67 extern cl::opt<std::string> LTORemarksFilename;
68 extern cl::opt<bool> LTOPassRemarksWithHotness;
74 ThreadCount("threads", cl::init(llvm::heavyweight_hardware_concurrency()));
76 Expected<std::unique_ptr<tool_output_file>>
77 setupOptimizationRemarks(LLVMContext &Ctx, int Count) {
78 if (LTOPassRemarksWithHotness)
79 Ctx.setDiagnosticHotnessRequested(true);
81 if (LTORemarksFilename.empty())
84 std::string FileName =
85 LTORemarksFilename + ".thin." + llvm::utostr(Count) + ".yaml";
87 auto DiagnosticOutputFile =
88 llvm::make_unique<tool_output_file>(FileName, EC, sys::fs::F_None);
90 return errorCodeToError(EC);
91 Ctx.setDiagnosticsOutputFile(
92 llvm::make_unique<yaml::Output>(DiagnosticOutputFile->os()));
93 DiagnosticOutputFile->keep();
94 return std::move(DiagnosticOutputFile);
97 // Simple helper to save temporary files for debug.
98 static void saveTempBitcode(const Module &TheModule, StringRef TempDir,
99 unsigned count, StringRef Suffix) {
102 // User asked to save temps, let dump the bitcode file after import.
103 std::string SaveTempPath = (TempDir + llvm::utostr(count) + Suffix).str();
105 raw_fd_ostream OS(SaveTempPath, EC, sys::fs::F_None);
107 report_fatal_error(Twine("Failed to open ") + SaveTempPath +
108 " to save optimized bitcode\n");
109 WriteBitcodeToFile(&TheModule, OS, /* ShouldPreserveUseListOrder */ true);
112 static const GlobalValueSummary *
113 getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
114 // If there is any strong definition anywhere, get it.
115 auto StrongDefForLinker = llvm::find_if(
116 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
117 auto Linkage = Summary->linkage();
118 return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
119 !GlobalValue::isWeakForLinker(Linkage);
121 if (StrongDefForLinker != GVSummaryList.end())
122 return StrongDefForLinker->get();
123 // Get the first *linker visible* definition for this global in the summary
125 auto FirstDefForLinker = llvm::find_if(
126 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
127 auto Linkage = Summary->linkage();
128 return !GlobalValue::isAvailableExternallyLinkage(Linkage);
130 // Extern templates can be emitted as available_externally.
131 if (FirstDefForLinker == GVSummaryList.end())
133 return FirstDefForLinker->get();
136 // Populate map of GUID to the prevailing copy for any multiply defined
137 // symbols. Currently assume first copy is prevailing, or any strong
138 // definition. Can be refined with Linker information in the future.
139 static void computePrevailingCopies(
140 const ModuleSummaryIndex &Index,
141 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy) {
142 auto HasMultipleCopies = [&](const GlobalValueSummaryList &GVSummaryList) {
143 return GVSummaryList.size() > 1;
146 for (auto &I : Index) {
147 if (HasMultipleCopies(I.second))
148 PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second);
152 static StringMap<MemoryBufferRef>
153 generateModuleMap(const std::vector<ThinLTOBuffer> &Modules) {
154 StringMap<MemoryBufferRef> ModuleMap;
155 for (auto &ModuleBuffer : Modules) {
156 assert(ModuleMap.find(ModuleBuffer.getBufferIdentifier()) ==
158 "Expect unique Buffer Identifier");
159 ModuleMap[ModuleBuffer.getBufferIdentifier()] = ModuleBuffer.getMemBuffer();
164 static void promoteModule(Module &TheModule, const ModuleSummaryIndex &Index) {
165 if (renameModuleForThinLTO(TheModule, Index))
166 report_fatal_error("renameModuleForThinLTO failed");
169 static std::unique_ptr<Module>
170 loadModuleFromBuffer(const MemoryBufferRef &Buffer, LLVMContext &Context,
171 bool Lazy, bool IsImporting) {
173 Expected<std::unique_ptr<Module>> ModuleOrErr =
175 ? getLazyBitcodeModule(Buffer, Context,
176 /* ShouldLazyLoadMetadata */ true, IsImporting)
177 : parseBitcodeFile(Buffer, Context);
179 handleAllErrors(ModuleOrErr.takeError(), [&](ErrorInfoBase &EIB) {
180 SMDiagnostic Err = SMDiagnostic(Buffer.getBufferIdentifier(),
181 SourceMgr::DK_Error, EIB.message());
182 Err.print("ThinLTO", errs());
184 report_fatal_error("Can't load module, abort.");
186 return std::move(ModuleOrErr.get());
190 crossImportIntoModule(Module &TheModule, const ModuleSummaryIndex &Index,
191 StringMap<MemoryBufferRef> &ModuleMap,
192 const FunctionImporter::ImportMapTy &ImportList) {
193 auto Loader = [&](StringRef Identifier) {
194 return loadModuleFromBuffer(ModuleMap[Identifier], TheModule.getContext(),
195 /*Lazy=*/true, /*IsImporting*/ true);
198 FunctionImporter Importer(Index, Loader);
199 Expected<bool> Result = Importer.importFunctions(TheModule, ImportList);
201 handleAllErrors(Result.takeError(), [&](ErrorInfoBase &EIB) {
202 SMDiagnostic Err = SMDiagnostic(TheModule.getModuleIdentifier(),
203 SourceMgr::DK_Error, EIB.message());
204 Err.print("ThinLTO", errs());
206 report_fatal_error("importFunctions failed");
210 static void optimizeModule(Module &TheModule, TargetMachine &TM,
212 // Populate the PassManager
213 PassManagerBuilder PMB;
214 PMB.LibraryInfo = new TargetLibraryInfoImpl(TM.getTargetTriple());
215 PMB.Inliner = createFunctionInliningPass();
216 // FIXME: should get it from the bitcode?
217 PMB.OptLevel = OptLevel;
218 PMB.LoopVectorize = true;
219 PMB.SLPVectorize = true;
220 PMB.VerifyInput = true;
221 PMB.VerifyOutput = false;
223 legacy::PassManager PM;
225 // Add the TTI (required to inform the vectorizer about register size for
227 PM.add(createTargetTransformInfoWrapperPass(TM.getTargetIRAnalysis()));
230 PMB.populateThinLTOPassManager(PM);
235 // Convert the PreservedSymbols map from "Name" based to "GUID" based.
236 static DenseSet<GlobalValue::GUID>
237 computeGUIDPreservedSymbols(const StringSet<> &PreservedSymbols,
238 const Triple &TheTriple) {
239 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols(PreservedSymbols.size());
240 for (auto &Entry : PreservedSymbols) {
241 StringRef Name = Entry.first();
242 if (TheTriple.isOSBinFormatMachO() && Name.size() > 0 && Name[0] == '_')
243 Name = Name.drop_front();
244 GUIDPreservedSymbols.insert(GlobalValue::getGUID(Name));
246 return GUIDPreservedSymbols;
249 std::unique_ptr<MemoryBuffer> codegenModule(Module &TheModule,
251 SmallVector<char, 128> OutputBuffer;
255 raw_svector_ostream OS(OutputBuffer);
256 legacy::PassManager PM;
258 // If the bitcode files contain ARC code and were compiled with optimization,
259 // the ObjCARCContractPass must be run, so do it unconditionally here.
260 PM.add(createObjCARCContractPass());
262 // Setup the codegen now.
263 if (TM.addPassesToEmitFile(PM, OS, TargetMachine::CGFT_ObjectFile,
264 /* DisableVerify */ true))
265 report_fatal_error("Failed to setup codegen");
267 // Run codegen now. resulting binary is in OutputBuffer.
270 return make_unique<ObjectMemoryBuffer>(std::move(OutputBuffer));
273 /// Manage caching for a single Module.
274 class ModuleCacheEntry {
275 SmallString<128> EntryPath;
278 // Create a cache entry. This compute a unique hash for the Module considering
279 // the current list of export/import, and offer an interface to query to
280 // access the content in the cache.
282 StringRef CachePath, const ModuleSummaryIndex &Index, StringRef ModuleID,
283 const FunctionImporter::ImportMapTy &ImportList,
284 const FunctionImporter::ExportSetTy &ExportList,
285 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
286 const GVSummaryMapTy &DefinedFunctions,
287 const DenseSet<GlobalValue::GUID> &PreservedSymbols, unsigned OptLevel,
288 const TargetMachineBuilder &TMBuilder) {
289 if (CachePath.empty())
292 if (!Index.modulePaths().count(ModuleID))
293 // The module does not have an entry, it can't have a hash at all
296 // Compute the unique hash for this entry
297 // This is based on the current compiler version, the module itself, the
298 // export list, the hash for every single module in the import list, the
299 // list of ResolvedODR for the module, and the list of preserved symbols.
301 // Include the hash for the current module
302 auto ModHash = Index.getModuleHash(ModuleID);
304 if (all_of(ModHash, [](uint32_t V) { return V == 0; }))
305 // No hash entry, no caching!
310 // Include the parts of the LTO configuration that affect code generation.
311 auto AddString = [&](StringRef Str) {
313 Hasher.update(ArrayRef<uint8_t>{0});
315 auto AddUnsigned = [&](unsigned I) {
321 Hasher.update(ArrayRef<uint8_t>{Data, 4});
324 // Start with the compiler revision
325 Hasher.update(LLVM_VERSION_STRING);
326 #ifdef HAVE_LLVM_REVISION
327 Hasher.update(LLVM_REVISION);
330 // Hash the optimization level and the target machine settings.
331 AddString(TMBuilder.MCpu);
332 // FIXME: Hash more of Options. For now all clients initialize Options from
333 // command-line flags (which is unsupported in production), but may set
334 // RelaxELFRelocations. The clang driver can also pass FunctionSections,
335 // DataSections and DebuggerTuning via command line flags.
336 AddUnsigned(TMBuilder.Options.RelaxELFRelocations);
337 AddUnsigned(TMBuilder.Options.FunctionSections);
338 AddUnsigned(TMBuilder.Options.DataSections);
339 AddUnsigned((unsigned)TMBuilder.Options.DebuggerTuning);
340 AddString(TMBuilder.MAttr);
341 if (TMBuilder.RelocModel)
342 AddUnsigned(*TMBuilder.RelocModel);
343 AddUnsigned(TMBuilder.CGOptLevel);
344 AddUnsigned(OptLevel);
346 Hasher.update(ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));
347 for (auto F : ExportList)
348 // The export list can impact the internalization, be conservative here
349 Hasher.update(ArrayRef<uint8_t>((uint8_t *)&F, sizeof(F)));
351 // Include the hash for every module we import functions from
352 for (auto &Entry : ImportList) {
353 auto ModHash = Index.getModuleHash(Entry.first());
354 Hasher.update(ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));
357 // Include the hash for the resolved ODR.
358 for (auto &Entry : ResolvedODR) {
359 Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&Entry.first,
360 sizeof(GlobalValue::GUID)));
361 Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&Entry.second,
362 sizeof(GlobalValue::LinkageTypes)));
365 // Include the hash for the preserved symbols.
366 for (auto &Entry : PreservedSymbols) {
367 if (DefinedFunctions.count(Entry))
369 ArrayRef<uint8_t>((const uint8_t *)&Entry, sizeof(GlobalValue::GUID)));
372 sys::path::append(EntryPath, CachePath, toHex(Hasher.result()));
375 // Access the path to this entry in the cache.
376 StringRef getEntryPath() { return EntryPath; }
378 // Try loading the buffer for this cache entry.
379 ErrorOr<std::unique_ptr<MemoryBuffer>> tryLoadingBuffer() {
380 if (EntryPath.empty())
381 return std::error_code();
382 return MemoryBuffer::getFile(EntryPath);
385 // Cache the Produced object file
386 void write(const MemoryBuffer &OutputBuffer) {
387 if (EntryPath.empty())
390 // Write to a temporary to avoid race condition
391 SmallString<128> TempFilename;
394 sys::fs::createTemporaryFile("Thin", "tmp.o", TempFD, TempFilename);
396 errs() << "Error: " << EC.message() << "\n";
397 report_fatal_error("ThinLTO: Can't get a temporary file");
400 raw_fd_ostream OS(TempFD, /* ShouldClose */ true);
401 OS << OutputBuffer.getBuffer();
403 // Rename to final destination (hopefully race condition won't matter here)
404 EC = sys::fs::rename(TempFilename, EntryPath);
406 sys::fs::remove(TempFilename);
407 raw_fd_ostream OS(EntryPath, EC, sys::fs::F_None);
409 report_fatal_error(Twine("Failed to open ") + EntryPath +
410 " to save cached entry\n");
411 OS << OutputBuffer.getBuffer();
416 static std::unique_ptr<MemoryBuffer>
417 ProcessThinLTOModule(Module &TheModule, ModuleSummaryIndex &Index,
418 StringMap<MemoryBufferRef> &ModuleMap, TargetMachine &TM,
419 const FunctionImporter::ImportMapTy &ImportList,
420 const FunctionImporter::ExportSetTy &ExportList,
421 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
422 const GVSummaryMapTy &DefinedGlobals,
423 const ThinLTOCodeGenerator::CachingOptions &CacheOptions,
424 bool DisableCodeGen, StringRef SaveTempsDir,
425 unsigned OptLevel, unsigned count) {
427 // "Benchmark"-like optimization: single-source case
428 bool SingleModule = (ModuleMap.size() == 1);
431 promoteModule(TheModule, Index);
433 // Apply summary-based LinkOnce/Weak resolution decisions.
434 thinLTOResolveWeakForLinkerModule(TheModule, DefinedGlobals);
436 // Save temps: after promotion.
437 saveTempBitcode(TheModule, SaveTempsDir, count, ".1.promoted.bc");
440 // Be friendly and don't nuke totally the module when the client didn't
441 // supply anything to preserve.
442 if (!ExportList.empty() || !GUIDPreservedSymbols.empty()) {
443 // Apply summary-based internalization decisions.
444 thinLTOInternalizeModule(TheModule, DefinedGlobals);
447 // Save internalized bitcode
448 saveTempBitcode(TheModule, SaveTempsDir, count, ".2.internalized.bc");
451 crossImportIntoModule(TheModule, Index, ModuleMap, ImportList);
453 // Save temps: after cross-module import.
454 saveTempBitcode(TheModule, SaveTempsDir, count, ".3.imported.bc");
457 optimizeModule(TheModule, TM, OptLevel);
459 saveTempBitcode(TheModule, SaveTempsDir, count, ".4.opt.bc");
461 if (DisableCodeGen) {
462 // Configured to stop before CodeGen, serialize the bitcode and return.
463 SmallVector<char, 128> OutputBuffer;
465 raw_svector_ostream OS(OutputBuffer);
466 ProfileSummaryInfo PSI(TheModule);
467 auto Index = buildModuleSummaryIndex(TheModule, nullptr, nullptr);
468 WriteBitcodeToFile(&TheModule, OS, true, &Index);
470 return make_unique<ObjectMemoryBuffer>(std::move(OutputBuffer));
473 return codegenModule(TheModule, TM);
476 /// Resolve LinkOnce/Weak symbols. Record resolutions in the \p ResolvedODR map
477 /// for caching, and in the \p Index for application during the ThinLTO
478 /// backends. This is needed for correctness for exported symbols (ensure
479 /// at least one copy kept) and a compile-time optimization (to drop duplicate
480 /// copies when possible).
481 static void resolveWeakForLinkerInIndex(
482 ModuleSummaryIndex &Index,
483 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>>
486 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
487 computePrevailingCopies(Index, PrevailingCopy);
489 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
490 const auto &Prevailing = PrevailingCopy.find(GUID);
491 // Not in map means that there was only one copy, which must be prevailing.
492 if (Prevailing == PrevailingCopy.end())
494 return Prevailing->second == S;
497 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
498 GlobalValue::GUID GUID,
499 GlobalValue::LinkageTypes NewLinkage) {
500 ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
503 thinLTOResolveWeakForLinkerInIndex(Index, isPrevailing, recordNewLinkage);
506 // Initialize the TargetMachine builder for a given Triple
507 static void initTMBuilder(TargetMachineBuilder &TMBuilder,
508 const Triple &TheTriple) {
509 // Set a default CPU for Darwin triples (copied from LTOCodeGenerator).
510 // FIXME this looks pretty terrible...
511 if (TMBuilder.MCpu.empty() && TheTriple.isOSDarwin()) {
512 if (TheTriple.getArch() == llvm::Triple::x86_64)
513 TMBuilder.MCpu = "core2";
514 else if (TheTriple.getArch() == llvm::Triple::x86)
515 TMBuilder.MCpu = "yonah";
516 else if (TheTriple.getArch() == llvm::Triple::aarch64)
517 TMBuilder.MCpu = "cyclone";
519 TMBuilder.TheTriple = std::move(TheTriple);
522 } // end anonymous namespace
524 void ThinLTOCodeGenerator::addModule(StringRef Identifier, StringRef Data) {
525 ThinLTOBuffer Buffer(Data, Identifier);
526 if (Modules.empty()) {
527 // First module added, so initialize the triple and some options
530 ErrorOr<std::string> TripleOrErr = expectedToErrorOrAndEmitErrors(
531 Context, getBitcodeTargetTriple(Buffer.getMemBuffer()));
533 TripleStr = *TripleOrErr;
534 Triple TheTriple(TripleStr);
535 initTMBuilder(TMBuilder, Triple(TheTriple));
541 ErrorOr<std::string> TripleOrErr = expectedToErrorOrAndEmitErrors(
542 Context, getBitcodeTargetTriple(Buffer.getMemBuffer()));
544 TripleStr = *TripleOrErr;
545 assert(TMBuilder.TheTriple.str() == TripleStr &&
546 "ThinLTO modules with different triple not supported");
549 Modules.push_back(Buffer);
552 void ThinLTOCodeGenerator::preserveSymbol(StringRef Name) {
553 PreservedSymbols.insert(Name);
556 void ThinLTOCodeGenerator::crossReferenceSymbol(StringRef Name) {
557 // FIXME: At the moment, we don't take advantage of this extra information,
558 // we're conservatively considering cross-references as preserved.
559 // CrossReferencedSymbols.insert(Name);
560 PreservedSymbols.insert(Name);
563 // TargetMachine factory
564 std::unique_ptr<TargetMachine> TargetMachineBuilder::create() const {
566 const Target *TheTarget =
567 TargetRegistry::lookupTarget(TheTriple.str(), ErrMsg);
569 report_fatal_error("Can't load target for this Triple: " + ErrMsg);
572 // Use MAttr as the default set of features.
573 SubtargetFeatures Features(MAttr);
574 Features.getDefaultSubtargetFeatures(TheTriple);
575 std::string FeatureStr = Features.getString();
577 return std::unique_ptr<TargetMachine>(TheTarget->createTargetMachine(
578 TheTriple.str(), MCpu, FeatureStr, Options, RelocModel,
579 CodeModel::Default, CGOptLevel));
583 * Produce the combined summary index from all the bitcode files:
586 std::unique_ptr<ModuleSummaryIndex> ThinLTOCodeGenerator::linkCombinedIndex() {
587 std::unique_ptr<ModuleSummaryIndex> CombinedIndex;
588 uint64_t NextModuleId = 0;
589 for (auto &ModuleBuffer : Modules) {
590 Expected<std::unique_ptr<object::ModuleSummaryIndexObjectFile>> ObjOrErr =
591 object::ModuleSummaryIndexObjectFile::create(
592 ModuleBuffer.getMemBuffer());
595 logAllUnhandledErrors(
596 ObjOrErr.takeError(), errs(),
597 "error: can't create ModuleSummaryIndexObjectFile for buffer: ");
600 auto Index = (*ObjOrErr)->takeIndex();
602 CombinedIndex->mergeFrom(std::move(Index), ++NextModuleId);
604 CombinedIndex = std::move(Index);
607 return CombinedIndex;
611 * Perform promotion and renaming of exported internal functions.
612 * Index is updated to reflect linkage changes from weak resolution.
614 void ThinLTOCodeGenerator::promote(Module &TheModule,
615 ModuleSummaryIndex &Index) {
616 auto ModuleCount = Index.modulePaths().size();
617 auto ModuleIdentifier = TheModule.getModuleIdentifier();
619 // Collect for each module the list of function it defines (GUID -> Summary).
620 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
621 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
623 // Convert the preserved symbols set from string to GUID
624 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
625 PreservedSymbols, Triple(TheModule.getTargetTriple()));
627 // Compute "dead" symbols, we don't want to import/export these!
628 auto DeadSymbols = computeDeadSymbols(Index, GUIDPreservedSymbols);
630 // Generate import/export list
631 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
632 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
633 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
634 ExportLists, &DeadSymbols);
636 // Resolve LinkOnce/Weak symbols.
637 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
638 resolveWeakForLinkerInIndex(Index, ResolvedODR);
640 thinLTOResolveWeakForLinkerModule(
641 TheModule, ModuleToDefinedGVSummaries[ModuleIdentifier]);
643 // Promote the exported values in the index, so that they are promoted
645 auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) {
646 const auto &ExportList = ExportLists.find(ModuleIdentifier);
647 return (ExportList != ExportLists.end() &&
648 ExportList->second.count(GUID)) ||
649 GUIDPreservedSymbols.count(GUID);
651 thinLTOInternalizeAndPromoteInIndex(Index, isExported);
653 promoteModule(TheModule, Index);
657 * Perform cross-module importing for the module identified by ModuleIdentifier.
659 void ThinLTOCodeGenerator::crossModuleImport(Module &TheModule,
660 ModuleSummaryIndex &Index) {
661 auto ModuleMap = generateModuleMap(Modules);
662 auto ModuleCount = Index.modulePaths().size();
664 // Collect for each module the list of function it defines (GUID -> Summary).
665 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
666 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
668 // Convert the preserved symbols set from string to GUID
669 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
670 PreservedSymbols, Triple(TheModule.getTargetTriple()));
672 // Compute "dead" symbols, we don't want to import/export these!
673 auto DeadSymbols = computeDeadSymbols(Index, GUIDPreservedSymbols);
675 // Generate import/export list
676 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
677 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
678 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
679 ExportLists, &DeadSymbols);
680 auto &ImportList = ImportLists[TheModule.getModuleIdentifier()];
682 crossImportIntoModule(TheModule, Index, ModuleMap, ImportList);
686 * Compute the list of summaries needed for importing into module.
688 void ThinLTOCodeGenerator::gatherImportedSummariesForModule(
689 StringRef ModulePath, ModuleSummaryIndex &Index,
690 std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex) {
691 auto ModuleCount = Index.modulePaths().size();
693 // Collect for each module the list of function it defines (GUID -> Summary).
694 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
695 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
697 // Generate import/export list
698 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
699 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
700 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
703 llvm::gatherImportedSummariesForModule(ModulePath, ModuleToDefinedGVSummaries,
704 ImportLists[ModulePath],
705 ModuleToSummariesForIndex);
709 * Emit the list of files needed for importing into module.
711 void ThinLTOCodeGenerator::emitImports(StringRef ModulePath,
712 StringRef OutputName,
713 ModuleSummaryIndex &Index) {
714 auto ModuleCount = Index.modulePaths().size();
716 // Collect for each module the list of function it defines (GUID -> Summary).
717 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
718 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
720 // Generate import/export list
721 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
722 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
723 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
727 if ((EC = EmitImportsFiles(ModulePath, OutputName, ImportLists[ModulePath])))
728 report_fatal_error(Twine("Failed to open ") + OutputName +
729 " to save imports lists\n");
733 * Perform internalization. Index is updated to reflect linkage changes.
735 void ThinLTOCodeGenerator::internalize(Module &TheModule,
736 ModuleSummaryIndex &Index) {
737 initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
738 auto ModuleCount = Index.modulePaths().size();
739 auto ModuleIdentifier = TheModule.getModuleIdentifier();
741 // Convert the preserved symbols set from string to GUID
742 auto GUIDPreservedSymbols =
743 computeGUIDPreservedSymbols(PreservedSymbols, TMBuilder.TheTriple);
745 // Collect for each module the list of function it defines (GUID -> Summary).
746 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
747 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
749 // Compute "dead" symbols, we don't want to import/export these!
750 auto DeadSymbols = computeDeadSymbols(Index, GUIDPreservedSymbols);
752 // Generate import/export list
753 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
754 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
755 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
756 ExportLists, &DeadSymbols);
757 auto &ExportList = ExportLists[ModuleIdentifier];
759 // Be friendly and don't nuke totally the module when the client didn't
760 // supply anything to preserve.
761 if (ExportList.empty() && GUIDPreservedSymbols.empty())
765 auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) {
766 const auto &ExportList = ExportLists.find(ModuleIdentifier);
767 return (ExportList != ExportLists.end() &&
768 ExportList->second.count(GUID)) ||
769 GUIDPreservedSymbols.count(GUID);
771 thinLTOInternalizeAndPromoteInIndex(Index, isExported);
772 thinLTOInternalizeModule(TheModule,
773 ModuleToDefinedGVSummaries[ModuleIdentifier]);
777 * Perform post-importing ThinLTO optimizations.
779 void ThinLTOCodeGenerator::optimize(Module &TheModule) {
780 initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
783 optimizeModule(TheModule, *TMBuilder.create(), OptLevel);
787 * Perform ThinLTO CodeGen.
789 std::unique_ptr<MemoryBuffer> ThinLTOCodeGenerator::codegen(Module &TheModule) {
790 initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
791 return codegenModule(TheModule, *TMBuilder.create());
794 /// Write out the generated object file, either from CacheEntryPath or from
795 /// OutputBuffer, preferring hard-link when possible.
796 /// Returns the path to the generated file in SavedObjectsDirectoryPath.
797 static std::string writeGeneratedObject(int count, StringRef CacheEntryPath,
798 StringRef SavedObjectsDirectoryPath,
799 const MemoryBuffer &OutputBuffer) {
800 SmallString<128> OutputPath(SavedObjectsDirectoryPath);
801 llvm::sys::path::append(OutputPath, Twine(count) + ".thinlto.o");
802 OutputPath.c_str(); // Ensure the string is null terminated.
803 if (sys::fs::exists(OutputPath))
804 sys::fs::remove(OutputPath);
806 // We don't return a memory buffer to the linker, just a list of files.
807 if (!CacheEntryPath.empty()) {
808 // Cache is enabled, hard-link the entry (or copy if hard-link fails).
809 auto Err = sys::fs::create_hard_link(CacheEntryPath, OutputPath);
811 return OutputPath.str();
812 // Hard linking failed, try to copy.
813 Err = sys::fs::copy_file(CacheEntryPath, OutputPath);
815 return OutputPath.str();
816 // Copy failed (could be because the CacheEntry was removed from the cache
817 // in the meantime by another process), fall back and try to write down the
818 // buffer to the output.
819 errs() << "error: can't link or copy from cached entry '" << CacheEntryPath
820 << "' to '" << OutputPath << "'\n";
822 // No cache entry, just write out the buffer.
824 raw_fd_ostream OS(OutputPath, Err, sys::fs::F_None);
826 report_fatal_error("Can't open output '" + OutputPath + "'\n");
827 OS << OutputBuffer.getBuffer();
828 return OutputPath.str();
831 // Main entry point for the ThinLTO processing
832 void ThinLTOCodeGenerator::run() {
833 // Prepare the resulting object vector
834 assert(ProducedBinaries.empty() && "The generator should not be reused");
835 if (SavedObjectsDirectoryPath.empty())
836 ProducedBinaries.resize(Modules.size());
838 sys::fs::create_directories(SavedObjectsDirectoryPath);
840 sys::fs::is_directory(SavedObjectsDirectoryPath, IsDir);
842 report_fatal_error("Unexistent dir: '" + SavedObjectsDirectoryPath + "'");
843 ProducedBinaryFiles.resize(Modules.size());
847 // Perform only parallel codegen and return.
850 for (auto &ModuleBuffer : Modules) {
851 Pool.async([&](int count) {
853 Context.setDiscardValueNames(LTODiscardValueNames);
857 loadModuleFromBuffer(ModuleBuffer.getMemBuffer(), Context, false,
858 /*IsImporting*/ false);
861 auto OutputBuffer = codegen(*TheModule);
862 if (SavedObjectsDirectoryPath.empty())
863 ProducedBinaries[count] = std::move(OutputBuffer);
865 ProducedBinaryFiles[count] = writeGeneratedObject(
866 count, "", SavedObjectsDirectoryPath, *OutputBuffer);
873 // Sequential linking phase
874 auto Index = linkCombinedIndex();
876 // Save temps: index.
877 if (!SaveTempsDir.empty()) {
878 auto SaveTempPath = SaveTempsDir + "index.bc";
880 raw_fd_ostream OS(SaveTempPath, EC, sys::fs::F_None);
882 report_fatal_error(Twine("Failed to open ") + SaveTempPath +
883 " to save optimized bitcode\n");
884 WriteIndexToFile(*Index, OS);
888 // Prepare the module map.
889 auto ModuleMap = generateModuleMap(Modules);
890 auto ModuleCount = Modules.size();
892 // Collect for each module the list of function it defines (GUID -> Summary).
893 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
894 Index->collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
896 // Convert the preserved symbols set from string to GUID, this is needed for
897 // computing the caching hash and the internalization.
898 auto GUIDPreservedSymbols =
899 computeGUIDPreservedSymbols(PreservedSymbols, TMBuilder.TheTriple);
901 // Compute "dead" symbols, we don't want to import/export these!
902 auto DeadSymbols = computeDeadSymbols(*Index, GUIDPreservedSymbols);
904 // Collect the import/export lists for all modules from the call-graph in the
906 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
907 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
908 ComputeCrossModuleImport(*Index, ModuleToDefinedGVSummaries, ImportLists,
909 ExportLists, &DeadSymbols);
911 // We use a std::map here to be able to have a defined ordering when
912 // producing a hash for the cache entry.
913 // FIXME: we should be able to compute the caching hash for the entry based
914 // on the index, and nuke this map.
915 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
917 // Resolve LinkOnce/Weak symbols, this has to be computed early because it
918 // impacts the caching.
919 resolveWeakForLinkerInIndex(*Index, ResolvedODR);
921 auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) {
922 const auto &ExportList = ExportLists.find(ModuleIdentifier);
923 return (ExportList != ExportLists.end() &&
924 ExportList->second.count(GUID)) ||
925 GUIDPreservedSymbols.count(GUID);
928 // Use global summary-based analysis to identify symbols that can be
929 // internalized (because they aren't exported or preserved as per callback).
930 // Changes are made in the index, consumed in the ThinLTO backends.
931 thinLTOInternalizeAndPromoteInIndex(*Index, isExported);
933 // Make sure that every module has an entry in the ExportLists and
934 // ResolvedODR maps to enable threaded access to these maps below.
935 for (auto &DefinedGVSummaries : ModuleToDefinedGVSummaries) {
936 ExportLists[DefinedGVSummaries.first()];
937 ResolvedODR[DefinedGVSummaries.first()];
940 // Compute the ordering we will process the inputs: the rough heuristic here
941 // is to sort them per size so that the largest module get schedule as soon as
942 // possible. This is purely a compile-time optimization.
943 std::vector<int> ModulesOrdering;
944 ModulesOrdering.resize(Modules.size());
945 std::iota(ModulesOrdering.begin(), ModulesOrdering.end(), 0);
946 std::sort(ModulesOrdering.begin(), ModulesOrdering.end(),
947 [&](int LeftIndex, int RightIndex) {
948 auto LSize = Modules[LeftIndex].getBuffer().size();
949 auto RSize = Modules[RightIndex].getBuffer().size();
950 return LSize > RSize;
953 // Parallel optimizer + codegen
955 ThreadPool Pool(ThreadCount);
956 for (auto IndexCount : ModulesOrdering) {
957 auto &ModuleBuffer = Modules[IndexCount];
958 Pool.async([&](int count) {
959 auto ModuleIdentifier = ModuleBuffer.getBufferIdentifier();
960 auto &ExportList = ExportLists[ModuleIdentifier];
962 auto &DefinedFunctions = ModuleToDefinedGVSummaries[ModuleIdentifier];
964 // The module may be cached, this helps handling it.
965 ModuleCacheEntry CacheEntry(CacheOptions.Path, *Index, ModuleIdentifier,
966 ImportLists[ModuleIdentifier], ExportList,
967 ResolvedODR[ModuleIdentifier],
968 DefinedFunctions, GUIDPreservedSymbols,
969 OptLevel, TMBuilder);
970 auto CacheEntryPath = CacheEntry.getEntryPath();
973 auto ErrOrBuffer = CacheEntry.tryLoadingBuffer();
974 DEBUG(dbgs() << "Cache " << (ErrOrBuffer ? "hit" : "miss") << " '"
975 << CacheEntryPath << "' for buffer " << count << " "
976 << ModuleIdentifier << "\n");
980 if (SavedObjectsDirectoryPath.empty())
981 ProducedBinaries[count] = std::move(ErrOrBuffer.get());
983 ProducedBinaryFiles[count] = writeGeneratedObject(
984 count, CacheEntryPath, SavedObjectsDirectoryPath,
991 Context.setDiscardValueNames(LTODiscardValueNames);
992 Context.enableDebugTypeODRUniquing();
993 auto DiagFileOrErr = setupOptimizationRemarks(Context, count);
994 if (!DiagFileOrErr) {
995 errs() << "Error: " << toString(DiagFileOrErr.takeError()) << "\n";
996 report_fatal_error("ThinLTO: Can't get an output file for the "
1002 loadModuleFromBuffer(ModuleBuffer.getMemBuffer(), Context, false,
1003 /*IsImporting*/ false);
1005 // Save temps: original file.
1006 saveTempBitcode(*TheModule, SaveTempsDir, count, ".0.original.bc");
1008 auto &ImportList = ImportLists[ModuleIdentifier];
1009 // Run the main process now, and generates a binary
1010 auto OutputBuffer = ProcessThinLTOModule(
1011 *TheModule, *Index, ModuleMap, *TMBuilder.create(), ImportList,
1012 ExportList, GUIDPreservedSymbols,
1013 ModuleToDefinedGVSummaries[ModuleIdentifier], CacheOptions,
1014 DisableCodeGen, SaveTempsDir, OptLevel, count);
1016 // Commit to the cache (if enabled)
1017 CacheEntry.write(*OutputBuffer);
1019 if (SavedObjectsDirectoryPath.empty()) {
1020 // We need to generated a memory buffer for the linker.
1021 if (!CacheEntryPath.empty()) {
1022 // Cache is enabled, reload from the cache
1023 // We do this to lower memory pressuree: the buffer is on the heap
1024 // and releasing it frees memory that can be used for the next input
1025 // file. The final binary link will read from the VFS cache
1026 // (hopefully!) or from disk if the memory pressure wasn't too high.
1027 auto ReloadedBufferOrErr = CacheEntry.tryLoadingBuffer();
1028 if (auto EC = ReloadedBufferOrErr.getError()) {
1029 // On error, keeping the preexisting buffer and printing a
1030 // diagnostic is more friendly than just crashing.
1031 errs() << "error: can't reload cached file '" << CacheEntryPath
1032 << "': " << EC.message() << "\n";
1034 OutputBuffer = std::move(*ReloadedBufferOrErr);
1037 ProducedBinaries[count] = std::move(OutputBuffer);
1040 ProducedBinaryFiles[count] = writeGeneratedObject(
1041 count, CacheEntryPath, SavedObjectsDirectoryPath, *OutputBuffer);
1046 CachePruning(CacheOptions.Path)
1047 .setPruningInterval(std::chrono::seconds(CacheOptions.PruningInterval))
1048 .setEntryExpiration(std::chrono::seconds(CacheOptions.Expiration))
1049 .setMaxSize(CacheOptions.MaxPercentageOfAvailableSpace)
1052 // If statistics were requested, print them out now.
1053 if (llvm::AreStatisticsEnabled())
1054 llvm::PrintStatistics();