1 //===-ThinLTOCodeGenerator.cpp - LLVM Link Time Optimizer -----------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file implements the Thin Link Time Optimization library. This library is
10 // intended to be used by linker to optimize code at link time.
12 //===----------------------------------------------------------------------===//
14 #include "llvm/LTO/legacy/ThinLTOCodeGenerator.h"
15 #include "llvm/Support/CommandLine.h"
17 #include "llvm/ADT/Statistic.h"
18 #include "llvm/ADT/StringExtras.h"
19 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
20 #include "llvm/Analysis/ProfileSummaryInfo.h"
21 #include "llvm/Analysis/TargetLibraryInfo.h"
22 #include "llvm/Analysis/TargetTransformInfo.h"
23 #include "llvm/Bitcode/BitcodeReader.h"
24 #include "llvm/Bitcode/BitcodeWriter.h"
25 #include "llvm/Bitcode/BitcodeWriterPass.h"
26 #include "llvm/Config/llvm-config.h"
27 #include "llvm/IR/DebugInfo.h"
28 #include "llvm/IR/DiagnosticPrinter.h"
29 #include "llvm/IR/LLVMContext.h"
30 #include "llvm/IR/LegacyPassManager.h"
31 #include "llvm/IR/Mangler.h"
32 #include "llvm/IR/PassTimingInfo.h"
33 #include "llvm/IR/RemarkStreamer.h"
34 #include "llvm/IR/Verifier.h"
35 #include "llvm/IRReader/IRReader.h"
36 #include "llvm/LTO/LTO.h"
37 #include "llvm/LTO/SummaryBasedOptimizations.h"
38 #include "llvm/MC/SubtargetFeature.h"
39 #include "llvm/Object/IRObjectFile.h"
40 #include "llvm/Support/CachePruning.h"
41 #include "llvm/Support/Debug.h"
42 #include "llvm/Support/Error.h"
43 #include "llvm/Support/FileUtilities.h"
44 #include "llvm/Support/Path.h"
45 #include "llvm/Support/SHA1.h"
46 #include "llvm/Support/SmallVectorMemoryBuffer.h"
47 #include "llvm/Support/TargetRegistry.h"
48 #include "llvm/Support/ThreadPool.h"
49 #include "llvm/Support/Threading.h"
50 #include "llvm/Support/ToolOutputFile.h"
51 #include "llvm/Support/VCSRevision.h"
52 #include "llvm/Target/TargetMachine.h"
53 #include "llvm/Transforms/IPO.h"
54 #include "llvm/Transforms/IPO/FunctionImport.h"
55 #include "llvm/Transforms/IPO/Internalize.h"
56 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
57 #include "llvm/Transforms/IPO/WholeProgramDevirt.h"
58 #include "llvm/Transforms/ObjCARC.h"
59 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
63 #if !defined(_MSC_VER) && !defined(__MINGW32__)
71 #define DEBUG_TYPE "thinlto"
74 // Flags -discard-value-names, defined in LTOCodeGenerator.cpp
75 extern cl::opt<bool> LTODiscardValueNames;
76 extern cl::opt<std::string> RemarksFilename;
77 extern cl::opt<std::string> RemarksPasses;
78 extern cl::opt<bool> RemarksWithHotness;
79 extern cl::opt<std::string> RemarksFormat;
85 ThreadCount("threads", cl::init(llvm::heavyweight_hardware_concurrency()));
87 // Simple helper to save temporary files for debug.
88 static void saveTempBitcode(const Module &TheModule, StringRef TempDir,
89 unsigned count, StringRef Suffix) {
92 // User asked to save temps, let dump the bitcode file after import.
93 std::string SaveTempPath = (TempDir + llvm::Twine(count) + Suffix).str();
95 raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
97 report_fatal_error(Twine("Failed to open ") + SaveTempPath +
98 " to save optimized bitcode\n");
99 WriteBitcodeToFile(TheModule, OS, /* ShouldPreserveUseListOrder */ true);
102 static const GlobalValueSummary *
103 getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
104 // If there is any strong definition anywhere, get it.
105 auto StrongDefForLinker = llvm::find_if(
106 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
107 auto Linkage = Summary->linkage();
108 return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
109 !GlobalValue::isWeakForLinker(Linkage);
111 if (StrongDefForLinker != GVSummaryList.end())
112 return StrongDefForLinker->get();
113 // Get the first *linker visible* definition for this global in the summary
115 auto FirstDefForLinker = llvm::find_if(
116 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
117 auto Linkage = Summary->linkage();
118 return !GlobalValue::isAvailableExternallyLinkage(Linkage);
120 // Extern templates can be emitted as available_externally.
121 if (FirstDefForLinker == GVSummaryList.end())
123 return FirstDefForLinker->get();
126 // Populate map of GUID to the prevailing copy for any multiply defined
127 // symbols. Currently assume first copy is prevailing, or any strong
128 // definition. Can be refined with Linker information in the future.
129 static void computePrevailingCopies(
130 const ModuleSummaryIndex &Index,
131 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy) {
132 auto HasMultipleCopies = [&](const GlobalValueSummaryList &GVSummaryList) {
133 return GVSummaryList.size() > 1;
136 for (auto &I : Index) {
137 if (HasMultipleCopies(I.second.SummaryList))
138 PrevailingCopy[I.first] =
139 getFirstDefinitionForLinker(I.second.SummaryList);
143 static StringMap<lto::InputFile *>
144 generateModuleMap(std::vector<std::unique_ptr<lto::InputFile>> &Modules) {
145 StringMap<lto::InputFile *> ModuleMap;
146 for (auto &M : Modules) {
147 assert(ModuleMap.find(M->getName()) == ModuleMap.end() &&
148 "Expect unique Buffer Identifier");
149 ModuleMap[M->getName()] = M.get();
154 static void promoteModule(Module &TheModule, const ModuleSummaryIndex &Index) {
155 if (renameModuleForThinLTO(TheModule, Index))
156 report_fatal_error("renameModuleForThinLTO failed");
160 class ThinLTODiagnosticInfo : public DiagnosticInfo {
163 ThinLTODiagnosticInfo(const Twine &DiagMsg,
164 DiagnosticSeverity Severity = DS_Error)
165 : DiagnosticInfo(DK_Linker, Severity), Msg(DiagMsg) {}
166 void print(DiagnosticPrinter &DP) const override { DP << Msg; }
170 /// Verify the module and strip broken debug info.
171 static void verifyLoadedModule(Module &TheModule) {
172 bool BrokenDebugInfo = false;
173 if (verifyModule(TheModule, &dbgs(), &BrokenDebugInfo))
174 report_fatal_error("Broken module found, compilation aborted!");
175 if (BrokenDebugInfo) {
176 TheModule.getContext().diagnose(ThinLTODiagnosticInfo(
177 "Invalid debug info found, debug info will be stripped", DS_Warning));
178 StripDebugInfo(TheModule);
182 static std::unique_ptr<Module> loadModuleFromInput(lto::InputFile *Input,
183 LLVMContext &Context,
186 auto &Mod = Input->getSingleBitcodeModule();
188 Expected<std::unique_ptr<Module>> ModuleOrErr =
189 Lazy ? Mod.getLazyModule(Context,
190 /* ShouldLazyLoadMetadata */ true, IsImporting)
191 : Mod.parseModule(Context);
193 handleAllErrors(ModuleOrErr.takeError(), [&](ErrorInfoBase &EIB) {
194 SMDiagnostic Err = SMDiagnostic(Mod.getModuleIdentifier(),
195 SourceMgr::DK_Error, EIB.message());
196 Err.print("ThinLTO", errs());
198 report_fatal_error("Can't load module, abort.");
201 verifyLoadedModule(*ModuleOrErr.get());
202 return std::move(*ModuleOrErr);
206 crossImportIntoModule(Module &TheModule, const ModuleSummaryIndex &Index,
207 StringMap<lto::InputFile*> &ModuleMap,
208 const FunctionImporter::ImportMapTy &ImportList) {
209 auto Loader = [&](StringRef Identifier) {
210 auto &Input = ModuleMap[Identifier];
211 return loadModuleFromInput(Input, TheModule.getContext(),
212 /*Lazy=*/true, /*IsImporting*/ true);
215 FunctionImporter Importer(Index, Loader);
216 Expected<bool> Result = Importer.importFunctions(TheModule, ImportList);
218 handleAllErrors(Result.takeError(), [&](ErrorInfoBase &EIB) {
219 SMDiagnostic Err = SMDiagnostic(TheModule.getModuleIdentifier(),
220 SourceMgr::DK_Error, EIB.message());
221 Err.print("ThinLTO", errs());
223 report_fatal_error("importFunctions failed");
225 // Verify again after cross-importing.
226 verifyLoadedModule(TheModule);
229 static void optimizeModule(Module &TheModule, TargetMachine &TM,
230 unsigned OptLevel, bool Freestanding,
231 ModuleSummaryIndex *Index) {
232 // Populate the PassManager
233 PassManagerBuilder PMB;
234 PMB.LibraryInfo = new TargetLibraryInfoImpl(TM.getTargetTriple());
236 PMB.LibraryInfo->disableAllFunctions();
237 PMB.Inliner = createFunctionInliningPass();
238 // FIXME: should get it from the bitcode?
239 PMB.OptLevel = OptLevel;
240 PMB.LoopVectorize = true;
241 PMB.SLPVectorize = true;
242 // Already did this in verifyLoadedModule().
243 PMB.VerifyInput = false;
244 PMB.VerifyOutput = false;
245 PMB.ImportSummary = Index;
247 legacy::PassManager PM;
249 // Add the TTI (required to inform the vectorizer about register size for
251 PM.add(createTargetTransformInfoWrapperPass(TM.getTargetIRAnalysis()));
254 PMB.populateThinLTOPassManager(PM);
260 addUsedSymbolToPreservedGUID(const lto::InputFile &File,
261 DenseSet<GlobalValue::GUID> &PreservedGUID) {
262 for (const auto &Sym : File.symbols()) {
264 PreservedGUID.insert(GlobalValue::getGUID(Sym.getIRName()));
268 // Convert the PreservedSymbols map from "Name" based to "GUID" based.
269 static DenseSet<GlobalValue::GUID>
270 computeGUIDPreservedSymbols(const StringSet<> &PreservedSymbols,
271 const Triple &TheTriple) {
272 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols(PreservedSymbols.size());
273 for (auto &Entry : PreservedSymbols) {
274 StringRef Name = Entry.first();
275 if (TheTriple.isOSBinFormatMachO() && Name.size() > 0 && Name[0] == '_')
276 Name = Name.drop_front();
277 GUIDPreservedSymbols.insert(GlobalValue::getGUID(Name));
279 return GUIDPreservedSymbols;
282 std::unique_ptr<MemoryBuffer> codegenModule(Module &TheModule,
284 SmallVector<char, 128> OutputBuffer;
288 raw_svector_ostream OS(OutputBuffer);
289 legacy::PassManager PM;
291 // If the bitcode files contain ARC code and were compiled with optimization,
292 // the ObjCARCContractPass must be run, so do it unconditionally here.
293 PM.add(createObjCARCContractPass());
295 // Setup the codegen now.
296 if (TM.addPassesToEmitFile(PM, OS, nullptr, CGFT_ObjectFile,
297 /* DisableVerify */ true))
298 report_fatal_error("Failed to setup codegen");
300 // Run codegen now. resulting binary is in OutputBuffer.
303 return std::make_unique<SmallVectorMemoryBuffer>(std::move(OutputBuffer));
306 /// Manage caching for a single Module.
307 class ModuleCacheEntry {
308 SmallString<128> EntryPath;
311 // Create a cache entry. This compute a unique hash for the Module considering
312 // the current list of export/import, and offer an interface to query to
313 // access the content in the cache.
315 StringRef CachePath, const ModuleSummaryIndex &Index, StringRef ModuleID,
316 const FunctionImporter::ImportMapTy &ImportList,
317 const FunctionImporter::ExportSetTy &ExportList,
318 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
319 const GVSummaryMapTy &DefinedGVSummaries, unsigned OptLevel,
320 bool Freestanding, const TargetMachineBuilder &TMBuilder) {
321 if (CachePath.empty())
324 if (!Index.modulePaths().count(ModuleID))
325 // The module does not have an entry, it can't have a hash at all
328 if (all_of(Index.getModuleHash(ModuleID),
329 [](uint32_t V) { return V == 0; }))
330 // No hash entry, no caching!
333 llvm::lto::Config Conf;
334 Conf.OptLevel = OptLevel;
335 Conf.Options = TMBuilder.Options;
336 Conf.CPU = TMBuilder.MCpu;
337 Conf.MAttrs.push_back(TMBuilder.MAttr);
338 Conf.RelocModel = TMBuilder.RelocModel;
339 Conf.CGOptLevel = TMBuilder.CGOptLevel;
340 Conf.Freestanding = Freestanding;
342 computeLTOCacheKey(Key, Conf, Index, ModuleID, ImportList, ExportList,
343 ResolvedODR, DefinedGVSummaries);
345 // This choice of file name allows the cache to be pruned (see pruneCache()
346 // in include/llvm/Support/CachePruning.h).
347 sys::path::append(EntryPath, CachePath, "llvmcache-" + Key);
350 // Access the path to this entry in the cache.
351 StringRef getEntryPath() { return EntryPath; }
353 // Try loading the buffer for this cache entry.
354 ErrorOr<std::unique_ptr<MemoryBuffer>> tryLoadingBuffer() {
355 if (EntryPath.empty())
356 return std::error_code();
357 SmallString<64> ResultPath;
358 Expected<sys::fs::file_t> FDOrErr = sys::fs::openNativeFileForRead(
359 Twine(EntryPath), sys::fs::OF_UpdateAtime, &ResultPath);
361 return errorToErrorCode(FDOrErr.takeError());
362 ErrorOr<std::unique_ptr<MemoryBuffer>> MBOrErr = MemoryBuffer::getOpenFile(
363 *FDOrErr, EntryPath, /*FileSize=*/-1, /*RequiresNullTerminator=*/false);
364 sys::fs::closeFile(*FDOrErr);
368 // Cache the Produced object file
369 void write(const MemoryBuffer &OutputBuffer) {
370 if (EntryPath.empty())
373 // Write to a temporary to avoid race condition
374 SmallString<128> TempFilename;
375 SmallString<128> CachePath(EntryPath);
376 llvm::sys::path::remove_filename(CachePath);
377 sys::path::append(TempFilename, CachePath, "Thin-%%%%%%.tmp.o");
379 if (auto Err = handleErrors(
380 llvm::writeFileAtomically(TempFilename, EntryPath,
381 OutputBuffer.getBuffer()),
382 [](const llvm::AtomicFileWriteError &E) {
383 std::string ErrorMsgBuffer;
384 llvm::raw_string_ostream S(ErrorMsgBuffer);
388 llvm::atomic_write_error::failed_to_create_uniq_file) {
389 errs() << "Error: " << ErrorMsgBuffer << "\n";
390 report_fatal_error("ThinLTO: Can't get a temporary file");
394 consumeError(std::move(Err));
399 static std::unique_ptr<MemoryBuffer>
400 ProcessThinLTOModule(Module &TheModule, ModuleSummaryIndex &Index,
401 StringMap<lto::InputFile *> &ModuleMap, TargetMachine &TM,
402 const FunctionImporter::ImportMapTy &ImportList,
403 const FunctionImporter::ExportSetTy &ExportList,
404 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
405 const GVSummaryMapTy &DefinedGlobals,
406 const ThinLTOCodeGenerator::CachingOptions &CacheOptions,
407 bool DisableCodeGen, StringRef SaveTempsDir,
408 bool Freestanding, unsigned OptLevel, unsigned count) {
410 // "Benchmark"-like optimization: single-source case
411 bool SingleModule = (ModuleMap.size() == 1);
414 promoteModule(TheModule, Index);
416 // Apply summary-based prevailing-symbol resolution decisions.
417 thinLTOResolvePrevailingInModule(TheModule, DefinedGlobals);
419 // Save temps: after promotion.
420 saveTempBitcode(TheModule, SaveTempsDir, count, ".1.promoted.bc");
423 // Be friendly and don't nuke totally the module when the client didn't
424 // supply anything to preserve.
425 if (!ExportList.empty() || !GUIDPreservedSymbols.empty()) {
426 // Apply summary-based internalization decisions.
427 thinLTOInternalizeModule(TheModule, DefinedGlobals);
430 // Save internalized bitcode
431 saveTempBitcode(TheModule, SaveTempsDir, count, ".2.internalized.bc");
434 crossImportIntoModule(TheModule, Index, ModuleMap, ImportList);
436 // Save temps: after cross-module import.
437 saveTempBitcode(TheModule, SaveTempsDir, count, ".3.imported.bc");
440 optimizeModule(TheModule, TM, OptLevel, Freestanding, &Index);
442 saveTempBitcode(TheModule, SaveTempsDir, count, ".4.opt.bc");
444 if (DisableCodeGen) {
445 // Configured to stop before CodeGen, serialize the bitcode and return.
446 SmallVector<char, 128> OutputBuffer;
448 raw_svector_ostream OS(OutputBuffer);
449 ProfileSummaryInfo PSI(TheModule);
450 auto Index = buildModuleSummaryIndex(TheModule, nullptr, &PSI);
451 WriteBitcodeToFile(TheModule, OS, true, &Index);
453 return std::make_unique<SmallVectorMemoryBuffer>(std::move(OutputBuffer));
456 return codegenModule(TheModule, TM);
459 /// Resolve prevailing symbols. Record resolutions in the \p ResolvedODR map
460 /// for caching, and in the \p Index for application during the ThinLTO
461 /// backends. This is needed for correctness for exported symbols (ensure
462 /// at least one copy kept) and a compile-time optimization (to drop duplicate
463 /// copies when possible).
464 static void resolvePrevailingInIndex(
465 ModuleSummaryIndex &Index,
466 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>>
468 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
469 const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
472 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
473 const auto &Prevailing = PrevailingCopy.find(GUID);
474 // Not in map means that there was only one copy, which must be prevailing.
475 if (Prevailing == PrevailingCopy.end())
477 return Prevailing->second == S;
480 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
481 GlobalValue::GUID GUID,
482 GlobalValue::LinkageTypes NewLinkage) {
483 ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
486 thinLTOResolvePrevailingInIndex(Index, isPrevailing, recordNewLinkage,
487 GUIDPreservedSymbols);
490 // Initialize the TargetMachine builder for a given Triple
491 static void initTMBuilder(TargetMachineBuilder &TMBuilder,
492 const Triple &TheTriple) {
493 // Set a default CPU for Darwin triples (copied from LTOCodeGenerator).
494 // FIXME this looks pretty terrible...
495 if (TMBuilder.MCpu.empty() && TheTriple.isOSDarwin()) {
496 if (TheTriple.getArch() == llvm::Triple::x86_64)
497 TMBuilder.MCpu = "core2";
498 else if (TheTriple.getArch() == llvm::Triple::x86)
499 TMBuilder.MCpu = "yonah";
500 else if (TheTriple.getArch() == llvm::Triple::aarch64 ||
501 TheTriple.getArch() == llvm::Triple::aarch64_32)
502 TMBuilder.MCpu = "cyclone";
504 TMBuilder.TheTriple = std::move(TheTriple);
507 } // end anonymous namespace
509 void ThinLTOCodeGenerator::addModule(StringRef Identifier, StringRef Data) {
510 MemoryBufferRef Buffer(Data, Identifier);
512 auto InputOrError = lto::InputFile::create(Buffer);
514 report_fatal_error("ThinLTO cannot create input file: " +
515 toString(InputOrError.takeError()));
517 auto TripleStr = (*InputOrError)->getTargetTriple();
518 Triple TheTriple(TripleStr);
521 initTMBuilder(TMBuilder, Triple(TheTriple));
522 else if (TMBuilder.TheTriple != TheTriple) {
523 if (!TMBuilder.TheTriple.isCompatibleWith(TheTriple))
524 report_fatal_error("ThinLTO modules with incompatible triples not "
526 initTMBuilder(TMBuilder, Triple(TMBuilder.TheTriple.merge(TheTriple)));
529 Modules.emplace_back(std::move(*InputOrError));
532 void ThinLTOCodeGenerator::preserveSymbol(StringRef Name) {
533 PreservedSymbols.insert(Name);
536 void ThinLTOCodeGenerator::crossReferenceSymbol(StringRef Name) {
537 // FIXME: At the moment, we don't take advantage of this extra information,
538 // we're conservatively considering cross-references as preserved.
539 // CrossReferencedSymbols.insert(Name);
540 PreservedSymbols.insert(Name);
543 // TargetMachine factory
544 std::unique_ptr<TargetMachine> TargetMachineBuilder::create() const {
546 const Target *TheTarget =
547 TargetRegistry::lookupTarget(TheTriple.str(), ErrMsg);
549 report_fatal_error("Can't load target for this Triple: " + ErrMsg);
552 // Use MAttr as the default set of features.
553 SubtargetFeatures Features(MAttr);
554 Features.getDefaultSubtargetFeatures(TheTriple);
555 std::string FeatureStr = Features.getString();
557 return std::unique_ptr<TargetMachine>(
558 TheTarget->createTargetMachine(TheTriple.str(), MCpu, FeatureStr, Options,
559 RelocModel, None, CGOptLevel));
563 * Produce the combined summary index from all the bitcode files:
566 std::unique_ptr<ModuleSummaryIndex> ThinLTOCodeGenerator::linkCombinedIndex() {
567 std::unique_ptr<ModuleSummaryIndex> CombinedIndex =
568 std::make_unique<ModuleSummaryIndex>(/*HaveGVs=*/false);
569 uint64_t NextModuleId = 0;
570 for (auto &Mod : Modules) {
571 auto &M = Mod->getSingleBitcodeModule();
573 M.readSummary(*CombinedIndex, Mod->getName(), NextModuleId++)) {
575 logAllUnhandledErrors(
576 std::move(Err), errs(),
577 "error: can't create module summary index for buffer: ");
581 return CombinedIndex;
586 const StringMap<FunctionImporter::ExportSetTy> &ExportLists;
587 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols;
589 IsExported(const StringMap<FunctionImporter::ExportSetTy> &ExportLists,
590 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols)
591 : ExportLists(ExportLists), GUIDPreservedSymbols(GUIDPreservedSymbols) {}
593 bool operator()(StringRef ModuleIdentifier, ValueInfo VI) const {
594 const auto &ExportList = ExportLists.find(ModuleIdentifier);
595 return (ExportList != ExportLists.end() && ExportList->second.count(VI)) ||
596 GUIDPreservedSymbols.count(VI.getGUID());
600 struct IsPrevailing {
601 const DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy;
602 IsPrevailing(const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
604 : PrevailingCopy(PrevailingCopy) {}
606 bool operator()(GlobalValue::GUID GUID, const GlobalValueSummary *S) const {
607 const auto &Prevailing = PrevailingCopy.find(GUID);
608 // Not in map means that there was only one copy, which must be prevailing.
609 if (Prevailing == PrevailingCopy.end())
611 return Prevailing->second == S;
616 static void computeDeadSymbolsInIndex(
617 ModuleSummaryIndex &Index,
618 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
619 // We have no symbols resolution available. And can't do any better now in the
620 // case where the prevailing symbol is in a native object. It can be refined
621 // with linker information in the future.
622 auto isPrevailing = [&](GlobalValue::GUID G) {
623 return PrevailingType::Unknown;
625 computeDeadSymbolsWithConstProp(Index, GUIDPreservedSymbols, isPrevailing,
626 /* ImportEnabled = */ true);
630 * Perform promotion and renaming of exported internal functions.
631 * Index is updated to reflect linkage changes from weak resolution.
633 void ThinLTOCodeGenerator::promote(Module &TheModule, ModuleSummaryIndex &Index,
634 const lto::InputFile &File) {
635 auto ModuleCount = Index.modulePaths().size();
636 auto ModuleIdentifier = TheModule.getModuleIdentifier();
638 // Collect for each module the list of function it defines (GUID -> Summary).
639 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
640 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
642 // Convert the preserved symbols set from string to GUID
643 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
644 PreservedSymbols, Triple(TheModule.getTargetTriple()));
646 // Add used symbol to the preserved symbols.
647 addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
649 // Compute "dead" symbols, we don't want to import/export these!
650 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
652 // Generate import/export list
653 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
654 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
655 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
658 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
659 computePrevailingCopies(Index, PrevailingCopy);
661 // Resolve prevailing symbols
662 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
663 resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
666 thinLTOResolvePrevailingInModule(
667 TheModule, ModuleToDefinedGVSummaries[ModuleIdentifier]);
669 // Promote the exported values in the index, so that they are promoted
671 thinLTOInternalizeAndPromoteInIndex(
672 Index, IsExported(ExportLists, GUIDPreservedSymbols),
673 IsPrevailing(PrevailingCopy));
675 promoteModule(TheModule, Index);
679 * Perform cross-module importing for the module identified by ModuleIdentifier.
681 void ThinLTOCodeGenerator::crossModuleImport(Module &TheModule,
682 ModuleSummaryIndex &Index,
683 const lto::InputFile &File) {
684 auto ModuleMap = generateModuleMap(Modules);
685 auto ModuleCount = Index.modulePaths().size();
687 // Collect for each module the list of function it defines (GUID -> Summary).
688 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
689 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
691 // Convert the preserved symbols set from string to GUID
692 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
693 PreservedSymbols, Triple(TheModule.getTargetTriple()));
695 addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
697 // Compute "dead" symbols, we don't want to import/export these!
698 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
700 // Generate import/export list
701 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
702 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
703 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
705 auto &ImportList = ImportLists[TheModule.getModuleIdentifier()];
707 crossImportIntoModule(TheModule, Index, ModuleMap, ImportList);
711 * Compute the list of summaries needed for importing into module.
713 void ThinLTOCodeGenerator::gatherImportedSummariesForModule(
714 Module &TheModule, ModuleSummaryIndex &Index,
715 std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex,
716 const lto::InputFile &File) {
717 auto ModuleCount = Index.modulePaths().size();
718 auto ModuleIdentifier = TheModule.getModuleIdentifier();
720 // Collect for each module the list of function it defines (GUID -> Summary).
721 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
722 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
724 // Convert the preserved symbols set from string to GUID
725 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
726 PreservedSymbols, Triple(TheModule.getTargetTriple()));
728 addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
730 // Compute "dead" symbols, we don't want to import/export these!
731 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
733 // Generate import/export list
734 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
735 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
736 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
739 llvm::gatherImportedSummariesForModule(
740 ModuleIdentifier, ModuleToDefinedGVSummaries,
741 ImportLists[ModuleIdentifier], ModuleToSummariesForIndex);
745 * Emit the list of files needed for importing into module.
747 void ThinLTOCodeGenerator::emitImports(Module &TheModule, StringRef OutputName,
748 ModuleSummaryIndex &Index,
749 const lto::InputFile &File) {
750 auto ModuleCount = Index.modulePaths().size();
751 auto ModuleIdentifier = TheModule.getModuleIdentifier();
753 // Collect for each module the list of function it defines (GUID -> Summary).
754 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
755 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
757 // Convert the preserved symbols set from string to GUID
758 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
759 PreservedSymbols, Triple(TheModule.getTargetTriple()));
761 addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
763 // Compute "dead" symbols, we don't want to import/export these!
764 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
766 // Generate import/export list
767 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
768 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
769 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
772 std::map<std::string, GVSummaryMapTy> ModuleToSummariesForIndex;
773 llvm::gatherImportedSummariesForModule(
774 ModuleIdentifier, ModuleToDefinedGVSummaries,
775 ImportLists[ModuleIdentifier], ModuleToSummariesForIndex);
778 if ((EC = EmitImportsFiles(ModuleIdentifier, OutputName,
779 ModuleToSummariesForIndex)))
780 report_fatal_error(Twine("Failed to open ") + OutputName +
781 " to save imports lists\n");
785 * Perform internalization. Runs promote and internalization together.
786 * Index is updated to reflect linkage changes.
788 void ThinLTOCodeGenerator::internalize(Module &TheModule,
789 ModuleSummaryIndex &Index,
790 const lto::InputFile &File) {
791 initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
792 auto ModuleCount = Index.modulePaths().size();
793 auto ModuleIdentifier = TheModule.getModuleIdentifier();
795 // Convert the preserved symbols set from string to GUID
796 auto GUIDPreservedSymbols =
797 computeGUIDPreservedSymbols(PreservedSymbols, TMBuilder.TheTriple);
799 addUsedSymbolToPreservedGUID(File, GUIDPreservedSymbols);
801 // Collect for each module the list of function it defines (GUID -> Summary).
802 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
803 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
805 // Compute "dead" symbols, we don't want to import/export these!
806 computeDeadSymbolsInIndex(Index, GUIDPreservedSymbols);
808 // Generate import/export list
809 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
810 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
811 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
813 auto &ExportList = ExportLists[ModuleIdentifier];
815 // Be friendly and don't nuke totally the module when the client didn't
816 // supply anything to preserve.
817 if (ExportList.empty() && GUIDPreservedSymbols.empty())
820 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
821 computePrevailingCopies(Index, PrevailingCopy);
823 // Resolve prevailing symbols
824 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
825 resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
828 // Promote the exported values in the index, so that they are promoted
830 thinLTOInternalizeAndPromoteInIndex(
831 Index, IsExported(ExportLists, GUIDPreservedSymbols),
832 IsPrevailing(PrevailingCopy));
834 promoteModule(TheModule, Index);
837 thinLTOResolvePrevailingInModule(
838 TheModule, ModuleToDefinedGVSummaries[ModuleIdentifier]);
840 thinLTOInternalizeModule(TheModule,
841 ModuleToDefinedGVSummaries[ModuleIdentifier]);
845 * Perform post-importing ThinLTO optimizations.
847 void ThinLTOCodeGenerator::optimize(Module &TheModule) {
848 initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
851 optimizeModule(TheModule, *TMBuilder.create(), OptLevel, Freestanding,
855 /// Write out the generated object file, either from CacheEntryPath or from
856 /// OutputBuffer, preferring hard-link when possible.
857 /// Returns the path to the generated file in SavedObjectsDirectoryPath.
859 ThinLTOCodeGenerator::writeGeneratedObject(int count, StringRef CacheEntryPath,
860 const MemoryBuffer &OutputBuffer) {
861 auto ArchName = TMBuilder.TheTriple.getArchName();
862 SmallString<128> OutputPath(SavedObjectsDirectoryPath);
863 llvm::sys::path::append(OutputPath,
864 Twine(count) + "." + ArchName + ".thinlto.o");
865 OutputPath.c_str(); // Ensure the string is null terminated.
866 if (sys::fs::exists(OutputPath))
867 sys::fs::remove(OutputPath);
869 // We don't return a memory buffer to the linker, just a list of files.
870 if (!CacheEntryPath.empty()) {
871 // Cache is enabled, hard-link the entry (or copy if hard-link fails).
872 auto Err = sys::fs::create_hard_link(CacheEntryPath, OutputPath);
874 return OutputPath.str();
875 // Hard linking failed, try to copy.
876 Err = sys::fs::copy_file(CacheEntryPath, OutputPath);
878 return OutputPath.str();
879 // Copy failed (could be because the CacheEntry was removed from the cache
880 // in the meantime by another process), fall back and try to write down the
881 // buffer to the output.
882 errs() << "error: can't link or copy from cached entry '" << CacheEntryPath
883 << "' to '" << OutputPath << "'\n";
885 // No cache entry, just write out the buffer.
887 raw_fd_ostream OS(OutputPath, Err, sys::fs::OF_None);
889 report_fatal_error("Can't open output '" + OutputPath + "'\n");
890 OS << OutputBuffer.getBuffer();
891 return OutputPath.str();
894 // Main entry point for the ThinLTO processing
895 void ThinLTOCodeGenerator::run() {
896 // Prepare the resulting object vector
897 assert(ProducedBinaries.empty() && "The generator should not be reused");
898 if (SavedObjectsDirectoryPath.empty())
899 ProducedBinaries.resize(Modules.size());
901 sys::fs::create_directories(SavedObjectsDirectoryPath);
903 sys::fs::is_directory(SavedObjectsDirectoryPath, IsDir);
905 report_fatal_error("Unexistent dir: '" + SavedObjectsDirectoryPath + "'");
906 ProducedBinaryFiles.resize(Modules.size());
910 // Perform only parallel codegen and return.
913 for (auto &Mod : Modules) {
914 Pool.async([&](int count) {
916 Context.setDiscardValueNames(LTODiscardValueNames);
919 auto TheModule = loadModuleFromInput(Mod.get(), Context, false,
920 /*IsImporting*/ false);
923 auto OutputBuffer = codegenModule(*TheModule, *TMBuilder.create());
924 if (SavedObjectsDirectoryPath.empty())
925 ProducedBinaries[count] = std::move(OutputBuffer);
927 ProducedBinaryFiles[count] =
928 writeGeneratedObject(count, "", *OutputBuffer);
935 // Sequential linking phase
936 auto Index = linkCombinedIndex();
938 // Save temps: index.
939 if (!SaveTempsDir.empty()) {
940 auto SaveTempPath = SaveTempsDir + "index.bc";
942 raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
944 report_fatal_error(Twine("Failed to open ") + SaveTempPath +
945 " to save optimized bitcode\n");
946 WriteIndexToFile(*Index, OS);
950 // Prepare the module map.
951 auto ModuleMap = generateModuleMap(Modules);
952 auto ModuleCount = Modules.size();
954 // Collect for each module the list of function it defines (GUID -> Summary).
955 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
956 Index->collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
958 // Convert the preserved symbols set from string to GUID, this is needed for
959 // computing the caching hash and the internalization.
960 auto GUIDPreservedSymbols =
961 computeGUIDPreservedSymbols(PreservedSymbols, TMBuilder.TheTriple);
963 // Add used symbol from inputs to the preserved symbols.
964 for (const auto &M : Modules)
965 addUsedSymbolToPreservedGUID(*M, GUIDPreservedSymbols);
967 // Compute "dead" symbols, we don't want to import/export these!
968 computeDeadSymbolsInIndex(*Index, GUIDPreservedSymbols);
970 // Synthesize entry counts for functions in the combined index.
971 computeSyntheticCounts(*Index);
973 // Perform index-based WPD. This will return immediately if there are
974 // no index entries in the typeIdMetadata map (e.g. if we are instead
975 // performing IR-based WPD in hybrid regular/thin LTO mode).
976 std::map<ValueInfo, std::vector<VTableSlotSummary>> LocalWPDTargetsMap;
977 std::set<GlobalValue::GUID> ExportedGUIDs;
978 runWholeProgramDevirtOnIndex(*Index, ExportedGUIDs, LocalWPDTargetsMap);
979 for (auto GUID : ExportedGUIDs)
980 GUIDPreservedSymbols.insert(GUID);
982 // Collect the import/export lists for all modules from the call-graph in the
984 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
985 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
986 ComputeCrossModuleImport(*Index, ModuleToDefinedGVSummaries, ImportLists,
989 // We use a std::map here to be able to have a defined ordering when
990 // producing a hash for the cache entry.
991 // FIXME: we should be able to compute the caching hash for the entry based
992 // on the index, and nuke this map.
993 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
995 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
996 computePrevailingCopies(*Index, PrevailingCopy);
998 // Resolve prevailing symbols, this has to be computed early because it
999 // impacts the caching.
1000 resolvePrevailingInIndex(*Index, ResolvedODR, GUIDPreservedSymbols,
1003 // Use global summary-based analysis to identify symbols that can be
1004 // internalized (because they aren't exported or preserved as per callback).
1005 // Changes are made in the index, consumed in the ThinLTO backends.
1006 updateIndexWPDForExports(*Index,
1007 IsExported(ExportLists, GUIDPreservedSymbols),
1008 LocalWPDTargetsMap);
1009 thinLTOInternalizeAndPromoteInIndex(
1010 *Index, IsExported(ExportLists, GUIDPreservedSymbols),
1011 IsPrevailing(PrevailingCopy));
1013 // Make sure that every module has an entry in the ExportLists, ImportList,
1014 // GVSummary and ResolvedODR maps to enable threaded access to these maps
1016 for (auto &Module : Modules) {
1017 auto ModuleIdentifier = Module->getName();
1018 ExportLists[ModuleIdentifier];
1019 ImportLists[ModuleIdentifier];
1020 ResolvedODR[ModuleIdentifier];
1021 ModuleToDefinedGVSummaries[ModuleIdentifier];
1024 // Compute the ordering we will process the inputs: the rough heuristic here
1025 // is to sort them per size so that the largest module get schedule as soon as
1026 // possible. This is purely a compile-time optimization.
1027 std::vector<int> ModulesOrdering;
1028 ModulesOrdering.resize(Modules.size());
1029 std::iota(ModulesOrdering.begin(), ModulesOrdering.end(), 0);
1030 llvm::sort(ModulesOrdering, [&](int LeftIndex, int RightIndex) {
1032 Modules[LeftIndex]->getSingleBitcodeModule().getBuffer().size();
1034 Modules[RightIndex]->getSingleBitcodeModule().getBuffer().size();
1035 return LSize > RSize;
1038 // Parallel optimizer + codegen
1040 ThreadPool Pool(ThreadCount);
1041 for (auto IndexCount : ModulesOrdering) {
1042 auto &Mod = Modules[IndexCount];
1043 Pool.async([&](int count) {
1044 auto ModuleIdentifier = Mod->getName();
1045 auto &ExportList = ExportLists[ModuleIdentifier];
1047 auto &DefinedGVSummaries = ModuleToDefinedGVSummaries[ModuleIdentifier];
1049 // The module may be cached, this helps handling it.
1050 ModuleCacheEntry CacheEntry(CacheOptions.Path, *Index, ModuleIdentifier,
1051 ImportLists[ModuleIdentifier], ExportList,
1052 ResolvedODR[ModuleIdentifier],
1053 DefinedGVSummaries, OptLevel, Freestanding,
1055 auto CacheEntryPath = CacheEntry.getEntryPath();
1058 auto ErrOrBuffer = CacheEntry.tryLoadingBuffer();
1059 LLVM_DEBUG(dbgs() << "Cache " << (ErrOrBuffer ? "hit" : "miss")
1060 << " '" << CacheEntryPath << "' for buffer "
1061 << count << " " << ModuleIdentifier << "\n");
1065 if (SavedObjectsDirectoryPath.empty())
1066 ProducedBinaries[count] = std::move(ErrOrBuffer.get());
1068 ProducedBinaryFiles[count] = writeGeneratedObject(
1069 count, CacheEntryPath, *ErrOrBuffer.get());
1074 LLVMContext Context;
1075 Context.setDiscardValueNames(LTODiscardValueNames);
1076 Context.enableDebugTypeODRUniquing();
1077 auto DiagFileOrErr = lto::setupOptimizationRemarks(
1078 Context, RemarksFilename, RemarksPasses, RemarksFormat,
1079 RemarksWithHotness, count);
1080 if (!DiagFileOrErr) {
1081 errs() << "Error: " << toString(DiagFileOrErr.takeError()) << "\n";
1082 report_fatal_error("ThinLTO: Can't get an output file for the "
1087 auto TheModule = loadModuleFromInput(Mod.get(), Context, false,
1088 /*IsImporting*/ false);
1090 // Save temps: original file.
1091 saveTempBitcode(*TheModule, SaveTempsDir, count, ".0.original.bc");
1093 auto &ImportList = ImportLists[ModuleIdentifier];
1094 // Run the main process now, and generates a binary
1095 auto OutputBuffer = ProcessThinLTOModule(
1096 *TheModule, *Index, ModuleMap, *TMBuilder.create(), ImportList,
1097 ExportList, GUIDPreservedSymbols,
1098 ModuleToDefinedGVSummaries[ModuleIdentifier], CacheOptions,
1099 DisableCodeGen, SaveTempsDir, Freestanding, OptLevel, count);
1101 // Commit to the cache (if enabled)
1102 CacheEntry.write(*OutputBuffer);
1104 if (SavedObjectsDirectoryPath.empty()) {
1105 // We need to generated a memory buffer for the linker.
1106 if (!CacheEntryPath.empty()) {
1107 // When cache is enabled, reload from the cache if possible.
1108 // Releasing the buffer from the heap and reloading it from the
1109 // cache file with mmap helps us to lower memory pressure.
1110 // The freed memory can be used for the next input file.
1111 // The final binary link will read from the VFS cache (hopefully!)
1112 // or from disk (if the memory pressure was too high).
1113 auto ReloadedBufferOrErr = CacheEntry.tryLoadingBuffer();
1114 if (auto EC = ReloadedBufferOrErr.getError()) {
1115 // On error, keep the preexisting buffer and print a diagnostic.
1116 errs() << "error: can't reload cached file '" << CacheEntryPath
1117 << "': " << EC.message() << "\n";
1119 OutputBuffer = std::move(*ReloadedBufferOrErr);
1122 ProducedBinaries[count] = std::move(OutputBuffer);
1125 ProducedBinaryFiles[count] = writeGeneratedObject(
1126 count, CacheEntryPath, *OutputBuffer);
1131 pruneCache(CacheOptions.Path, CacheOptions.Policy);
1133 // If statistics were requested, print them out now.
1134 if (llvm::AreStatisticsEnabled())
1135 llvm::PrintStatistics();
1136 reportAndResetTimings();