1 //===-LTO.cpp - LLVM Link Time Optimizer ----------------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file implements functions and classes used to support LTO.
12 //===----------------------------------------------------------------------===//
14 #include "llvm/LTO/LTO.h"
15 #include "llvm/Analysis/TargetLibraryInfo.h"
16 #include "llvm/Analysis/TargetTransformInfo.h"
17 #include "llvm/Bitcode/BitcodeReader.h"
18 #include "llvm/Bitcode/BitcodeWriter.h"
19 #include "llvm/CodeGen/Analysis.h"
20 #include "llvm/IR/AutoUpgrade.h"
21 #include "llvm/IR/DiagnosticPrinter.h"
22 #include "llvm/IR/LegacyPassManager.h"
23 #include "llvm/LTO/LTOBackend.h"
24 #include "llvm/Linker/IRMover.h"
25 #include "llvm/Object/ModuleSummaryIndexObjectFile.h"
26 #include "llvm/Support/ManagedStatic.h"
27 #include "llvm/Support/MemoryBuffer.h"
28 #include "llvm/Support/Path.h"
29 #include "llvm/Support/SHA1.h"
30 #include "llvm/Support/SourceMgr.h"
31 #include "llvm/Support/TargetRegistry.h"
32 #include "llvm/Support/ThreadPool.h"
33 #include "llvm/Support/Threading.h"
34 #include "llvm/Support/raw_ostream.h"
35 #include "llvm/Target/TargetMachine.h"
36 #include "llvm/Target/TargetOptions.h"
37 #include "llvm/Transforms/IPO.h"
38 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
39 #include "llvm/Transforms/Utils/SplitModule.h"
45 using namespace object;
47 #define DEBUG_TYPE "lto"
49 // Returns a unique hash for the Module considering the current list of
50 // export/import and other global analysis results.
51 // The hash is produced in \p Key.
52 static void computeCacheKey(
53 SmallString<40> &Key, const Config &Conf, const ModuleSummaryIndex &Index,
54 StringRef ModuleID, const FunctionImporter::ImportMapTy &ImportList,
55 const FunctionImporter::ExportSetTy &ExportList,
56 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
57 const GVSummaryMapTy &DefinedGlobals) {
58 // Compute the unique hash for this entry.
59 // This is based on the current compiler version, the module itself, the
60 // export list, the hash for every single module in the import list, the
61 // list of ResolvedODR for the module, and the list of preserved symbols.
64 // Start with the compiler revision
65 Hasher.update(LLVM_VERSION_STRING);
66 #ifdef HAVE_LLVM_REVISION
67 Hasher.update(LLVM_REVISION);
70 // Include the parts of the LTO configuration that affect code generation.
71 auto AddString = [&](StringRef Str) {
73 Hasher.update(ArrayRef<uint8_t>{0});
75 auto AddUnsigned = [&](unsigned I) {
81 Hasher.update(ArrayRef<uint8_t>{Data, 4});
84 // FIXME: Hash more of Options. For now all clients initialize Options from
85 // command-line flags (which is unsupported in production), but may set
86 // RelaxELFRelocations. The clang driver can also pass FunctionSections,
87 // DataSections and DebuggerTuning via command line flags.
88 AddUnsigned(Conf.Options.RelaxELFRelocations);
89 AddUnsigned(Conf.Options.FunctionSections);
90 AddUnsigned(Conf.Options.DataSections);
91 AddUnsigned((unsigned)Conf.Options.DebuggerTuning);
92 for (auto &A : Conf.MAttrs)
94 AddUnsigned(Conf.RelocModel);
95 AddUnsigned(Conf.CodeModel);
96 AddUnsigned(Conf.CGOptLevel);
97 AddUnsigned(Conf.OptLevel);
98 AddString(Conf.OptPipeline);
99 AddString(Conf.AAPipeline);
100 AddString(Conf.OverrideTriple);
101 AddString(Conf.DefaultTriple);
103 // Include the hash for the current module
104 auto ModHash = Index.getModuleHash(ModuleID);
105 Hasher.update(ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));
106 for (auto F : ExportList)
107 // The export list can impact the internalization, be conservative here
108 Hasher.update(ArrayRef<uint8_t>((uint8_t *)&F, sizeof(F)));
110 // Include the hash for every module we import functions from
111 for (auto &Entry : ImportList) {
112 auto ModHash = Index.getModuleHash(Entry.first());
113 Hasher.update(ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));
116 // Include the hash for the resolved ODR.
117 for (auto &Entry : ResolvedODR) {
118 Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&Entry.first,
119 sizeof(GlobalValue::GUID)));
120 Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&Entry.second,
121 sizeof(GlobalValue::LinkageTypes)));
124 // Include the hash for the linkage type to reflect internalization and weak
126 for (auto &GS : DefinedGlobals) {
127 GlobalValue::LinkageTypes Linkage = GS.second->linkage();
129 ArrayRef<uint8_t>((const uint8_t *)&Linkage, sizeof(Linkage)));
132 if (!Conf.SampleProfile.empty()) {
133 auto FileOrErr = MemoryBuffer::getFile(Conf.SampleProfile);
135 Hasher.update(FileOrErr.get()->getBuffer());
138 Key = toHex(Hasher.result());
141 static void thinLTOResolveWeakForLinkerGUID(
142 GlobalValueSummaryList &GVSummaryList, GlobalValue::GUID GUID,
143 DenseSet<GlobalValueSummary *> &GlobalInvolvedWithAlias,
144 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
146 function_ref<void(StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes)>
148 for (auto &S : GVSummaryList) {
149 GlobalValue::LinkageTypes OriginalLinkage = S->linkage();
150 if (!GlobalValue::isWeakForLinker(OriginalLinkage))
152 // We need to emit only one of these. The prevailing module will keep it,
153 // but turned into a weak, while the others will drop it when possible.
154 // This is both a compile-time optimization and a correctness
155 // transformation. This is necessary for correctness when we have exported
156 // a reference - we need to convert the linkonce to weak to
157 // ensure a copy is kept to satisfy the exported reference.
158 // FIXME: We may want to split the compile time and correctness
159 // aspects into separate routines.
160 if (isPrevailing(GUID, S.get())) {
161 if (GlobalValue::isLinkOnceLinkage(OriginalLinkage))
162 S->setLinkage(GlobalValue::getWeakLinkage(
163 GlobalValue::isLinkOnceODRLinkage(OriginalLinkage)));
165 // Alias and aliasee can't be turned into available_externally.
166 else if (!isa<AliasSummary>(S.get()) &&
167 !GlobalInvolvedWithAlias.count(S.get()) &&
168 (GlobalValue::isLinkOnceODRLinkage(OriginalLinkage) ||
169 GlobalValue::isWeakODRLinkage(OriginalLinkage)))
170 S->setLinkage(GlobalValue::AvailableExternallyLinkage);
171 if (S->linkage() != OriginalLinkage)
172 recordNewLinkage(S->modulePath(), GUID, S->linkage());
176 // Resolve Weak and LinkOnce values in the \p Index.
178 // We'd like to drop these functions if they are no longer referenced in the
179 // current module. However there is a chance that another module is still
180 // referencing them because of the import. We make sure we always emit at least
182 void llvm::thinLTOResolveWeakForLinkerInIndex(
183 ModuleSummaryIndex &Index,
184 function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
186 function_ref<void(StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes)>
188 // We won't optimize the globals that are referenced by an alias for now
189 // Ideally we should turn the alias into a global and duplicate the definition
191 DenseSet<GlobalValueSummary *> GlobalInvolvedWithAlias;
192 for (auto &I : Index)
193 for (auto &S : I.second)
194 if (auto AS = dyn_cast<AliasSummary>(S.get()))
195 GlobalInvolvedWithAlias.insert(&AS->getAliasee());
197 for (auto &I : Index)
198 thinLTOResolveWeakForLinkerGUID(I.second, I.first, GlobalInvolvedWithAlias,
199 isPrevailing, recordNewLinkage);
202 static void thinLTOInternalizeAndPromoteGUID(
203 GlobalValueSummaryList &GVSummaryList, GlobalValue::GUID GUID,
204 function_ref<bool(StringRef, GlobalValue::GUID)> isExported) {
205 for (auto &S : GVSummaryList) {
206 if (isExported(S->modulePath(), GUID)) {
207 if (GlobalValue::isLocalLinkage(S->linkage()))
208 S->setLinkage(GlobalValue::ExternalLinkage);
209 } else if (!GlobalValue::isLocalLinkage(S->linkage()))
210 S->setLinkage(GlobalValue::InternalLinkage);
214 // Update the linkages in the given \p Index to mark exported values
215 // as external and non-exported values as internal.
216 void llvm::thinLTOInternalizeAndPromoteInIndex(
217 ModuleSummaryIndex &Index,
218 function_ref<bool(StringRef, GlobalValue::GUID)> isExported) {
219 for (auto &I : Index)
220 thinLTOInternalizeAndPromoteGUID(I.second, I.first, isExported);
223 struct InputFile::InputModule {
225 std::unique_ptr<Module> Mod;
227 // The range of ModuleSymbolTable entries for this input module.
228 size_t SymBegin, SymEnd;
231 // Requires a destructor for std::vector<InputModule>.
232 InputFile::~InputFile() = default;
234 Expected<std::unique_ptr<InputFile>> InputFile::create(MemoryBufferRef Object) {
235 std::unique_ptr<InputFile> File(new InputFile);
237 ErrorOr<MemoryBufferRef> BCOrErr =
238 IRObjectFile::findBitcodeInMemBuffer(Object);
240 return errorCodeToError(BCOrErr.getError());
242 Expected<std::vector<BitcodeModule>> BMsOrErr =
243 getBitcodeModuleList(*BCOrErr);
245 return BMsOrErr.takeError();
247 if (BMsOrErr->empty())
248 return make_error<StringError>("Bitcode file does not contain any modules",
249 inconvertibleErrorCode());
251 // Create an InputModule for each module in the InputFile, and add it to the
252 // ModuleSymbolTable.
253 for (auto BM : *BMsOrErr) {
254 Expected<std::unique_ptr<Module>> MOrErr =
255 BM.getLazyModule(File->Ctx, /*ShouldLazyLoadMetadata*/ true,
256 /*IsImporting*/ false);
258 return MOrErr.takeError();
260 size_t SymBegin = File->SymTab.symbols().size();
261 File->SymTab.addModule(MOrErr->get());
262 size_t SymEnd = File->SymTab.symbols().size();
264 for (const auto &C : (*MOrErr)->getComdatSymbolTable()) {
265 auto P = File->ComdatMap.insert(
266 std::make_pair(&C.second, File->Comdats.size()));
269 File->Comdats.push_back(C.first());
272 File->Mods.push_back({BM, std::move(*MOrErr), SymBegin, SymEnd});
275 return std::move(File);
278 Expected<int> InputFile::Symbol::getComdatIndex() const {
281 const GlobalObject *GO = getGV()->getBaseObject();
283 return make_error<StringError>("Unable to determine comdat of alias!",
284 inconvertibleErrorCode());
285 if (const Comdat *C = GO->getComdat()) {
286 auto I = File->ComdatMap.find(C);
287 assert(I != File->ComdatMap.end());
293 StringRef InputFile::getName() const {
294 return Mods[0].BM.getModuleIdentifier();
297 StringRef InputFile::getSourceFileName() const {
298 return Mods[0].Mod->getSourceFileName();
301 iterator_range<InputFile::symbol_iterator>
302 InputFile::module_symbols(InputModule &IM) {
303 return llvm::make_range(
304 symbol_iterator(SymTab.symbols().data() + IM.SymBegin, SymTab, this),
305 symbol_iterator(SymTab.symbols().data() + IM.SymEnd, SymTab, this));
308 LTO::RegularLTOState::RegularLTOState(unsigned ParallelCodeGenParallelismLevel,
310 : ParallelCodeGenParallelismLevel(ParallelCodeGenParallelismLevel),
313 LTO::ThinLTOState::ThinLTOState(ThinBackend Backend) : Backend(Backend) {
316 createInProcessThinBackend(llvm::heavyweight_hardware_concurrency());
319 LTO::LTO(Config Conf, ThinBackend Backend,
320 unsigned ParallelCodeGenParallelismLevel)
321 : Conf(std::move(Conf)),
322 RegularLTO(ParallelCodeGenParallelismLevel, this->Conf),
323 ThinLTO(std::move(Backend)) {}
325 // Requires a destructor for MapVector<BitcodeModule>.
326 LTO::~LTO() = default;
328 // Add the given symbol to the GlobalResolutions map, and resolve its partition.
329 void LTO::addSymbolToGlobalRes(SmallPtrSet<GlobalValue *, 8> &Used,
330 const InputFile::Symbol &Sym,
331 SymbolResolution Res, unsigned Partition) {
332 GlobalValue *GV = Sym.isGV() ? Sym.getGV() : nullptr;
334 auto &GlobalRes = GlobalResolutions[Sym.getName()];
336 GlobalRes.UnnamedAddr &= GV->hasGlobalUnnamedAddr();
338 GlobalRes.IRName = GV->getName();
340 if (Res.VisibleToRegularObj || (GV && Used.count(GV)) ||
341 (GlobalRes.Partition != GlobalResolution::Unknown &&
342 GlobalRes.Partition != Partition))
343 GlobalRes.Partition = GlobalResolution::External;
345 GlobalRes.Partition = Partition;
348 static void writeToResolutionFile(raw_ostream &OS, InputFile *Input,
349 ArrayRef<SymbolResolution> Res) {
350 StringRef Path = Input->getName();
352 auto ResI = Res.begin();
353 for (const InputFile::Symbol &Sym : Input->symbols()) {
354 assert(ResI != Res.end());
355 SymbolResolution Res = *ResI++;
357 OS << "-r=" << Path << ',' << Sym.getName() << ',';
360 if (Res.FinalDefinitionInLinkageUnit)
362 if (Res.VisibleToRegularObj)
366 assert(ResI == Res.end());
369 Error LTO::add(std::unique_ptr<InputFile> Input,
370 ArrayRef<SymbolResolution> Res) {
371 assert(!CalledGetMaxTasks);
373 if (Conf.ResolutionFile)
374 writeToResolutionFile(*Conf.ResolutionFile, Input.get(), Res);
376 const SymbolResolution *ResI = Res.begin();
377 for (InputFile::InputModule &IM : Input->Mods)
378 if (Error Err = addModule(*Input, IM, ResI, Res.end()))
381 assert(ResI == Res.end());
382 return Error::success();
385 Error LTO::addModule(InputFile &Input, InputFile::InputModule &IM,
386 const SymbolResolution *&ResI,
387 const SymbolResolution *ResE) {
388 // FIXME: move to backend
391 if (M.getDataLayoutStr().empty())
392 return make_error<StringError>("input module has no datalayout",
393 inconvertibleErrorCode());
395 if (!Conf.OverrideTriple.empty())
396 M.setTargetTriple(Conf.OverrideTriple);
397 else if (M.getTargetTriple().empty())
398 M.setTargetTriple(Conf.DefaultTriple);
400 Expected<bool> HasThinLTOSummary = IM.BM.hasSummary();
401 if (!HasThinLTOSummary)
402 return HasThinLTOSummary.takeError();
404 if (*HasThinLTOSummary)
405 return addThinLTO(IM.BM, M, Input.module_symbols(IM), ResI, ResE);
407 return addRegularLTO(IM.BM, ResI, ResE);
410 // Add a regular LTO object to the link.
411 Error LTO::addRegularLTO(BitcodeModule BM, const SymbolResolution *&ResI,
412 const SymbolResolution *ResE) {
413 if (!RegularLTO.CombinedModule) {
414 RegularLTO.CombinedModule =
415 llvm::make_unique<Module>("ld-temp.o", RegularLTO.Ctx);
416 RegularLTO.Mover = llvm::make_unique<IRMover>(*RegularLTO.CombinedModule);
418 Expected<std::unique_ptr<Module>> MOrErr =
419 BM.getLazyModule(RegularLTO.Ctx, /*ShouldLazyLoadMetadata*/ true,
420 /*IsImporting*/ false);
422 return MOrErr.takeError();
424 Module &M = **MOrErr;
425 if (Error Err = M.materializeMetadata())
429 ModuleSymbolTable SymTab;
430 SymTab.addModule(&M);
432 SmallPtrSet<GlobalValue *, 8> Used;
433 collectUsedGlobalVariables(M, Used, /*CompilerUsed*/ false);
435 std::vector<GlobalValue *> Keep;
437 for (GlobalVariable &GV : M.globals())
438 if (GV.hasAppendingLinkage())
441 for (const InputFile::Symbol &Sym :
442 make_range(InputFile::symbol_iterator(SymTab.symbols().begin(), SymTab,
444 InputFile::symbol_iterator(SymTab.symbols().end(), SymTab,
446 assert(ResI != ResE);
447 SymbolResolution Res = *ResI++;
448 addSymbolToGlobalRes(Used, Sym, Res, 0);
450 if (Sym.getFlags() & object::BasicSymbolRef::SF_Undefined)
452 if (Res.Prevailing && Sym.isGV()) {
453 GlobalValue *GV = Sym.getGV();
455 switch (GV->getLinkage()) {
458 case GlobalValue::LinkOnceAnyLinkage:
459 GV->setLinkage(GlobalValue::WeakAnyLinkage);
461 case GlobalValue::LinkOnceODRLinkage:
462 GV->setLinkage(GlobalValue::WeakODRLinkage);
466 // Common resolution: collect the maximum size/alignment over all commons.
467 // We also record if we see an instance of a common as prevailing, so that
468 // if none is prevailing we can ignore it later.
469 if (Sym.getFlags() & object::BasicSymbolRef::SF_Common) {
470 // FIXME: We should figure out what to do about commons defined by asm.
471 // For now they aren't reported correctly by ModuleSymbolTable.
472 auto &CommonRes = RegularLTO.Commons[Sym.getGV()->getName()];
473 CommonRes.Size = std::max(CommonRes.Size, Sym.getCommonSize());
474 CommonRes.Align = std::max(CommonRes.Align, Sym.getCommonAlignment());
475 CommonRes.Prevailing |= Res.Prevailing;
478 // FIXME: use proposed local attribute for FinalDefinitionInLinkageUnit.
481 return RegularLTO.Mover->move(std::move(*MOrErr), Keep,
482 [](GlobalValue &, IRMover::ValueAdder) {},
483 /* LinkModuleInlineAsm */ true,
484 /* IsPerformingImport */ false);
487 // Add a ThinLTO object to the link.
488 // FIXME: This function should not need to take as many parameters once we have
489 // a bitcode symbol table.
490 Error LTO::addThinLTO(BitcodeModule BM, Module &M,
491 iterator_range<InputFile::symbol_iterator> Syms,
492 const SymbolResolution *&ResI,
493 const SymbolResolution *ResE) {
494 SmallPtrSet<GlobalValue *, 8> Used;
495 collectUsedGlobalVariables(M, Used, /*CompilerUsed*/ false);
497 Expected<std::unique_ptr<ModuleSummaryIndex>> SummaryOrErr = BM.getSummary();
499 return SummaryOrErr.takeError();
500 ThinLTO.CombinedIndex.mergeFrom(std::move(*SummaryOrErr),
501 ThinLTO.ModuleMap.size());
503 for (const InputFile::Symbol &Sym : Syms) {
504 assert(ResI != ResE);
505 SymbolResolution Res = *ResI++;
506 addSymbolToGlobalRes(Used, Sym, Res, ThinLTO.ModuleMap.size() + 1);
508 if (Res.Prevailing && Sym.isGV())
509 ThinLTO.PrevailingModuleForGUID[Sym.getGV()->getGUID()] =
510 BM.getModuleIdentifier();
513 if (!ThinLTO.ModuleMap.insert({BM.getModuleIdentifier(), BM}).second)
514 return make_error<StringError>(
515 "Expected at most one ThinLTO module per bitcode file",
516 inconvertibleErrorCode());
518 return Error::success();
521 unsigned LTO::getMaxTasks() const {
522 CalledGetMaxTasks = true;
523 return RegularLTO.ParallelCodeGenParallelismLevel + ThinLTO.ModuleMap.size();
526 Error LTO::run(AddStreamFn AddStream, NativeObjectCache Cache) {
527 // Save the status of having a regularLTO combined module, as
528 // this is needed for generating the ThinLTO Task ID, and
529 // the CombinedModule will be moved at the end of runRegularLTO.
530 bool HasRegularLTO = RegularLTO.CombinedModule != nullptr;
531 // Invoke regular LTO if there was a regular LTO module to start with.
533 if (auto E = runRegularLTO(AddStream))
535 return runThinLTO(AddStream, Cache, HasRegularLTO);
538 Error LTO::runRegularLTO(AddStreamFn AddStream) {
539 // Make sure commons have the right size/alignment: we kept the largest from
540 // all the prevailing when adding the inputs, and we apply it here.
541 const DataLayout &DL = RegularLTO.CombinedModule->getDataLayout();
542 for (auto &I : RegularLTO.Commons) {
543 if (!I.second.Prevailing)
544 // Don't do anything if no instance of this common was prevailing.
546 GlobalVariable *OldGV = RegularLTO.CombinedModule->getNamedGlobal(I.first);
547 if (OldGV && DL.getTypeAllocSize(OldGV->getValueType()) == I.second.Size) {
548 // Don't create a new global if the type is already correct, just make
549 // sure the alignment is correct.
550 OldGV->setAlignment(I.second.Align);
554 ArrayType::get(Type::getInt8Ty(RegularLTO.Ctx), I.second.Size);
555 auto *GV = new GlobalVariable(*RegularLTO.CombinedModule, Ty, false,
556 GlobalValue::CommonLinkage,
557 ConstantAggregateZero::get(Ty), "");
558 GV->setAlignment(I.second.Align);
560 OldGV->replaceAllUsesWith(ConstantExpr::getBitCast(GV, OldGV->getType()));
562 OldGV->eraseFromParent();
564 GV->setName(I.first);
568 if (Conf.PreOptModuleHook &&
569 !Conf.PreOptModuleHook(0, *RegularLTO.CombinedModule))
570 return Error::success();
572 if (!Conf.CodeGenOnly) {
573 for (const auto &R : GlobalResolutions) {
574 if (R.second.IRName.empty())
576 if (R.second.Partition != 0 &&
577 R.second.Partition != GlobalResolution::External)
581 RegularLTO.CombinedModule->getNamedValue(R.second.IRName);
582 // Ignore symbols defined in other partitions.
583 if (!GV || GV->hasLocalLinkage())
585 GV->setUnnamedAddr(R.second.UnnamedAddr ? GlobalValue::UnnamedAddr::Global
586 : GlobalValue::UnnamedAddr::None);
587 if (R.second.Partition == 0)
588 GV->setLinkage(GlobalValue::InternalLinkage);
591 if (Conf.PostInternalizeModuleHook &&
592 !Conf.PostInternalizeModuleHook(0, *RegularLTO.CombinedModule))
593 return Error::success();
595 return backend(Conf, AddStream, RegularLTO.ParallelCodeGenParallelismLevel,
596 std::move(RegularLTO.CombinedModule));
599 /// This class defines the interface to the ThinLTO backend.
600 class lto::ThinBackendProc {
603 ModuleSummaryIndex &CombinedIndex;
604 const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries;
607 ThinBackendProc(Config &Conf, ModuleSummaryIndex &CombinedIndex,
608 const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries)
609 : Conf(Conf), CombinedIndex(CombinedIndex),
610 ModuleToDefinedGVSummaries(ModuleToDefinedGVSummaries) {}
612 virtual ~ThinBackendProc() {}
614 unsigned Task, BitcodeModule BM,
615 const FunctionImporter::ImportMapTy &ImportList,
616 const FunctionImporter::ExportSetTy &ExportList,
617 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
618 MapVector<StringRef, BitcodeModule> &ModuleMap) = 0;
619 virtual Error wait() = 0;
623 class InProcessThinBackend : public ThinBackendProc {
624 ThreadPool BackendThreadPool;
625 AddStreamFn AddStream;
626 NativeObjectCache Cache;
632 InProcessThinBackend(
633 Config &Conf, ModuleSummaryIndex &CombinedIndex,
634 unsigned ThinLTOParallelismLevel,
635 const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
636 AddStreamFn AddStream, NativeObjectCache Cache)
637 : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries),
638 BackendThreadPool(ThinLTOParallelismLevel),
639 AddStream(std::move(AddStream)), Cache(std::move(Cache)) {}
641 Error runThinLTOBackendThread(
642 AddStreamFn AddStream, NativeObjectCache Cache, unsigned Task,
643 BitcodeModule BM, ModuleSummaryIndex &CombinedIndex,
644 const FunctionImporter::ImportMapTy &ImportList,
645 const FunctionImporter::ExportSetTy &ExportList,
646 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
647 const GVSummaryMapTy &DefinedGlobals,
648 MapVector<StringRef, BitcodeModule> &ModuleMap) {
649 auto RunThinBackend = [&](AddStreamFn AddStream) {
650 LTOLLVMContext BackendContext(Conf);
651 Expected<std::unique_ptr<Module>> MOrErr = BM.parseModule(BackendContext);
653 return MOrErr.takeError();
655 return thinBackend(Conf, Task, AddStream, **MOrErr, CombinedIndex,
656 ImportList, DefinedGlobals, ModuleMap);
659 auto ModuleID = BM.getModuleIdentifier();
661 if (!Cache || !CombinedIndex.modulePaths().count(ModuleID) ||
662 all_of(CombinedIndex.getModuleHash(ModuleID),
663 [](uint32_t V) { return V == 0; }))
664 // Cache disabled or no entry for this module in the combined index or
666 return RunThinBackend(AddStream);
669 // The module may be cached, this helps handling it.
670 computeCacheKey(Key, Conf, CombinedIndex, ModuleID, ImportList, ExportList,
671 ResolvedODR, DefinedGlobals);
672 if (AddStreamFn CacheAddStream = Cache(Task, Key))
673 return RunThinBackend(CacheAddStream);
675 return Error::success();
679 unsigned Task, BitcodeModule BM,
680 const FunctionImporter::ImportMapTy &ImportList,
681 const FunctionImporter::ExportSetTy &ExportList,
682 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
683 MapVector<StringRef, BitcodeModule> &ModuleMap) override {
684 StringRef ModulePath = BM.getModuleIdentifier();
685 assert(ModuleToDefinedGVSummaries.count(ModulePath));
686 const GVSummaryMapTy &DefinedGlobals =
687 ModuleToDefinedGVSummaries.find(ModulePath)->second;
688 BackendThreadPool.async(
689 [=](BitcodeModule BM, ModuleSummaryIndex &CombinedIndex,
690 const FunctionImporter::ImportMapTy &ImportList,
691 const FunctionImporter::ExportSetTy &ExportList,
692 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>
694 const GVSummaryMapTy &DefinedGlobals,
695 MapVector<StringRef, BitcodeModule> &ModuleMap) {
696 Error E = runThinLTOBackendThread(
697 AddStream, Cache, Task, BM, CombinedIndex, ImportList,
698 ExportList, ResolvedODR, DefinedGlobals, ModuleMap);
700 std::unique_lock<std::mutex> L(ErrMu);
702 Err = joinErrors(std::move(*Err), std::move(E));
707 BM, std::ref(CombinedIndex), std::ref(ImportList),
708 std::ref(ExportList), std::ref(ResolvedODR), std::ref(DefinedGlobals),
709 std::ref(ModuleMap));
710 return Error::success();
713 Error wait() override {
714 BackendThreadPool.wait();
716 return std::move(*Err);
718 return Error::success();
721 } // end anonymous namespace
723 ThinBackend lto::createInProcessThinBackend(unsigned ParallelismLevel) {
724 return [=](Config &Conf, ModuleSummaryIndex &CombinedIndex,
725 const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
726 AddStreamFn AddStream, NativeObjectCache Cache) {
727 return llvm::make_unique<InProcessThinBackend>(
728 Conf, CombinedIndex, ParallelismLevel, ModuleToDefinedGVSummaries,
733 // Given the original \p Path to an output file, replace any path
734 // prefix matching \p OldPrefix with \p NewPrefix. Also, create the
735 // resulting directory if it does not yet exist.
736 std::string lto::getThinLTOOutputFile(const std::string &Path,
737 const std::string &OldPrefix,
738 const std::string &NewPrefix) {
739 if (OldPrefix.empty() && NewPrefix.empty())
741 SmallString<128> NewPath(Path);
742 llvm::sys::path::replace_path_prefix(NewPath, OldPrefix, NewPrefix);
743 StringRef ParentPath = llvm::sys::path::parent_path(NewPath.str());
744 if (!ParentPath.empty()) {
745 // Make sure the new directory exists, creating it if necessary.
746 if (std::error_code EC = llvm::sys::fs::create_directories(ParentPath))
747 llvm::errs() << "warning: could not create directory '" << ParentPath
748 << "': " << EC.message() << '\n';
750 return NewPath.str();
754 class WriteIndexesThinBackend : public ThinBackendProc {
755 std::string OldPrefix, NewPrefix;
756 bool ShouldEmitImportsFiles;
758 std::string LinkedObjectsFileName;
759 std::unique_ptr<llvm::raw_fd_ostream> LinkedObjectsFile;
762 WriteIndexesThinBackend(
763 Config &Conf, ModuleSummaryIndex &CombinedIndex,
764 const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
765 std::string OldPrefix, std::string NewPrefix, bool ShouldEmitImportsFiles,
766 std::string LinkedObjectsFileName)
767 : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries),
768 OldPrefix(OldPrefix), NewPrefix(NewPrefix),
769 ShouldEmitImportsFiles(ShouldEmitImportsFiles),
770 LinkedObjectsFileName(LinkedObjectsFileName) {}
773 unsigned Task, BitcodeModule BM,
774 const FunctionImporter::ImportMapTy &ImportList,
775 const FunctionImporter::ExportSetTy &ExportList,
776 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
777 MapVector<StringRef, BitcodeModule> &ModuleMap) override {
778 StringRef ModulePath = BM.getModuleIdentifier();
779 std::string NewModulePath =
780 getThinLTOOutputFile(ModulePath, OldPrefix, NewPrefix);
783 if (!LinkedObjectsFileName.empty()) {
784 if (!LinkedObjectsFile) {
785 LinkedObjectsFile = llvm::make_unique<raw_fd_ostream>(
786 LinkedObjectsFileName, EC, sys::fs::OpenFlags::F_None);
788 return errorCodeToError(EC);
790 *LinkedObjectsFile << NewModulePath << '\n';
793 std::map<std::string, GVSummaryMapTy> ModuleToSummariesForIndex;
794 gatherImportedSummariesForModule(ModulePath, ModuleToDefinedGVSummaries,
795 ImportList, ModuleToSummariesForIndex);
797 raw_fd_ostream OS(NewModulePath + ".thinlto.bc", EC,
798 sys::fs::OpenFlags::F_None);
800 return errorCodeToError(EC);
801 WriteIndexToFile(CombinedIndex, OS, &ModuleToSummariesForIndex);
803 if (ShouldEmitImportsFiles)
804 return errorCodeToError(
805 EmitImportsFiles(ModulePath, NewModulePath + ".imports", ImportList));
806 return Error::success();
809 Error wait() override { return Error::success(); }
811 } // end anonymous namespace
813 ThinBackend lto::createWriteIndexesThinBackend(std::string OldPrefix,
814 std::string NewPrefix,
815 bool ShouldEmitImportsFiles,
816 std::string LinkedObjectsFile) {
817 return [=](Config &Conf, ModuleSummaryIndex &CombinedIndex,
818 const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
819 AddStreamFn AddStream, NativeObjectCache Cache) {
820 return llvm::make_unique<WriteIndexesThinBackend>(
821 Conf, CombinedIndex, ModuleToDefinedGVSummaries, OldPrefix, NewPrefix,
822 ShouldEmitImportsFiles, LinkedObjectsFile);
826 Error LTO::runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache,
827 bool HasRegularLTO) {
828 if (ThinLTO.ModuleMap.empty())
829 return Error::success();
831 if (Conf.CombinedIndexHook && !Conf.CombinedIndexHook(ThinLTO.CombinedIndex))
832 return Error::success();
834 // Collect for each module the list of function it defines (GUID ->
836 StringMap<std::map<GlobalValue::GUID, GlobalValueSummary *>>
837 ModuleToDefinedGVSummaries(ThinLTO.ModuleMap.size());
838 ThinLTO.CombinedIndex.collectDefinedGVSummariesPerModule(
839 ModuleToDefinedGVSummaries);
840 // Create entries for any modules that didn't have any GV summaries
841 // (either they didn't have any GVs to start with, or we suppressed
842 // generation of the summaries because they e.g. had inline assembly
843 // uses that couldn't be promoted/renamed on export). This is so
844 // InProcessThinBackend::start can still launch a backend thread, which
845 // is passed the map of summaries for the module, without any special
846 // handling for this case.
847 for (auto &Mod : ThinLTO.ModuleMap)
848 if (!ModuleToDefinedGVSummaries.count(Mod.first))
849 ModuleToDefinedGVSummaries.try_emplace(Mod.first);
851 StringMap<FunctionImporter::ImportMapTy> ImportLists(
852 ThinLTO.ModuleMap.size());
853 StringMap<FunctionImporter::ExportSetTy> ExportLists(
854 ThinLTO.ModuleMap.size());
855 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
857 if (Conf.OptLevel > 0) {
858 ComputeCrossModuleImport(ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
859 ImportLists, ExportLists);
861 std::set<GlobalValue::GUID> ExportedGUIDs;
862 for (auto &Res : GlobalResolutions) {
863 if (!Res.second.IRName.empty() &&
864 Res.second.Partition == GlobalResolution::External)
865 ExportedGUIDs.insert(GlobalValue::getGUID(Res.second.IRName));
868 auto isPrevailing = [&](GlobalValue::GUID GUID,
869 const GlobalValueSummary *S) {
870 return ThinLTO.PrevailingModuleForGUID[GUID] == S->modulePath();
872 auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) {
873 const auto &ExportList = ExportLists.find(ModuleIdentifier);
874 return (ExportList != ExportLists.end() &&
875 ExportList->second.count(GUID)) ||
876 ExportedGUIDs.count(GUID);
878 thinLTOInternalizeAndPromoteInIndex(ThinLTO.CombinedIndex, isExported);
880 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
881 GlobalValue::GUID GUID,
882 GlobalValue::LinkageTypes NewLinkage) {
883 ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
886 thinLTOResolveWeakForLinkerInIndex(ThinLTO.CombinedIndex, isPrevailing,
890 std::unique_ptr<ThinBackendProc> BackendProc =
891 ThinLTO.Backend(Conf, ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
894 // Partition numbers for ThinLTO jobs start at 1 (see comments for
895 // GlobalResolution in LTO.h). Task numbers, however, start at
896 // ParallelCodeGenParallelismLevel if an LTO module is present, as tasks 0
897 // through ParallelCodeGenParallelismLevel-1 are reserved for parallel code
898 // generation partitions.
900 HasRegularLTO ? RegularLTO.ParallelCodeGenParallelismLevel : 0;
901 unsigned Partition = 1;
903 for (auto &Mod : ThinLTO.ModuleMap) {
904 if (Error E = BackendProc->start(Task, Mod.second, ImportLists[Mod.first],
905 ExportLists[Mod.first],
906 ResolvedODR[Mod.first], ThinLTO.ModuleMap))
913 return BackendProc->wait();