1 //===- ModuleManager.cpp - Module Manager ---------------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file defines the ModuleManager class, which manages a set of loaded
11 // modules for the ASTReader.
13 //===----------------------------------------------------------------------===//
15 #include "clang/Serialization/ModuleManager.h"
16 #include "clang/Basic/FileManager.h"
17 #include "clang/Basic/LLVM.h"
18 #include "clang/Basic/MemoryBufferCache.h"
19 #include "clang/Basic/VirtualFileSystem.h"
20 #include "clang/Frontend/PCHContainerOperations.h"
21 #include "clang/Lex/HeaderSearch.h"
22 #include "clang/Lex/ModuleMap.h"
23 #include "clang/Serialization/GlobalModuleIndex.h"
24 #include "clang/Serialization/Module.h"
25 #include "llvm/ADT/STLExtras.h"
26 #include "llvm/ADT/SetVector.h"
27 #include "llvm/ADT/SmallPtrSet.h"
28 #include "llvm/ADT/SmallVector.h"
29 #include "llvm/ADT/StringRef.h"
30 #include "llvm/ADT/iterator.h"
31 #include "llvm/Support/Chrono.h"
32 #include "llvm/Support/DOTGraphTraits.h"
33 #include "llvm/Support/ErrorOr.h"
34 #include "llvm/Support/GraphWriter.h"
35 #include "llvm/Support/MemoryBuffer.h"
40 #include <system_error>
42 using namespace clang;
43 using namespace serialization;
45 ModuleFile *ModuleManager::lookupByFileName(StringRef Name) const {
46 const FileEntry *Entry = FileMgr.getFile(Name, /*openFile=*/false,
47 /*cacheFailure=*/false);
54 ModuleFile *ModuleManager::lookupByModuleName(StringRef Name) const {
55 if (const Module *Mod = HeaderSearchInfo.getModuleMap().findModule(Name))
56 if (const FileEntry *File = Mod->getASTFile())
62 ModuleFile *ModuleManager::lookup(const FileEntry *File) const {
63 auto Known = Modules.find(File);
64 if (Known == Modules.end())
70 std::unique_ptr<llvm::MemoryBuffer>
71 ModuleManager::lookupBuffer(StringRef Name) {
72 const FileEntry *Entry = FileMgr.getFile(Name, /*openFile=*/false,
73 /*cacheFailure=*/false);
74 return std::move(InMemoryBuffers[Entry]);
77 static bool checkSignature(ASTFileSignature Signature,
78 ASTFileSignature ExpectedSignature,
79 std::string &ErrorStr) {
80 if (!ExpectedSignature || Signature == ExpectedSignature)
84 Signature ? "signature mismatch" : "could not read module signature";
88 static void updateModuleImports(ModuleFile &MF, ModuleFile *ImportedBy,
89 SourceLocation ImportLoc) {
91 MF.ImportedBy.insert(ImportedBy);
92 ImportedBy->Imports.insert(&MF);
94 if (!MF.DirectlyImported)
95 MF.ImportLoc = ImportLoc;
97 MF.DirectlyImported = true;
101 ModuleManager::AddModuleResult
102 ModuleManager::addModule(StringRef FileName, ModuleKind Type,
103 SourceLocation ImportLoc, ModuleFile *ImportedBy,
105 off_t ExpectedSize, time_t ExpectedModTime,
106 ASTFileSignature ExpectedSignature,
107 ASTFileSignatureReader ReadSignature,
109 std::string &ErrorStr) {
112 // Look for the file entry. This only fails if the expected size or
113 // modification time differ.
114 const FileEntry *Entry;
115 if (Type == MK_ExplicitModule || Type == MK_PrebuiltModule) {
116 // If we're not expecting to pull this file out of the module cache, it
117 // might have a different mtime due to being moved across filesystems in
118 // a distributed build. The size must still match, though. (As must the
119 // contents, but we can't check that.)
122 if (lookupModuleFile(FileName, ExpectedSize, ExpectedModTime, Entry)) {
123 ErrorStr = "module file out of date";
127 if (!Entry && FileName != "-") {
128 ErrorStr = "module file not found";
132 // Check whether we already loaded this module, before
133 if (ModuleFile *ModuleEntry = Modules.lookup(Entry)) {
134 // Check the stored signature.
135 if (checkSignature(ModuleEntry->Signature, ExpectedSignature, ErrorStr))
138 Module = ModuleEntry;
139 updateModuleImports(*ModuleEntry, ImportedBy, ImportLoc);
140 return AlreadyLoaded;
143 // Allocate a new module.
144 auto NewModule = llvm::make_unique<ModuleFile>(Type, Generation);
145 NewModule->Index = Chain.size();
146 NewModule->FileName = FileName.str();
147 NewModule->File = Entry;
148 NewModule->ImportLoc = ImportLoc;
149 NewModule->InputFilesValidationTimestamp = 0;
151 if (NewModule->Kind == MK_ImplicitModule) {
152 std::string TimestampFilename = NewModule->getTimestampFilename();
154 // A cached stat value would be fine as well.
155 if (!FileMgr.getNoncachedStatValue(TimestampFilename, Status))
156 NewModule->InputFilesValidationTimestamp =
157 llvm::sys::toTimeT(Status.getLastModificationTime());
160 // Load the contents of the module
161 if (std::unique_ptr<llvm::MemoryBuffer> Buffer = lookupBuffer(FileName)) {
162 // The buffer was already provided for us.
163 NewModule->Buffer = &PCMCache->addBuffer(FileName, std::move(Buffer));
164 } else if (llvm::MemoryBuffer *Buffer = PCMCache->lookupBuffer(FileName)) {
165 NewModule->Buffer = Buffer;
167 // Open the AST file.
168 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Buf((std::error_code()));
169 if (FileName == "-") {
170 Buf = llvm::MemoryBuffer::getSTDIN();
172 // Leave the FileEntry open so if it gets read again by another
173 // ModuleManager it must be the same underlying file.
174 // FIXME: Because FileManager::getFile() doesn't guarantee that it will
175 // give us an open file, this may not be 100% reliable.
176 Buf = FileMgr.getBufferForFile(NewModule->File,
177 /*IsVolatile=*/false,
178 /*ShouldClose=*/false);
182 ErrorStr = Buf.getError().message();
186 NewModule->Buffer = &PCMCache->addBuffer(FileName, std::move(*Buf));
189 // Initialize the stream.
190 NewModule->Data = PCHContainerRdr.ExtractPCH(*NewModule->Buffer);
192 // Read the signature eagerly now so that we can check it. Avoid calling
193 // ReadSignature unless there's something to check though.
194 if (ExpectedSignature && checkSignature(ReadSignature(NewModule->Data),
195 ExpectedSignature, ErrorStr)) {
196 // Try to remove the buffer. If it can't be removed, then it was already
197 // validated by this process.
198 if (!PCMCache->tryToRemoveBuffer(NewModule->FileName))
199 FileMgr.invalidateCache(NewModule->File);
203 // We're keeping this module. Store it everywhere.
204 Module = Modules[Entry] = NewModule.get();
206 updateModuleImports(*NewModule, ImportedBy, ImportLoc);
208 if (!NewModule->isModule())
209 PCHChain.push_back(NewModule.get());
211 Roots.push_back(NewModule.get());
213 Chain.push_back(std::move(NewModule));
217 void ModuleManager::removeModules(
218 ModuleIterator First,
219 llvm::SmallPtrSetImpl<ModuleFile *> &LoadedSuccessfully,
225 // Explicitly clear VisitOrder since we might not notice it is stale.
228 // Collect the set of module file pointers that we'll be removing.
229 llvm::SmallPtrSet<ModuleFile *, 4> victimSet(
230 (llvm::pointer_iterator<ModuleIterator>(First)),
231 (llvm::pointer_iterator<ModuleIterator>(Last)));
233 auto IsVictim = [&](ModuleFile *MF) {
234 return victimSet.count(MF);
236 // Remove any references to the now-destroyed modules.
237 for (auto I = begin(); I != First; ++I) {
238 I->Imports.remove_if(IsVictim);
239 I->ImportedBy.remove_if(IsVictim);
241 Roots.erase(std::remove_if(Roots.begin(), Roots.end(), IsVictim),
244 // Remove the modules from the PCH chain.
245 for (auto I = First; I != Last; ++I) {
246 if (!I->isModule()) {
247 PCHChain.erase(std::find(PCHChain.begin(), PCHChain.end(), &*I),
253 // Delete the modules and erase them from the various structures.
254 for (ModuleIterator victim = First; victim != Last; ++victim) {
255 Modules.erase(victim->File);
258 StringRef ModuleName = victim->ModuleName;
259 if (Module *mod = modMap->findModule(ModuleName)) {
260 mod->setASTFile(nullptr);
264 // Files that didn't make it through ReadASTCore successfully will be
265 // rebuilt (or there was an error). Invalidate them so that we can load the
266 // new files that will be renamed over the old ones.
268 // The PCMCache tracks whether the module was successfully loaded in another
269 // thread/context; in that case, it won't need to be rebuilt (and we can't
270 // safely invalidate it anyway).
271 if (LoadedSuccessfully.count(&*victim) == 0 &&
272 !PCMCache->tryToRemoveBuffer(victim->FileName))
273 FileMgr.invalidateCache(victim->File);
276 // Delete the modules.
277 Chain.erase(Chain.begin() + (First - begin()), Chain.end());
281 ModuleManager::addInMemoryBuffer(StringRef FileName,
282 std::unique_ptr<llvm::MemoryBuffer> Buffer) {
283 const FileEntry *Entry =
284 FileMgr.getVirtualFile(FileName, Buffer->getBufferSize(), 0);
285 InMemoryBuffers[Entry] = std::move(Buffer);
288 ModuleManager::VisitState *ModuleManager::allocateVisitState() {
289 // Fast path: if we have a cached state, use it.
290 if (FirstVisitState) {
291 VisitState *Result = FirstVisitState;
292 FirstVisitState = FirstVisitState->NextState;
293 Result->NextState = nullptr;
297 // Allocate and return a new state.
298 return new VisitState(size());
301 void ModuleManager::returnVisitState(VisitState *State) {
302 assert(State->NextState == nullptr && "Visited state is in list?");
303 State->NextState = FirstVisitState;
304 FirstVisitState = State;
307 void ModuleManager::setGlobalIndex(GlobalModuleIndex *Index) {
310 ModulesInCommonWithGlobalIndex.clear();
314 // Notify the global module index about all of the modules we've already
316 for (ModuleFile &M : *this)
317 if (!GlobalIndex->loadedModuleFile(&M))
318 ModulesInCommonWithGlobalIndex.push_back(&M);
321 void ModuleManager::moduleFileAccepted(ModuleFile *MF) {
322 if (!GlobalIndex || GlobalIndex->loadedModuleFile(MF))
325 ModulesInCommonWithGlobalIndex.push_back(MF);
328 ModuleManager::ModuleManager(FileManager &FileMgr, MemoryBufferCache &PCMCache,
329 const PCHContainerReader &PCHContainerRdr,
330 const HeaderSearch& HeaderSearchInfo)
331 : FileMgr(FileMgr), PCMCache(&PCMCache), PCHContainerRdr(PCHContainerRdr),
332 HeaderSearchInfo(HeaderSearchInfo) {}
334 ModuleManager::~ModuleManager() { delete FirstVisitState; }
336 void ModuleManager::visit(llvm::function_ref<bool(ModuleFile &M)> Visitor,
337 llvm::SmallPtrSetImpl<ModuleFile *> *ModuleFilesHit) {
338 // If the visitation order vector is the wrong size, recompute the order.
339 if (VisitOrder.size() != Chain.size()) {
342 VisitOrder.reserve(N);
344 // Record the number of incoming edges for each module. When we
345 // encounter a module with no incoming edges, push it into the queue
346 // to seed the queue.
347 SmallVector<ModuleFile *, 4> Queue;
349 llvm::SmallVector<unsigned, 4> UnusedIncomingEdges;
350 UnusedIncomingEdges.resize(size());
351 for (ModuleFile &M : llvm::reverse(*this)) {
352 unsigned Size = M.ImportedBy.size();
353 UnusedIncomingEdges[M.Index] = Size;
358 // Traverse the graph, making sure to visit a module before visiting any
359 // of its dependencies.
360 while (!Queue.empty()) {
361 ModuleFile *CurrentModule = Queue.pop_back_val();
362 VisitOrder.push_back(CurrentModule);
364 // For any module that this module depends on, push it on the
365 // stack (if it hasn't already been marked as visited).
366 for (auto M = CurrentModule->Imports.rbegin(),
367 MEnd = CurrentModule->Imports.rend();
369 // Remove our current module as an impediment to visiting the
370 // module we depend on. If we were the last unvisited module
371 // that depends on this particular module, push it into the
372 // queue to be visited.
373 unsigned &NumUnusedEdges = UnusedIncomingEdges[(*M)->Index];
374 if (NumUnusedEdges && (--NumUnusedEdges == 0))
379 assert(VisitOrder.size() == N && "Visitation order is wrong?");
381 delete FirstVisitState;
382 FirstVisitState = nullptr;
385 VisitState *State = allocateVisitState();
386 unsigned VisitNumber = State->NextVisitNumber++;
388 // If the caller has provided us with a hit-set that came from the global
389 // module index, mark every module file in common with the global module
390 // index that is *not* in that set as 'visited'.
391 if (ModuleFilesHit && !ModulesInCommonWithGlobalIndex.empty()) {
392 for (unsigned I = 0, N = ModulesInCommonWithGlobalIndex.size(); I != N; ++I)
394 ModuleFile *M = ModulesInCommonWithGlobalIndex[I];
395 if (!ModuleFilesHit->count(M))
396 State->VisitNumber[M->Index] = VisitNumber;
400 for (unsigned I = 0, N = VisitOrder.size(); I != N; ++I) {
401 ModuleFile *CurrentModule = VisitOrder[I];
402 // Should we skip this module file?
403 if (State->VisitNumber[CurrentModule->Index] == VisitNumber)
407 assert(State->VisitNumber[CurrentModule->Index] == VisitNumber - 1);
408 State->VisitNumber[CurrentModule->Index] = VisitNumber;
409 if (!Visitor(*CurrentModule))
412 // The visitor has requested that cut off visitation of any
413 // module that the current module depends on. To indicate this
414 // behavior, we mark all of the reachable modules as having been visited.
415 ModuleFile *NextModule = CurrentModule;
417 // For any module that this module depends on, push it on the
418 // stack (if it hasn't already been marked as visited).
419 for (llvm::SetVector<ModuleFile *>::iterator
420 M = NextModule->Imports.begin(),
421 MEnd = NextModule->Imports.end();
423 if (State->VisitNumber[(*M)->Index] != VisitNumber) {
424 State->Stack.push_back(*M);
425 State->VisitNumber[(*M)->Index] = VisitNumber;
429 if (State->Stack.empty())
432 // Pop the next module off the stack.
433 NextModule = State->Stack.pop_back_val();
437 returnVisitState(State);
440 bool ModuleManager::lookupModuleFile(StringRef FileName,
442 time_t ExpectedModTime,
443 const FileEntry *&File) {
444 if (FileName == "-") {
449 // Open the file immediately to ensure there is no race between stat'ing and
451 File = FileMgr.getFile(FileName, /*openFile=*/true, /*cacheFailure=*/false);
455 if ((ExpectedSize && ExpectedSize != File->getSize()) ||
456 (ExpectedModTime && ExpectedModTime != File->getModificationTime()))
457 // Do not destroy File, as it may be referenced. If we need to rebuild it,
458 // it will be destroyed by removeModules.
468 struct GraphTraits<ModuleManager> {
469 using NodeRef = ModuleFile *;
470 using ChildIteratorType = llvm::SetVector<ModuleFile *>::const_iterator;
471 using nodes_iterator = pointer_iterator<ModuleManager::ModuleConstIterator>;
473 static ChildIteratorType child_begin(NodeRef Node) {
474 return Node->Imports.begin();
477 static ChildIteratorType child_end(NodeRef Node) {
478 return Node->Imports.end();
481 static nodes_iterator nodes_begin(const ModuleManager &Manager) {
482 return nodes_iterator(Manager.begin());
485 static nodes_iterator nodes_end(const ModuleManager &Manager) {
486 return nodes_iterator(Manager.end());
491 struct DOTGraphTraits<ModuleManager> : public DefaultDOTGraphTraits {
492 explicit DOTGraphTraits(bool IsSimple = false)
493 : DefaultDOTGraphTraits(IsSimple) {}
495 static bool renderGraphFromBottomUp() { return true; }
497 std::string getNodeLabel(ModuleFile *M, const ModuleManager&) {
498 return M->ModuleName;
504 void ModuleManager::viewGraph() {
505 llvm::ViewGraph(*this, "Modules");