1 //===--- PrecompiledPreamble.cpp - Build precompiled preambles --*- C++ -*-===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // Helper class to build precompiled preamble.
12 //===----------------------------------------------------------------------===//
14 #include "clang/Frontend/PrecompiledPreamble.h"
15 #include "clang/AST/DeclObjC.h"
16 #include "clang/Basic/TargetInfo.h"
17 #include "clang/Basic/VirtualFileSystem.h"
18 #include "clang/Frontend/CompilerInstance.h"
19 #include "clang/Frontend/CompilerInvocation.h"
20 #include "clang/Frontend/FrontendActions.h"
21 #include "clang/Frontend/FrontendOptions.h"
22 #include "clang/Lex/Lexer.h"
23 #include "clang/Lex/PreprocessorOptions.h"
24 #include "clang/Serialization/ASTWriter.h"
25 #include "llvm/ADT/StringExtras.h"
26 #include "llvm/ADT/StringSet.h"
27 #include "llvm/Support/CrashRecoveryContext.h"
28 #include "llvm/Support/FileSystem.h"
29 #include "llvm/Support/Mutex.h"
30 #include "llvm/Support/MutexGuard.h"
32 using namespace clang;
36 /// Keeps a track of files to be deleted in destructor.
37 class TemporaryFiles {
39 // A static instance to be used by all clients.
40 static TemporaryFiles &getInstance();
43 // Disallow constructing the class directly.
44 TemporaryFiles() = default;
46 TemporaryFiles(const TemporaryFiles &) = delete;
51 /// Adds \p File to a set of tracked files.
52 void addFile(StringRef File);
54 /// Remove \p File from disk and from the set of tracked files.
55 void removeFile(StringRef File);
58 llvm::sys::SmartMutex<false> Mutex;
59 llvm::StringSet<> Files;
62 TemporaryFiles &TemporaryFiles::getInstance() {
63 static TemporaryFiles Instance;
67 TemporaryFiles::~TemporaryFiles() {
68 llvm::MutexGuard Guard(Mutex);
69 for (const auto &File : Files)
70 llvm::sys::fs::remove(File.getKey());
73 void TemporaryFiles::addFile(StringRef File) {
74 llvm::MutexGuard Guard(Mutex);
75 auto IsInserted = Files.insert(File).second;
77 assert(IsInserted && "File has already been added");
80 void TemporaryFiles::removeFile(StringRef File) {
81 llvm::MutexGuard Guard(Mutex);
82 auto WasPresent = Files.erase(File);
84 assert(WasPresent && "File was not tracked");
85 llvm::sys::fs::remove(File);
88 class PreambleMacroCallbacks : public PPCallbacks {
90 PreambleMacroCallbacks(PreambleCallbacks &Callbacks) : Callbacks(Callbacks) {}
92 void MacroDefined(const Token &MacroNameTok,
93 const MacroDirective *MD) override {
94 Callbacks.HandleMacroDefined(MacroNameTok, MD);
98 PreambleCallbacks &Callbacks;
101 class PrecompilePreambleAction : public ASTFrontendAction {
103 PrecompilePreambleAction(PreambleCallbacks &Callbacks)
104 : Callbacks(Callbacks) {}
106 std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
107 StringRef InFile) override;
109 bool hasEmittedPreamblePCH() const { return HasEmittedPreamblePCH; }
111 void setEmittedPreamblePCH(ASTWriter &Writer) {
112 this->HasEmittedPreamblePCH = true;
113 Callbacks.AfterPCHEmitted(Writer);
116 bool shouldEraseOutputFiles() override { return !hasEmittedPreamblePCH(); }
117 bool hasCodeCompletionSupport() const override { return false; }
118 bool hasASTFileSupport() const override { return false; }
119 TranslationUnitKind getTranslationUnitKind() override { return TU_Prefix; }
122 friend class PrecompilePreambleConsumer;
124 bool HasEmittedPreamblePCH = false;
125 PreambleCallbacks &Callbacks;
128 class PrecompilePreambleConsumer : public PCHGenerator {
130 PrecompilePreambleConsumer(PrecompilePreambleAction &Action,
131 const Preprocessor &PP, StringRef isysroot,
132 std::unique_ptr<raw_ostream> Out)
133 : PCHGenerator(PP, "", isysroot, std::make_shared<PCHBuffer>(),
134 ArrayRef<std::shared_ptr<ModuleFileExtension>>(),
135 /*AllowASTWithErrors=*/true),
136 Action(Action), Out(std::move(Out)) {}
138 bool HandleTopLevelDecl(DeclGroupRef DG) override {
139 Action.Callbacks.HandleTopLevelDecl(DG);
143 void HandleTranslationUnit(ASTContext &Ctx) override {
144 PCHGenerator::HandleTranslationUnit(Ctx);
145 if (!hasEmittedPCH())
148 // Write the generated bitstream to "Out".
150 // Make sure it hits disk now.
153 llvm::SmallVector<char, 0> Empty;
154 getPCH() = std::move(Empty);
156 Action.setEmittedPreamblePCH(getWriter());
160 PrecompilePreambleAction &Action;
161 std::unique_ptr<raw_ostream> Out;
164 std::unique_ptr<ASTConsumer>
165 PrecompilePreambleAction::CreateASTConsumer(CompilerInstance &CI,
169 std::string OutputFile;
170 std::unique_ptr<raw_ostream> OS =
171 GeneratePCHAction::ComputeASTConsumerArguments(CI, InFile, Sysroot,
176 if (!CI.getFrontendOpts().RelocatablePCH)
179 CI.getPreprocessor().addPPCallbacks(
180 llvm::make_unique<PreambleMacroCallbacks>(Callbacks));
181 return llvm::make_unique<PrecompilePreambleConsumer>(
182 *this, CI.getPreprocessor(), Sysroot, std::move(OS));
185 template <class T> bool moveOnNoError(llvm::ErrorOr<T> Val, T &Output) {
188 Output = std::move(*Val);
194 PreambleBounds clang::ComputePreambleBounds(const LangOptions &LangOpts,
195 llvm::MemoryBuffer *Buffer,
197 auto Pre = Lexer::ComputePreamble(Buffer->getBuffer(), LangOpts, MaxLines);
198 return PreambleBounds(Pre.first, Pre.second);
201 llvm::ErrorOr<PrecompiledPreamble> PrecompiledPreamble::Build(
202 const CompilerInvocation &Invocation,
203 const llvm::MemoryBuffer *MainFileBuffer, PreambleBounds Bounds,
204 DiagnosticsEngine &Diagnostics, IntrusiveRefCntPtr<vfs::FileSystem> VFS,
205 std::shared_ptr<PCHContainerOperations> PCHContainerOps,
206 PreambleCallbacks &Callbacks) {
207 assert(VFS && "VFS is null");
210 return BuildPreambleError::PreambleIsEmpty;
212 auto PreambleInvocation = std::make_shared<CompilerInvocation>(Invocation);
213 FrontendOptions &FrontendOpts = PreambleInvocation->getFrontendOpts();
214 PreprocessorOptions &PreprocessorOpts =
215 PreambleInvocation->getPreprocessorOpts();
217 // Create a temporary file for the precompiled preamble. In rare
218 // circumstances, this can fail.
219 llvm::ErrorOr<PrecompiledPreamble::TempPCHFile> PreamblePCHFile =
220 PrecompiledPreamble::TempPCHFile::CreateNewPreamblePCHFile();
221 if (!PreamblePCHFile)
222 return BuildPreambleError::CouldntCreateTempFile;
224 // Save the preamble text for later; we'll need to compare against it for
225 // subsequent reparses.
226 std::vector<char> PreambleBytes(MainFileBuffer->getBufferStart(),
227 MainFileBuffer->getBufferStart() +
229 bool PreambleEndsAtStartOfLine = Bounds.PreambleEndsAtStartOfLine;
231 // Tell the compiler invocation to generate a temporary precompiled header.
232 FrontendOpts.ProgramAction = frontend::GeneratePCH;
233 // FIXME: Generate the precompiled header into memory?
234 FrontendOpts.OutputFile = PreamblePCHFile->getFilePath();
235 PreprocessorOpts.PrecompiledPreambleBytes.first = 0;
236 PreprocessorOpts.PrecompiledPreambleBytes.second = false;
238 // Create the compiler instance to use for building the precompiled preamble.
239 std::unique_ptr<CompilerInstance> Clang(
240 new CompilerInstance(std::move(PCHContainerOps)));
242 // Recover resources if we crash before exiting this method.
243 llvm::CrashRecoveryContextCleanupRegistrar<CompilerInstance> CICleanup(
246 Clang->setInvocation(std::move(PreambleInvocation));
247 Clang->setDiagnostics(&Diagnostics);
249 // Create the target instance.
250 Clang->setTarget(TargetInfo::CreateTargetInfo(
251 Clang->getDiagnostics(), Clang->getInvocation().TargetOpts));
252 if (!Clang->hasTarget())
253 return BuildPreambleError::CouldntCreateTargetInfo;
255 // Inform the target of the language options.
257 // FIXME: We shouldn't need to do this, the target should be immutable once
258 // created. This complexity should be lifted elsewhere.
259 Clang->getTarget().adjust(Clang->getLangOpts());
261 assert(Clang->getFrontendOpts().Inputs.size() == 1 &&
262 "Invocation must have exactly one source file!");
263 assert(Clang->getFrontendOpts().Inputs[0].getKind().getFormat() ==
265 "FIXME: AST inputs not yet supported here!");
266 assert(Clang->getFrontendOpts().Inputs[0].getKind().getLanguage() !=
267 InputKind::LLVM_IR &&
268 "IR inputs not support here!");
270 // Clear out old caches and data.
272 ProcessWarningOptions(Diagnostics, Clang->getDiagnosticOpts());
275 createVFSFromCompilerInvocation(Clang->getInvocation(), Diagnostics, VFS);
277 return BuildPreambleError::CouldntCreateVFSOverlay;
279 // Create a file manager object to provide access to and cache the filesystem.
280 Clang->setFileManager(new FileManager(Clang->getFileSystemOpts(), VFS));
282 // Create the source manager.
283 Clang->setSourceManager(
284 new SourceManager(Diagnostics, Clang->getFileManager()));
286 auto PreambleDepCollector = std::make_shared<DependencyCollector>();
287 Clang->addDependencyCollector(PreambleDepCollector);
289 // Remap the main source file to the preamble buffer.
290 StringRef MainFilePath = FrontendOpts.Inputs[0].getFile();
291 auto PreambleInputBuffer = llvm::MemoryBuffer::getMemBufferCopy(
292 MainFileBuffer->getBuffer().slice(0, Bounds.Size), MainFilePath);
293 if (PreprocessorOpts.RetainRemappedFileBuffers) {
294 // MainFileBuffer will be deleted by unique_ptr after leaving the method.
295 PreprocessorOpts.addRemappedFile(MainFilePath, PreambleInputBuffer.get());
297 // In that case, remapped buffer will be deleted by CompilerInstance on
298 // BeginSourceFile, so we call release() to avoid double deletion.
299 PreprocessorOpts.addRemappedFile(MainFilePath,
300 PreambleInputBuffer.release());
303 std::unique_ptr<PrecompilePreambleAction> Act;
304 Act.reset(new PrecompilePreambleAction(Callbacks));
305 if (!Act->BeginSourceFile(*Clang.get(), Clang->getFrontendOpts().Inputs[0]))
306 return BuildPreambleError::BeginSourceFileFailed;
310 // Run the callbacks.
311 Callbacks.AfterExecute(*Clang);
313 Act->EndSourceFile();
315 if (!Act->hasEmittedPreamblePCH())
316 return BuildPreambleError::CouldntEmitPCH;
318 // Keep track of all of the files that the source manager knows about,
319 // so we can verify whether they have changed or not.
320 llvm::StringMap<PrecompiledPreamble::PreambleFileHash> FilesInPreamble;
322 SourceManager &SourceMgr = Clang->getSourceManager();
323 for (auto &Filename : PreambleDepCollector->getDependencies()) {
324 const FileEntry *File = Clang->getFileManager().getFile(Filename);
325 if (!File || File == SourceMgr.getFileEntryForID(SourceMgr.getMainFileID()))
327 if (time_t ModTime = File->getModificationTime()) {
328 FilesInPreamble[File->getName()] =
329 PrecompiledPreamble::PreambleFileHash::createForFile(File->getSize(),
332 llvm::MemoryBuffer *Buffer = SourceMgr.getMemoryBufferForFile(File);
333 FilesInPreamble[File->getName()] =
334 PrecompiledPreamble::PreambleFileHash::createForMemoryBuffer(Buffer);
338 return PrecompiledPreamble(
339 std::move(*PreamblePCHFile), std::move(PreambleBytes),
340 PreambleEndsAtStartOfLine, std::move(FilesInPreamble));
343 PreambleBounds PrecompiledPreamble::getBounds() const {
344 return PreambleBounds(PreambleBytes.size(), PreambleEndsAtStartOfLine);
347 bool PrecompiledPreamble::CanReuse(const CompilerInvocation &Invocation,
348 const llvm::MemoryBuffer *MainFileBuffer,
349 PreambleBounds Bounds,
350 vfs::FileSystem *VFS) const {
353 Bounds.Size <= MainFileBuffer->getBufferSize() &&
354 "Buffer is too large. Bounds were calculated from a different buffer?");
356 auto PreambleInvocation = std::make_shared<CompilerInvocation>(Invocation);
357 PreprocessorOptions &PreprocessorOpts =
358 PreambleInvocation->getPreprocessorOpts();
363 // We've previously computed a preamble. Check whether we have the same
364 // preamble now that we did before, and that there's enough space in
365 // the main-file buffer within the precompiled preamble to fit the
367 if (PreambleBytes.size() != Bounds.Size ||
368 PreambleEndsAtStartOfLine != Bounds.PreambleEndsAtStartOfLine ||
369 memcmp(PreambleBytes.data(), MainFileBuffer->getBufferStart(),
372 // The preamble has not changed. We may be able to re-use the precompiled
375 // Check that none of the files used by the preamble have changed.
376 // First, make a record of those files that have been overridden via
377 // remapping or unsaved_files.
378 std::map<llvm::sys::fs::UniqueID, PreambleFileHash> OverriddenFiles;
379 for (const auto &R : PreprocessorOpts.RemappedFiles) {
381 if (!moveOnNoError(VFS->status(R.second), Status)) {
382 // If we can't stat the file we're remapping to, assume that something
383 // horrible happened.
387 OverriddenFiles[Status.getUniqueID()] = PreambleFileHash::createForFile(
388 Status.getSize(), llvm::sys::toTimeT(Status.getLastModificationTime()));
391 for (const auto &RB : PreprocessorOpts.RemappedFileBuffers) {
393 if (!moveOnNoError(VFS->status(RB.first), Status))
396 OverriddenFiles[Status.getUniqueID()] =
397 PreambleFileHash::createForMemoryBuffer(RB.second);
400 // Check whether anything has changed.
401 for (const auto &F : FilesInPreamble) {
403 if (!moveOnNoError(VFS->status(F.first()), Status)) {
404 // If we can't stat the file, assume that something horrible happened.
408 std::map<llvm::sys::fs::UniqueID, PreambleFileHash>::iterator Overridden =
409 OverriddenFiles.find(Status.getUniqueID());
410 if (Overridden != OverriddenFiles.end()) {
411 // This file was remapped; check whether the newly-mapped file
412 // matches up with the previous mapping.
413 if (Overridden->second != F.second)
418 // The file was not remapped; check whether it has changed on disk.
419 if (Status.getSize() != uint64_t(F.second.Size) ||
420 llvm::sys::toTimeT(Status.getLastModificationTime()) !=
427 void PrecompiledPreamble::AddImplicitPreamble(
428 CompilerInvocation &CI, llvm::MemoryBuffer *MainFileBuffer) const {
429 auto &PreprocessorOpts = CI.getPreprocessorOpts();
431 // Configure ImpicitPCHInclude.
432 PreprocessorOpts.PrecompiledPreambleBytes.first = PreambleBytes.size();
433 PreprocessorOpts.PrecompiledPreambleBytes.second = PreambleEndsAtStartOfLine;
434 PreprocessorOpts.ImplicitPCHInclude = PCHFile.getFilePath();
435 PreprocessorOpts.DisablePCHValidation = true;
437 // Remap main file to point to MainFileBuffer.
438 auto MainFilePath = CI.getFrontendOpts().Inputs[0].getFile();
439 PreprocessorOpts.addRemappedFile(MainFilePath, MainFileBuffer);
442 PrecompiledPreamble::PrecompiledPreamble(
443 TempPCHFile PCHFile, std::vector<char> PreambleBytes,
444 bool PreambleEndsAtStartOfLine,
445 llvm::StringMap<PreambleFileHash> FilesInPreamble)
446 : PCHFile(std::move(PCHFile)), FilesInPreamble(FilesInPreamble),
447 PreambleBytes(std::move(PreambleBytes)),
448 PreambleEndsAtStartOfLine(PreambleEndsAtStartOfLine) {}
450 llvm::ErrorOr<PrecompiledPreamble::TempPCHFile>
451 PrecompiledPreamble::TempPCHFile::CreateNewPreamblePCHFile() {
452 // FIXME: This is a hack so that we can override the preamble file during
453 // crash-recovery testing, which is the only case where the preamble files
454 // are not necessarily cleaned up.
455 const char *TmpFile = ::getenv("CINDEXTEST_PREAMBLE_FILE");
457 return TempPCHFile::createFromCustomPath(TmpFile);
458 return TempPCHFile::createInSystemTempDir("preamble", "pch");
461 llvm::ErrorOr<PrecompiledPreamble::TempPCHFile>
462 PrecompiledPreamble::TempPCHFile::createInSystemTempDir(const Twine &Prefix,
464 llvm::SmallString<64> File;
465 auto EC = llvm::sys::fs::createTemporaryFile(Prefix, Suffix, /*ref*/ File);
468 return TempPCHFile(std::move(File).str());
471 llvm::ErrorOr<PrecompiledPreamble::TempPCHFile>
472 PrecompiledPreamble::TempPCHFile::createFromCustomPath(const Twine &Path) {
473 return TempPCHFile(Path.str());
476 PrecompiledPreamble::TempPCHFile::TempPCHFile(std::string FilePath)
477 : FilePath(std::move(FilePath)) {
478 TemporaryFiles::getInstance().addFile(*this->FilePath);
481 PrecompiledPreamble::TempPCHFile::TempPCHFile(TempPCHFile &&Other) {
482 FilePath = std::move(Other.FilePath);
483 Other.FilePath = None;
486 PrecompiledPreamble::TempPCHFile &PrecompiledPreamble::TempPCHFile::
487 operator=(TempPCHFile &&Other) {
488 RemoveFileIfPresent();
490 FilePath = std::move(Other.FilePath);
491 Other.FilePath = None;
495 PrecompiledPreamble::TempPCHFile::~TempPCHFile() { RemoveFileIfPresent(); }
497 void PrecompiledPreamble::TempPCHFile::RemoveFileIfPresent() {
499 TemporaryFiles::getInstance().removeFile(*FilePath);
504 llvm::StringRef PrecompiledPreamble::TempPCHFile::getFilePath() const {
505 assert(FilePath && "TempPCHFile doesn't have a FilePath. Had it been moved?");
509 PrecompiledPreamble::PreambleFileHash
510 PrecompiledPreamble::PreambleFileHash::createForFile(off_t Size,
512 PreambleFileHash Result;
514 Result.ModTime = ModTime;
519 PrecompiledPreamble::PreambleFileHash
520 PrecompiledPreamble::PreambleFileHash::createForMemoryBuffer(
521 const llvm::MemoryBuffer *Buffer) {
522 PreambleFileHash Result;
523 Result.Size = Buffer->getBufferSize();
527 MD5Ctx.update(Buffer->getBuffer().data());
528 MD5Ctx.final(Result.MD5);
533 void PreambleCallbacks::AfterExecute(CompilerInstance &CI) {}
534 void PreambleCallbacks::AfterPCHEmitted(ASTWriter &Writer) {}
535 void PreambleCallbacks::HandleTopLevelDecl(DeclGroupRef DG) {}
536 void PreambleCallbacks::HandleMacroDefined(const Token &MacroNameTok,
537 const MacroDirective *MD) {}
539 std::error_code clang::make_error_code(BuildPreambleError Error) {
540 return std::error_code(static_cast<int>(Error), BuildPreambleErrorCategory());
543 const char *BuildPreambleErrorCategory::name() const noexcept {
544 return "build-preamble.error";
547 std::string BuildPreambleErrorCategory::message(int condition) const {
548 switch (static_cast<BuildPreambleError>(condition)) {
549 case BuildPreambleError::PreambleIsEmpty:
550 return "Preamble is empty";
551 case BuildPreambleError::CouldntCreateTempFile:
552 return "Could not create temporary file for PCH";
553 case BuildPreambleError::CouldntCreateTargetInfo:
554 return "CreateTargetInfo() return null";
555 case BuildPreambleError::CouldntCreateVFSOverlay:
556 return "Could not create VFS Overlay";
557 case BuildPreambleError::BeginSourceFileFailed:
558 return "BeginSourceFile() return an error";
559 case BuildPreambleError::CouldntEmitPCH:
560 return "Could not emit PCH";
562 llvm_unreachable("unexpected BuildPreambleError");