1 //===- DependencyScanningWorker.cpp - clang-scan-deps worker --------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
10 #include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
11 #include "clang/Frontend/CompilerInstance.h"
12 #include "clang/Frontend/CompilerInvocation.h"
13 #include "clang/Frontend/FrontendActions.h"
14 #include "clang/Frontend/TextDiagnosticPrinter.h"
15 #include "clang/Frontend/Utils.h"
16 #include "clang/Lex/PreprocessorOptions.h"
17 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
18 #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
19 #include "clang/Tooling/Tooling.h"
21 using namespace clang;
22 using namespace tooling;
23 using namespace dependencies;
27 /// Forwards the gatherered dependencies to the consumer.
28 class DependencyConsumerForwarder : public DependencyFileGenerator {
30 DependencyConsumerForwarder(std::unique_ptr<DependencyOutputOptions> Opts,
31 DependencyConsumer &C)
32 : DependencyFileGenerator(*Opts), Opts(std::move(Opts)), C(C) {}
34 void finishedMainFile(DiagnosticsEngine &Diags) override {
35 C.handleDependencyOutputOpts(*Opts);
36 llvm::SmallString<256> CanonPath;
37 for (const auto &File : getDependencies()) {
39 llvm::sys::path::remove_dots(CanonPath, /*remove_dot_dot=*/true);
40 C.handleFileDependency(CanonPath);
45 std::unique_ptr<DependencyOutputOptions> Opts;
46 DependencyConsumer &C;
49 using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
51 /// A listener that collects the imported modules and optionally the input
53 class PrebuiltModuleListener : public ASTReaderListener {
55 PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles,
56 llvm::StringSet<> &InputFiles, bool VisitInputFiles,
57 llvm::SmallVector<std::string> &NewModuleFiles)
58 : PrebuiltModuleFiles(PrebuiltModuleFiles), InputFiles(InputFiles),
59 VisitInputFiles(VisitInputFiles), NewModuleFiles(NewModuleFiles) {}
61 bool needsImportVisitation() const override { return true; }
62 bool needsInputFileVisitation() override { return VisitInputFiles; }
63 bool needsSystemInputFileVisitation() override { return VisitInputFiles; }
65 void visitImport(StringRef ModuleName, StringRef Filename) override {
66 if (PrebuiltModuleFiles.insert({ModuleName.str(), Filename.str()}).second)
67 NewModuleFiles.push_back(Filename.str());
70 bool visitInputFile(StringRef Filename, bool isSystem, bool isOverridden,
71 bool isExplicitModule) override {
72 InputFiles.insert(Filename);
77 PrebuiltModuleFilesT &PrebuiltModuleFiles;
78 llvm::StringSet<> &InputFiles;
80 llvm::SmallVector<std::string> &NewModuleFiles;
83 /// Visit the given prebuilt module and collect all of the modules it
84 /// transitively imports and contributing input files.
85 static void visitPrebuiltModule(StringRef PrebuiltModuleFilename,
87 PrebuiltModuleFilesT &ModuleFiles,
88 llvm::StringSet<> &InputFiles,
89 bool VisitInputFiles) {
90 // List of module files to be processed.
91 llvm::SmallVector<std::string> Worklist{PrebuiltModuleFilename.str()};
92 PrebuiltModuleListener Listener(ModuleFiles, InputFiles, VisitInputFiles,
95 while (!Worklist.empty())
96 ASTReader::readASTFileControlBlock(
97 Worklist.pop_back_val(), CI.getFileManager(),
98 CI.getPCHContainerReader(),
99 /*FindModuleFileExtensions=*/false, Listener,
100 /*ValidateDiagnosticOptions=*/false);
103 /// Transform arbitrary file name into an object-like file name.
104 static std::string makeObjFileName(StringRef FileName) {
105 SmallString<128> ObjFileName(FileName);
106 llvm::sys::path::replace_extension(ObjFileName, "o");
107 return std::string(ObjFileName.str());
110 /// Deduce the dependency target based on the output file and input files.
112 deduceDepTarget(const std::string &OutputFile,
113 const SmallVectorImpl<FrontendInputFile> &InputFiles) {
114 if (OutputFile != "-")
117 if (InputFiles.empty() || !InputFiles.front().isFile())
118 return "clang-scan-deps\\ dependency";
120 return makeObjFileName(InputFiles.front().getFile());
123 /// Sanitize diagnostic options for dependency scan.
124 static void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) {
125 // Don't print 'X warnings and Y errors generated'.
126 DiagOpts.ShowCarets = false;
127 // Don't write out diagnostic file.
128 DiagOpts.DiagnosticSerializationFile.clear();
129 // Don't treat warnings as errors.
130 DiagOpts.Warnings.push_back("no-error");
133 /// A clang tool that runs the preprocessor in a mode that's optimized for
134 /// dependency scanning for the given compiler invocation.
135 class DependencyScanningAction : public tooling::ToolAction {
137 DependencyScanningAction(
138 StringRef WorkingDirectory, DependencyConsumer &Consumer,
139 llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS,
140 ScanningOutputFormat Format, bool OptimizeArgs, bool DisableFree,
141 llvm::Optional<StringRef> ModuleName = None)
142 : WorkingDirectory(WorkingDirectory), Consumer(Consumer),
143 DepFS(std::move(DepFS)), Format(Format), OptimizeArgs(OptimizeArgs),
144 DisableFree(DisableFree), ModuleName(ModuleName) {}
146 bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
147 FileManager *FileMgr,
148 std::shared_ptr<PCHContainerOperations> PCHContainerOps,
149 DiagnosticConsumer *DiagConsumer) override {
150 // Make a deep copy of the original Clang invocation.
151 CompilerInvocation OriginalInvocation(*Invocation);
152 // Restore the value of DisableFree, which may be modified by Tooling.
153 OriginalInvocation.getFrontendOpts().DisableFree = DisableFree;
155 // Create a compiler instance to handle the actual work.
156 CompilerInstance ScanInstance(std::move(PCHContainerOps));
157 ScanInstance.setInvocation(std::move(Invocation));
159 // Create the compiler's actual diagnostics engine.
160 sanitizeDiagOpts(ScanInstance.getDiagnosticOpts());
161 ScanInstance.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false);
162 if (!ScanInstance.hasDiagnostics())
165 ScanInstance.getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath =
168 ScanInstance.getFrontendOpts().GenerateGlobalModuleIndex = false;
169 ScanInstance.getFrontendOpts().UseGlobalModuleIndex = false;
171 FileMgr->getFileSystemOpts().WorkingDir = std::string(WorkingDirectory);
172 ScanInstance.setFileManager(FileMgr);
173 ScanInstance.createSourceManager(*FileMgr);
175 llvm::StringSet<> PrebuiltModulesInputFiles;
176 // Store the list of prebuilt module files into header search options. This
177 // will prevent the implicit build to create duplicate modules and will
178 // force reuse of the existing prebuilt module files instead.
179 if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
181 ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, ScanInstance,
182 ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles,
183 PrebuiltModulesInputFiles, /*VisitInputFiles=*/DepFS != nullptr);
185 // Use the dependency scanning optimized file system if requested to do so.
187 // Support for virtual file system overlays on top of the caching
189 FileMgr->setVirtualFileSystem(createVFSFromCompilerInvocation(
190 ScanInstance.getInvocation(), ScanInstance.getDiagnostics(), DepFS));
192 llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> LocalDepFS =
194 ScanInstance.getPreprocessorOpts().DependencyDirectivesForFile =
195 [LocalDepFS = std::move(LocalDepFS)](FileEntryRef File)
196 -> Optional<ArrayRef<dependency_directives_scan::Directive>> {
197 if (llvm::ErrorOr<EntryRef> Entry =
198 LocalDepFS->getOrCreateFileSystemEntry(File.getName()))
199 return Entry->getDirectiveTokens();
204 // Create the dependency collector that will collect the produced
207 // This also moves the existing dependency output options from the
208 // invocation to the collector. The options in the invocation are reset,
209 // which ensures that the compiler won't create new dependency collectors,
210 // and thus won't write out the extra '.d' files to disk.
211 auto Opts = std::make_unique<DependencyOutputOptions>();
212 std::swap(*Opts, ScanInstance.getInvocation().getDependencyOutputOpts());
213 // We need at least one -MT equivalent for the generator of make dependency
215 if (Opts->Targets.empty())
217 deduceDepTarget(ScanInstance.getFrontendOpts().OutputFile,
218 ScanInstance.getFrontendOpts().Inputs)};
219 Opts->IncludeSystemHeaders = true;
222 case ScanningOutputFormat::Make:
223 ScanInstance.addDependencyCollector(
224 std::make_shared<DependencyConsumerForwarder>(std::move(Opts),
227 case ScanningOutputFormat::Full:
228 ScanInstance.addDependencyCollector(std::make_shared<ModuleDepCollector>(
229 std::move(Opts), ScanInstance, Consumer,
230 std::move(OriginalInvocation), OptimizeArgs));
234 // Consider different header search and diagnostic options to create
235 // different modules. This avoids the unsound aliasing of module PCMs.
237 // TODO: Implement diagnostic bucketing to reduce the impact of strict
239 ScanInstance.getHeaderSearchOpts().ModulesStrictContextHash = true;
241 std::unique_ptr<FrontendAction> Action;
244 Action = std::make_unique<GetDependenciesByModuleNameAction>(*ModuleName);
246 Action = std::make_unique<ReadPCHAndPreprocessAction>();
248 const bool Result = ScanInstance.ExecuteAction(*Action);
250 FileMgr->clearStatCache();
255 StringRef WorkingDirectory;
256 DependencyConsumer &Consumer;
257 llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
258 ScanningOutputFormat Format;
261 llvm::Optional<StringRef> ModuleName;
264 } // end anonymous namespace
266 DependencyScanningWorker::DependencyScanningWorker(
267 DependencyScanningService &Service,
268 llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
269 : Format(Service.getFormat()), OptimizeArgs(Service.canOptimizeArgs()) {
270 PCHContainerOps = std::make_shared<PCHContainerOperations>();
271 PCHContainerOps->registerReader(
272 std::make_unique<ObjectFilePCHContainerReader>());
273 // We don't need to write object files, but the current PCH implementation
274 // requires the writer to be registered as well.
275 PCHContainerOps->registerWriter(
276 std::make_unique<ObjectFilePCHContainerWriter>());
279 llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(std::move(FS));
280 InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
281 OverlayFS->pushOverlay(InMemoryFS);
284 if (Service.getMode() == ScanningMode::DependencyDirectivesScan)
285 DepFS = new DependencyScanningWorkerFilesystem(Service.getSharedCache(),
287 if (Service.canReuseFileManager())
288 Files = new FileManager(FileSystemOptions(), RealFS);
292 runWithDiags(DiagnosticOptions *DiagOpts,
293 llvm::function_ref<bool(DiagnosticConsumer &, DiagnosticOptions &)>
295 sanitizeDiagOpts(*DiagOpts);
297 // Capture the emitted diagnostics and report them to the client
298 // in the case of a failure.
299 std::string DiagnosticOutput;
300 llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput);
301 TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, DiagOpts);
303 if (BodyShouldSucceed(DiagPrinter, *DiagOpts))
304 return llvm::Error::success();
305 return llvm::make_error<llvm::StringError>(DiagnosticsOS.str(),
306 llvm::inconvertibleErrorCode());
309 llvm::Error DependencyScanningWorker::computeDependencies(
310 StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
311 DependencyConsumer &Consumer, llvm::Optional<StringRef> ModuleName) {
312 // Reset what might have been modified in the previous worker invocation.
313 RealFS->setCurrentWorkingDirectory(WorkingDirectory);
315 Files->setVirtualFileSystem(RealFS);
317 llvm::IntrusiveRefCntPtr<FileManager> CurrentFiles =
318 Files ? Files : new FileManager(FileSystemOptions(), RealFS);
320 Optional<std::vector<std::string>> ModifiedCommandLine;
322 ModifiedCommandLine = CommandLine;
323 InMemoryFS->addFile(*ModuleName, 0, llvm::MemoryBuffer::getMemBuffer(""));
324 ModifiedCommandLine->emplace_back(*ModuleName);
327 const std::vector<std::string> &FinalCommandLine =
328 ModifiedCommandLine ? *ModifiedCommandLine : CommandLine;
330 std::vector<const char *> FinalCCommandLine(CommandLine.size(), nullptr);
331 llvm::transform(CommandLine, FinalCCommandLine.begin(),
332 [](const std::string &Str) { return Str.c_str(); });
334 return runWithDiags(CreateAndPopulateDiagOpts(FinalCCommandLine).release(),
335 [&](DiagnosticConsumer &DC, DiagnosticOptions &DiagOpts) {
336 // DisableFree is modified by Tooling for running
337 // in-process; preserve the original value, which is
338 // always true for a driver invocation.
339 bool DisableFree = true;
340 DependencyScanningAction Action(
341 WorkingDirectory, Consumer, DepFS, Format,
342 OptimizeArgs, DisableFree, ModuleName);
343 // Create an invocation that uses the underlying file
344 // system to ensure that any file system requests that
345 // are made by the driver do not go through the
346 // dependency scanning filesystem.
347 ToolInvocation Invocation(FinalCommandLine, &Action,
350 Invocation.setDiagnosticConsumer(&DC);
351 Invocation.setDiagnosticOptions(&DiagOpts);
352 return Invocation.run();