1 //===-- ClangModulesDeclVendor.cpp ------------------------------*- C++ -*-===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
12 #include "clang/Basic/TargetInfo.h"
13 #include "clang/Frontend/CompilerInstance.h"
14 #include "clang/Frontend/FrontendActions.h"
15 #include "clang/Lex/Preprocessor.h"
16 #include "clang/Lex/PreprocessorOptions.h"
17 #include "clang/Parse/Parser.h"
18 #include "clang/Sema/Lookup.h"
19 #include "clang/Serialization/ASTReader.h"
20 #include "llvm/Support/FileSystem.h"
21 #include "llvm/Support/Path.h"
22 #include "llvm/Support/Threading.h"
24 #include "ClangHost.h"
25 #include "ClangModulesDeclVendor.h"
27 #include "lldb/Core/ModuleList.h"
28 #include "lldb/Host/Host.h"
29 #include "lldb/Host/HostInfo.h"
30 #include "lldb/Symbol/CompileUnit.h"
31 #include "lldb/Target/Target.h"
32 #include "lldb/Utility/FileSpec.h"
33 #include "lldb/Utility/LLDBAssert.h"
34 #include "lldb/Utility/Log.h"
35 #include "lldb/Utility/StreamString.h"
37 using namespace lldb_private;
40 // Any Clang compiler requires a consumer for diagnostics. This one stores
41 // them as strings so we can provide them to the user in case a module failed
43 class StoringDiagnosticConsumer : public clang::DiagnosticConsumer {
45 StoringDiagnosticConsumer();
47 void HandleDiagnostic(clang::DiagnosticsEngine::Level DiagLevel,
48 const clang::Diagnostic &info) override;
50 void ClearDiagnostics();
52 void DumpDiagnostics(Stream &error_stream);
55 typedef std::pair<clang::DiagnosticsEngine::Level, std::string>
57 std::vector<IDAndDiagnostic> m_diagnostics;
61 // The private implementation of our ClangModulesDeclVendor. Contains all the
62 // Clang state required to load modules.
63 class ClangModulesDeclVendorImpl : public ClangModulesDeclVendor {
65 ClangModulesDeclVendorImpl(
66 llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> diagnostics_engine,
67 std::shared_ptr<clang::CompilerInvocation> compiler_invocation,
68 std::unique_ptr<clang::CompilerInstance> compiler_instance,
69 std::unique_ptr<clang::Parser> parser);
71 ~ClangModulesDeclVendorImpl() override = default;
73 bool AddModule(ModulePath &path, ModuleVector *exported_modules,
74 Stream &error_stream) override;
76 bool AddModulesForCompileUnit(CompileUnit &cu, ModuleVector &exported_modules,
77 Stream &error_stream) override;
79 uint32_t FindDecls(const ConstString &name, bool append, uint32_t max_matches,
80 std::vector<clang::NamedDecl *> &decls) override;
82 void ForEachMacro(const ModuleVector &modules,
83 std::function<bool(const std::string &)> handler) override;
85 clang::ExternalASTMerger::ImporterSource GetImporterSource() override;
88 ReportModuleExportsHelper(std::set<ClangModulesDeclVendor::ModuleID> &exports,
89 clang::Module *module);
91 void ReportModuleExports(ModuleVector &exports, clang::Module *module);
93 clang::ModuleLoadResult DoGetModule(clang::ModuleIdPath path,
96 bool m_enabled = false;
98 llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> m_diagnostics_engine;
99 std::shared_ptr<clang::CompilerInvocation> m_compiler_invocation;
100 std::unique_ptr<clang::CompilerInstance> m_compiler_instance;
101 std::unique_ptr<clang::Parser> m_parser;
102 size_t m_source_location_index =
103 0; // used to give name components fake SourceLocations
105 typedef std::vector<ConstString> ImportedModule;
106 typedef std::map<ImportedModule, clang::Module *> ImportedModuleMap;
107 typedef std::set<ModuleID> ImportedModuleSet;
108 ImportedModuleMap m_imported_modules;
109 ImportedModuleSet m_user_imported_modules;
110 const clang::ExternalASTMerger::OriginMap m_origin_map;
112 } // anonymous namespace
114 StoringDiagnosticConsumer::StoringDiagnosticConsumer() {
115 m_log = lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_EXPRESSIONS);
118 void StoringDiagnosticConsumer::HandleDiagnostic(
119 clang::DiagnosticsEngine::Level DiagLevel, const clang::Diagnostic &info) {
120 llvm::SmallVector<char, 256> diagnostic_string;
122 info.FormatDiagnostic(diagnostic_string);
124 m_diagnostics.push_back(
125 IDAndDiagnostic(DiagLevel, std::string(diagnostic_string.data(),
126 diagnostic_string.size())));
129 void StoringDiagnosticConsumer::ClearDiagnostics() { m_diagnostics.clear(); }
131 void StoringDiagnosticConsumer::DumpDiagnostics(Stream &error_stream) {
132 for (IDAndDiagnostic &diag : m_diagnostics) {
133 switch (diag.first) {
135 error_stream.PutCString(diag.second);
136 error_stream.PutChar('\n');
138 case clang::DiagnosticsEngine::Level::Ignored:
144 ClangModulesDeclVendor::ClangModulesDeclVendor() {}
146 ClangModulesDeclVendor::~ClangModulesDeclVendor() {}
148 ClangModulesDeclVendorImpl::ClangModulesDeclVendorImpl(
149 llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> diagnostics_engine,
150 std::shared_ptr<clang::CompilerInvocation> compiler_invocation,
151 std::unique_ptr<clang::CompilerInstance> compiler_instance,
152 std::unique_ptr<clang::Parser> parser)
153 : m_diagnostics_engine(std::move(diagnostics_engine)),
154 m_compiler_invocation(std::move(compiler_invocation)),
155 m_compiler_instance(std::move(compiler_instance)),
156 m_parser(std::move(parser)), m_origin_map() {}
158 void ClangModulesDeclVendorImpl::ReportModuleExportsHelper(
159 std::set<ClangModulesDeclVendor::ModuleID> &exports,
160 clang::Module *module) {
161 if (exports.count(reinterpret_cast<ClangModulesDeclVendor::ModuleID>(module)))
164 exports.insert(reinterpret_cast<ClangModulesDeclVendor::ModuleID>(module));
166 llvm::SmallVector<clang::Module *, 2> sub_exports;
168 module->getExportedModules(sub_exports);
170 for (clang::Module *module : sub_exports) {
171 ReportModuleExportsHelper(exports, module);
175 void ClangModulesDeclVendorImpl::ReportModuleExports(
176 ClangModulesDeclVendor::ModuleVector &exports, clang::Module *module) {
177 std::set<ClangModulesDeclVendor::ModuleID> exports_set;
179 ReportModuleExportsHelper(exports_set, module);
181 for (ModuleID module : exports_set) {
182 exports.push_back(module);
186 bool ClangModulesDeclVendorImpl::AddModule(ModulePath &path,
187 ModuleVector *exported_modules,
188 Stream &error_stream) {
191 if (m_compiler_instance->hadModuleLoaderFatalFailure()) {
192 error_stream.PutCString("error: Couldn't load a module because the module "
193 "loader is in a fatal state.\n");
197 // Check if we've already imported this module.
199 std::vector<ConstString> imported_module;
201 for (ConstString path_component : path) {
202 imported_module.push_back(path_component);
206 ImportedModuleMap::iterator mi = m_imported_modules.find(imported_module);
208 if (mi != m_imported_modules.end()) {
209 if (exported_modules) {
210 ReportModuleExports(*exported_modules, mi->second);
216 if (!m_compiler_instance->getPreprocessor()
217 .getHeaderSearchInfo()
218 .lookupModule(path[0].GetStringRef())) {
219 error_stream.Printf("error: Header search couldn't locate module %s\n",
220 path[0].AsCString());
224 llvm::SmallVector<std::pair<clang::IdentifierInfo *, clang::SourceLocation>,
229 clang::SourceManager &source_manager =
230 m_compiler_instance->getASTContext().getSourceManager();
232 for (ConstString path_component : path) {
233 clang_path.push_back(std::make_pair(
234 &m_compiler_instance->getASTContext().Idents.get(
235 path_component.GetStringRef()),
236 source_manager.getLocForStartOfFile(source_manager.getMainFileID())
237 .getLocWithOffset(m_source_location_index++)));
241 StoringDiagnosticConsumer *diagnostic_consumer =
242 static_cast<StoringDiagnosticConsumer *>(
243 m_compiler_instance->getDiagnostics().getClient());
245 diagnostic_consumer->ClearDiagnostics();
247 clang::Module *top_level_module = DoGetModule(clang_path.front(), false);
249 if (!top_level_module) {
250 diagnostic_consumer->DumpDiagnostics(error_stream);
251 error_stream.Printf("error: Couldn't load top-level module %s\n",
252 path[0].AsCString());
256 clang::Module *submodule = top_level_module;
258 for (size_t ci = 1; ci < path.size(); ++ci) {
259 llvm::StringRef component = path[ci].GetStringRef();
260 submodule = submodule->findSubmodule(component.str());
262 diagnostic_consumer->DumpDiagnostics(error_stream);
263 error_stream.Printf("error: Couldn't load submodule %s\n",
264 component.str().c_str());
269 clang::Module *requested_module = DoGetModule(clang_path, true);
271 if (requested_module != nullptr) {
272 if (exported_modules) {
273 ReportModuleExports(*exported_modules, requested_module);
276 m_imported_modules[imported_module] = requested_module;
286 bool ClangModulesDeclVendor::LanguageSupportsClangModules(
287 lldb::LanguageType language) {
291 // C++ and friends to be added
292 case lldb::LanguageType::eLanguageTypeC:
293 case lldb::LanguageType::eLanguageTypeC11:
294 case lldb::LanguageType::eLanguageTypeC89:
295 case lldb::LanguageType::eLanguageTypeC99:
296 case lldb::LanguageType::eLanguageTypeObjC:
301 bool ClangModulesDeclVendorImpl::AddModulesForCompileUnit(
302 CompileUnit &cu, ClangModulesDeclVendor::ModuleVector &exported_modules,
303 Stream &error_stream) {
304 if (LanguageSupportsClangModules(cu.GetLanguage())) {
305 std::vector<ConstString> imported_modules = cu.GetImportedModules();
307 for (ConstString imported_module : imported_modules) {
308 std::vector<ConstString> path;
310 path.push_back(imported_module);
312 if (!AddModule(path, &exported_modules, error_stream)) {
323 // ClangImporter::lookupValue
326 ClangModulesDeclVendorImpl::FindDecls(const ConstString &name, bool append,
327 uint32_t max_matches,
328 std::vector<clang::NamedDecl *> &decls) {
336 clang::IdentifierInfo &ident =
337 m_compiler_instance->getASTContext().Idents.get(name.GetStringRef());
339 clang::LookupResult lookup_result(
340 m_compiler_instance->getSema(), clang::DeclarationName(&ident),
341 clang::SourceLocation(), clang::Sema::LookupOrdinaryName);
343 m_compiler_instance->getSema().LookupName(
345 m_compiler_instance->getSema().getScopeForContext(
346 m_compiler_instance->getASTContext().getTranslationUnitDecl()));
348 uint32_t num_matches = 0;
350 for (clang::NamedDecl *named_decl : lookup_result) {
351 if (num_matches >= max_matches)
354 decls.push_back(named_decl);
361 void ClangModulesDeclVendorImpl::ForEachMacro(
362 const ClangModulesDeclVendor::ModuleVector &modules,
363 std::function<bool(const std::string &)> handler) {
368 typedef std::map<ModuleID, ssize_t> ModulePriorityMap;
369 ModulePriorityMap module_priorities;
371 ssize_t priority = 0;
373 for (ModuleID module : modules) {
374 module_priorities[module] = priority++;
377 if (m_compiler_instance->getPreprocessor().getExternalSource()) {
378 m_compiler_instance->getPreprocessor()
380 ->ReadDefinedMacros();
383 for (clang::Preprocessor::macro_iterator
384 mi = m_compiler_instance->getPreprocessor().macro_begin(),
385 me = m_compiler_instance->getPreprocessor().macro_end();
387 const clang::IdentifierInfo *ii = nullptr;
390 if (clang::IdentifierInfoLookup *lookup =
391 m_compiler_instance->getPreprocessor()
392 .getIdentifierTable()
393 .getExternalIdentifierLookup()) {
394 lookup->get(mi->first->getName());
401 ssize_t found_priority = -1;
402 clang::MacroInfo *macro_info = nullptr;
404 for (clang::ModuleMacro *module_macro :
405 m_compiler_instance->getPreprocessor().getLeafModuleMacros(ii)) {
406 clang::Module *module = module_macro->getOwningModule();
409 ModulePriorityMap::iterator pi =
410 module_priorities.find(reinterpret_cast<ModuleID>(module));
412 if (pi != module_priorities.end() && pi->second > found_priority) {
413 macro_info = module_macro->getMacroInfo();
414 found_priority = pi->second;
418 clang::Module *top_level_module = module->getTopLevelModule();
420 if (top_level_module != module) {
421 ModulePriorityMap::iterator pi = module_priorities.find(
422 reinterpret_cast<ModuleID>(top_level_module));
424 if ((pi != module_priorities.end()) && pi->second > found_priority) {
425 macro_info = module_macro->getMacroInfo();
426 found_priority = pi->second;
432 std::string macro_expansion = "#define ";
433 macro_expansion.append(mi->first->getName().str());
436 if (macro_info->isFunctionLike()) {
437 macro_expansion.append("(");
439 bool first_arg = true;
441 for (auto pi = macro_info->param_begin(),
442 pe = macro_info->param_end();
445 macro_expansion.append(", ");
450 macro_expansion.append((*pi)->getName().str());
453 if (macro_info->isC99Varargs()) {
455 macro_expansion.append("...");
457 macro_expansion.append(", ...");
459 } else if (macro_info->isGNUVarargs()) {
460 macro_expansion.append("...");
463 macro_expansion.append(")");
466 macro_expansion.append(" ");
468 bool first_token = true;
470 for (clang::MacroInfo::tokens_iterator ti = macro_info->tokens_begin(),
471 te = macro_info->tokens_end();
474 macro_expansion.append(" ");
479 if (ti->isLiteral()) {
480 if (const char *literal_data = ti->getLiteralData()) {
481 std::string token_str(literal_data, ti->getLength());
482 macro_expansion.append(token_str);
484 bool invalid = false;
485 const char *literal_source =
486 m_compiler_instance->getSourceManager().getCharacterData(
487 ti->getLocation(), &invalid);
490 lldbassert(0 && "Unhandled token kind");
491 macro_expansion.append("<unknown literal value>");
493 macro_expansion.append(
494 std::string(literal_source, ti->getLength()));
497 } else if (const char *punctuator_spelling =
498 clang::tok::getPunctuatorSpelling(ti->getKind())) {
499 macro_expansion.append(punctuator_spelling);
500 } else if (const char *keyword_spelling =
501 clang::tok::getKeywordSpelling(ti->getKind())) {
502 macro_expansion.append(keyword_spelling);
504 switch (ti->getKind()) {
505 case clang::tok::TokenKind::identifier:
506 macro_expansion.append(ti->getIdentifierInfo()->getName().str());
508 case clang::tok::TokenKind::raw_identifier:
509 macro_expansion.append(ti->getRawIdentifier().str());
512 macro_expansion.append(ti->getName());
518 if (handler(macro_expansion)) {
526 clang::ModuleLoadResult
527 ClangModulesDeclVendorImpl::DoGetModule(clang::ModuleIdPath path,
529 clang::Module::NameVisibilityKind visibility =
530 make_visible ? clang::Module::AllVisible : clang::Module::Hidden;
532 const bool is_inclusion_directive = false;
534 return m_compiler_instance->loadModule(path.front().second, path, visibility,
535 is_inclusion_directive);
538 clang::ExternalASTMerger::ImporterSource
539 ClangModulesDeclVendorImpl::GetImporterSource() {
540 return {m_compiler_instance->getASTContext(),
541 m_compiler_instance->getFileManager(), m_origin_map};
544 static const char *ModuleImportBufferName = "LLDBModulesMemoryBuffer";
546 lldb_private::ClangModulesDeclVendor *
547 ClangModulesDeclVendor::Create(Target &target) {
548 // FIXME we should insure programmatically that the expression parser's
549 // compiler and the modules runtime's
550 // compiler are both initialized in the same way – preferably by the same
553 if (!target.GetPlatform()->SupportsModules())
556 const ArchSpec &arch = target.GetArchitecture();
558 std::vector<std::string> compiler_invocation_arguments = {
561 "-fimplicit-module-maps",
566 arch.GetTriple().str(),
567 "-fmodules-validate-system-headers",
568 "-Werror=non-modular-include-in-framework-module"};
570 target.GetPlatform()->AddClangModuleCompilationOptions(
571 &target, compiler_invocation_arguments);
573 compiler_invocation_arguments.push_back(ModuleImportBufferName);
575 // Add additional search paths with { "-I", path } or { "-F", path } here.
578 llvm::SmallString<128> path;
579 auto props = ModuleList::GetGlobalModuleListProperties();
580 props.GetClangModulesCachePath().GetPath(path);
581 std::string module_cache_argument("-fmodules-cache-path=");
582 module_cache_argument.append(path.str());
583 compiler_invocation_arguments.push_back(module_cache_argument);
586 FileSpecList &module_search_paths = target.GetClangModuleSearchPaths();
588 for (size_t spi = 0, spe = module_search_paths.GetSize(); spi < spe; ++spi) {
589 const FileSpec &search_path = module_search_paths.GetFileSpecAtIndex(spi);
591 std::string search_path_argument = "-I";
592 search_path_argument.append(search_path.GetPath());
594 compiler_invocation_arguments.push_back(search_path_argument);
598 FileSpec clang_resource_dir = GetClangResourceDir();
600 if (FileSystem::Instance().IsDirectory(clang_resource_dir.GetPath())) {
601 compiler_invocation_arguments.push_back("-resource-dir");
602 compiler_invocation_arguments.push_back(clang_resource_dir.GetPath());
606 llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> diagnostics_engine =
607 clang::CompilerInstance::createDiagnostics(new clang::DiagnosticOptions,
608 new StoringDiagnosticConsumer);
610 std::vector<const char *> compiler_invocation_argument_cstrs;
611 compiler_invocation_argument_cstrs.reserve(
612 compiler_invocation_arguments.size());
613 for (const std::string &arg : compiler_invocation_arguments) {
614 compiler_invocation_argument_cstrs.push_back(arg.c_str());
617 std::shared_ptr<clang::CompilerInvocation> invocation =
618 clang::createInvocationFromCommandLine(compiler_invocation_argument_cstrs,
624 std::unique_ptr<llvm::MemoryBuffer> source_buffer =
625 llvm::MemoryBuffer::getMemBuffer(
626 "extern int __lldb __attribute__((unavailable));",
627 ModuleImportBufferName);
629 invocation->getPreprocessorOpts().addRemappedFile(ModuleImportBufferName,
630 source_buffer.release());
632 std::unique_ptr<clang::CompilerInstance> instance(
633 new clang::CompilerInstance);
635 instance->setDiagnostics(diagnostics_engine.get());
636 instance->setInvocation(invocation);
638 std::unique_ptr<clang::FrontendAction> action(new clang::SyntaxOnlyAction);
640 instance->setTarget(clang::TargetInfo::CreateTargetInfo(
641 *diagnostics_engine, instance->getInvocation().TargetOpts));
643 if (!instance->hasTarget())
646 instance->getTarget().adjust(instance->getLangOpts());
648 if (!action->BeginSourceFile(*instance,
649 instance->getFrontendOpts().Inputs[0]))
652 instance->getPreprocessor().enableIncrementalProcessing();
654 instance->createModuleManager();
656 instance->createSema(action->getTranslationUnitKind(), nullptr);
658 const bool skipFunctionBodies = false;
659 std::unique_ptr<clang::Parser> parser(new clang::Parser(
660 instance->getPreprocessor(), instance->getSema(), skipFunctionBodies));
662 instance->getPreprocessor().EnterMainSourceFile();
663 parser->Initialize();
665 clang::Parser::DeclGroupPtrTy parsed;
667 while (!parser->ParseTopLevelDecl(parsed))
670 return new ClangModulesDeclVendorImpl(std::move(diagnostics_engine),
671 std::move(invocation),
672 std::move(instance), std::move(parser));