1 //===--- Preprocess.cpp - C Language Family Preprocessor Implementation ---===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file implements the Preprocessor interface.
12 //===----------------------------------------------------------------------===//
14 // Options to support:
15 // -H - Print the name of each header file used.
16 // -d[DNI] - Dump various things.
17 // -fworking-directory - #line's with preprocessor's working dir.
19 // -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD
24 // "Multiple include guards may be useful for:\n"
26 //===----------------------------------------------------------------------===//
28 #include "clang/Lex/Preprocessor.h"
29 #include "clang/Basic/FileManager.h"
30 #include "clang/Basic/FileSystemStatCache.h"
31 #include "clang/Basic/SourceManager.h"
32 #include "clang/Basic/TargetInfo.h"
33 #include "clang/Lex/CodeCompletionHandler.h"
34 #include "clang/Lex/ExternalPreprocessorSource.h"
35 #include "clang/Lex/HeaderSearch.h"
36 #include "clang/Lex/LexDiagnostic.h"
37 #include "clang/Lex/LiteralSupport.h"
38 #include "clang/Lex/MacroArgs.h"
39 #include "clang/Lex/MacroInfo.h"
40 #include "clang/Lex/ModuleLoader.h"
41 #include "clang/Lex/PTHManager.h"
42 #include "clang/Lex/Pragma.h"
43 #include "clang/Lex/PreprocessingRecord.h"
44 #include "clang/Lex/PreprocessorOptions.h"
45 #include "clang/Lex/ScratchBuffer.h"
46 #include "llvm/ADT/APInt.h"
47 #include "llvm/ADT/DenseMap.h"
48 #include "llvm/ADT/SmallString.h"
49 #include "llvm/ADT/SmallVector.h"
50 #include "llvm/ADT/STLExtras.h"
51 #include "llvm/ADT/StringRef.h"
52 #include "llvm/ADT/StringSwitch.h"
53 #include "llvm/Support/Capacity.h"
54 #include "llvm/Support/ErrorHandling.h"
55 #include "llvm/Support/MemoryBuffer.h"
56 #include "llvm/Support/raw_ostream.h"
64 using namespace clang;
66 LLVM_INSTANTIATE_REGISTRY(PragmaHandlerRegistry)
68 //===----------------------------------------------------------------------===//
69 ExternalPreprocessorSource::~ExternalPreprocessorSource() { }
71 Preprocessor::Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
72 DiagnosticsEngine &diags, LangOptions &opts,
73 SourceManager &SM, MemoryBufferCache &PCMCache,
74 HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
75 IdentifierInfoLookup *IILookup, bool OwnsHeaders,
76 TranslationUnitKind TUKind)
77 : PPOpts(std::move(PPOpts)), Diags(&diags), LangOpts(opts), Target(nullptr),
78 AuxTarget(nullptr), FileMgr(Headers.getFileMgr()), SourceMgr(SM),
79 PCMCache(PCMCache), ScratchBuf(new ScratchBuffer(SourceMgr)),
80 HeaderInfo(Headers), TheModuleLoader(TheModuleLoader),
81 ExternalSource(nullptr), Identifiers(opts, IILookup),
82 PragmaHandlers(new PragmaNamespace(StringRef())),
83 IncrementalProcessing(false), TUKind(TUKind), CodeComplete(nullptr),
84 CodeCompletionFile(nullptr), CodeCompletionOffset(0),
85 LastTokenWasAt(false), ModuleImportExpectsIdentifier(false),
86 CodeCompletionReached(false), CodeCompletionII(nullptr),
87 MainFileDir(nullptr), SkipMainFilePreamble(0, true), CurPPLexer(nullptr),
88 CurDirLookup(nullptr), CurLexerKind(CLK_Lexer),
89 CurLexerSubmodule(nullptr), Callbacks(nullptr),
90 CurSubmoduleState(&NullSubmoduleState), MacroArgCache(nullptr),
91 Record(nullptr), MIChainHead(nullptr) {
92 OwnsHeaderSearch = OwnsHeaders;
94 CounterValue = 0; // __COUNTER__ starts at 0.
97 NumDirectives = NumDefined = NumUndefined = NumPragma = 0;
98 NumIf = NumElse = NumEndif = 0;
99 NumEnteredSourceFiles = 0;
100 NumMacroExpanded = NumFnMacroExpanded = NumBuiltinMacroExpanded = 0;
101 NumFastMacroExpanded = NumTokenPaste = NumFastTokenPaste = 0;
102 MaxIncludeStackDepth = 0;
105 // Default to discarding comments.
106 KeepComments = false;
107 KeepMacroComments = false;
108 SuppressIncludeNotFoundError = false;
110 // Macro expansion is enabled.
111 DisableMacroExpansion = false;
112 MacroExpansionInDirectivesOverride = false;
114 InMacroArgPreExpansion = false;
115 NumCachedTokenLexers = 0;
116 PragmasEnabled = true;
117 ParsingIfOrElifDirective = false;
118 PreprocessedOutput = false;
122 // We haven't read anything from the external source.
123 ReadMacrosFromExternalSource = false;
125 // "Poison" __VA_ARGS__, which can only appear in the expansion of a macro.
126 // This gets unpoisoned where it is allowed.
127 (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned();
128 SetPoisonReason(Ident__VA_ARGS__,diag::ext_pp_bad_vaargs_use);
130 // Initialize the pragma handlers.
131 RegisterBuiltinPragmas();
133 // Initialize builtin macros like __LINE__ and friends.
134 RegisterBuiltinMacros();
136 if(LangOpts.Borland) {
137 Ident__exception_info = getIdentifierInfo("_exception_info");
138 Ident___exception_info = getIdentifierInfo("__exception_info");
139 Ident_GetExceptionInfo = getIdentifierInfo("GetExceptionInformation");
140 Ident__exception_code = getIdentifierInfo("_exception_code");
141 Ident___exception_code = getIdentifierInfo("__exception_code");
142 Ident_GetExceptionCode = getIdentifierInfo("GetExceptionCode");
143 Ident__abnormal_termination = getIdentifierInfo("_abnormal_termination");
144 Ident___abnormal_termination = getIdentifierInfo("__abnormal_termination");
145 Ident_AbnormalTermination = getIdentifierInfo("AbnormalTermination");
147 Ident__exception_info = Ident__exception_code = nullptr;
148 Ident__abnormal_termination = Ident___exception_info = nullptr;
149 Ident___exception_code = Ident___abnormal_termination = nullptr;
150 Ident_GetExceptionInfo = Ident_GetExceptionCode = nullptr;
151 Ident_AbnormalTermination = nullptr;
154 if (this->PPOpts->GeneratePreamble)
155 PreambleConditionalStack.startRecording();
158 Preprocessor::~Preprocessor() {
159 assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!");
161 IncludeMacroStack.clear();
163 // Destroy any macro definitions.
164 while (MacroInfoChain *I = MIChainHead) {
165 MIChainHead = I->Next;
166 I->~MacroInfoChain();
169 // Free any cached macro expanders.
170 // This populates MacroArgCache, so all TokenLexers need to be destroyed
171 // before the code below that frees up the MacroArgCache list.
172 std::fill(TokenLexerCache, TokenLexerCache + NumCachedTokenLexers, nullptr);
173 CurTokenLexer.reset();
175 // Free any cached MacroArgs.
176 for (MacroArgs *ArgList = MacroArgCache; ArgList;)
177 ArgList = ArgList->deallocate();
179 // Delete the header search info, if we own it.
180 if (OwnsHeaderSearch)
184 void Preprocessor::Initialize(const TargetInfo &Target,
185 const TargetInfo *AuxTarget) {
186 assert((!this->Target || this->Target == &Target) &&
187 "Invalid override of target information");
188 this->Target = &Target;
190 assert((!this->AuxTarget || this->AuxTarget == AuxTarget) &&
191 "Invalid override of aux target information.");
192 this->AuxTarget = AuxTarget;
194 // Initialize information about built-ins.
195 BuiltinInfo.InitializeTarget(Target, AuxTarget);
196 HeaderInfo.setTarget(Target);
199 void Preprocessor::InitializeForModelFile() {
200 NumEnteredSourceFiles = 0;
203 PragmaHandlersBackup = std::move(PragmaHandlers);
204 PragmaHandlers = llvm::make_unique<PragmaNamespace>(StringRef());
205 RegisterBuiltinPragmas();
207 // Reset PredefinesFileID
208 PredefinesFileID = FileID();
211 void Preprocessor::FinalizeForModelFile() {
212 NumEnteredSourceFiles = 1;
214 PragmaHandlers = std::move(PragmaHandlersBackup);
217 void Preprocessor::setPTHManager(PTHManager* pm) {
219 FileMgr.addStatCache(PTH->createStatCache());
222 void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const {
223 llvm::errs() << tok::getTokenName(Tok.getKind()) << " '"
224 << getSpelling(Tok) << "'";
226 if (!DumpFlags) return;
228 llvm::errs() << "\t";
229 if (Tok.isAtStartOfLine())
230 llvm::errs() << " [StartOfLine]";
231 if (Tok.hasLeadingSpace())
232 llvm::errs() << " [LeadingSpace]";
233 if (Tok.isExpandDisabled())
234 llvm::errs() << " [ExpandDisabled]";
235 if (Tok.needsCleaning()) {
236 const char *Start = SourceMgr.getCharacterData(Tok.getLocation());
237 llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength())
241 llvm::errs() << "\tLoc=<";
242 DumpLocation(Tok.getLocation());
246 void Preprocessor::DumpLocation(SourceLocation Loc) const {
250 void Preprocessor::DumpMacro(const MacroInfo &MI) const {
251 llvm::errs() << "MACRO: ";
252 for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) {
253 DumpToken(MI.getReplacementToken(i));
256 llvm::errs() << "\n";
259 void Preprocessor::PrintStats() {
260 llvm::errs() << "\n*** Preprocessor Stats:\n";
261 llvm::errs() << NumDirectives << " directives found:\n";
262 llvm::errs() << " " << NumDefined << " #define.\n";
263 llvm::errs() << " " << NumUndefined << " #undef.\n";
264 llvm::errs() << " #include/#include_next/#import:\n";
265 llvm::errs() << " " << NumEnteredSourceFiles << " source files entered.\n";
266 llvm::errs() << " " << MaxIncludeStackDepth << " max include stack depth\n";
267 llvm::errs() << " " << NumIf << " #if/#ifndef/#ifdef.\n";
268 llvm::errs() << " " << NumElse << " #else/#elif.\n";
269 llvm::errs() << " " << NumEndif << " #endif.\n";
270 llvm::errs() << " " << NumPragma << " #pragma.\n";
271 llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n";
273 llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/"
274 << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, "
275 << NumFastMacroExpanded << " on the fast path.\n";
276 llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
277 << " token paste (##) operations performed, "
278 << NumFastTokenPaste << " on the fast path.\n";
280 llvm::errs() << "\nPreprocessor Memory: " << getTotalMemory() << "B total";
282 llvm::errs() << "\n BumpPtr: " << BP.getTotalMemory();
283 llvm::errs() << "\n Macro Expanded Tokens: "
284 << llvm::capacity_in_bytes(MacroExpandedTokens);
285 llvm::errs() << "\n Predefines Buffer: " << Predefines.capacity();
286 // FIXME: List information for all submodules.
287 llvm::errs() << "\n Macros: "
288 << llvm::capacity_in_bytes(CurSubmoduleState->Macros);
289 llvm::errs() << "\n #pragma push_macro Info: "
290 << llvm::capacity_in_bytes(PragmaPushMacroInfo);
291 llvm::errs() << "\n Poison Reasons: "
292 << llvm::capacity_in_bytes(PoisonReasons);
293 llvm::errs() << "\n Comment Handlers: "
294 << llvm::capacity_in_bytes(CommentHandlers) << "\n";
297 Preprocessor::macro_iterator
298 Preprocessor::macro_begin(bool IncludeExternalMacros) const {
299 if (IncludeExternalMacros && ExternalSource &&
300 !ReadMacrosFromExternalSource) {
301 ReadMacrosFromExternalSource = true;
302 ExternalSource->ReadDefinedMacros();
305 // Make sure we cover all macros in visible modules.
306 for (const ModuleMacro &Macro : ModuleMacros)
307 CurSubmoduleState->Macros.insert(std::make_pair(Macro.II, MacroState()));
309 return CurSubmoduleState->Macros.begin();
312 size_t Preprocessor::getTotalMemory() const {
313 return BP.getTotalMemory()
314 + llvm::capacity_in_bytes(MacroExpandedTokens)
315 + Predefines.capacity() /* Predefines buffer. */
316 // FIXME: Include sizes from all submodules, and include MacroInfo sizes,
318 + llvm::capacity_in_bytes(CurSubmoduleState->Macros)
319 + llvm::capacity_in_bytes(PragmaPushMacroInfo)
320 + llvm::capacity_in_bytes(PoisonReasons)
321 + llvm::capacity_in_bytes(CommentHandlers);
324 Preprocessor::macro_iterator
325 Preprocessor::macro_end(bool IncludeExternalMacros) const {
326 if (IncludeExternalMacros && ExternalSource &&
327 !ReadMacrosFromExternalSource) {
328 ReadMacrosFromExternalSource = true;
329 ExternalSource->ReadDefinedMacros();
332 return CurSubmoduleState->Macros.end();
335 /// \brief Compares macro tokens with a specified token value sequence.
336 static bool MacroDefinitionEquals(const MacroInfo *MI,
337 ArrayRef<TokenValue> Tokens) {
338 return Tokens.size() == MI->getNumTokens() &&
339 std::equal(Tokens.begin(), Tokens.end(), MI->tokens_begin());
342 StringRef Preprocessor::getLastMacroWithSpelling(
344 ArrayRef<TokenValue> Tokens) const {
345 SourceLocation BestLocation;
346 StringRef BestSpelling;
347 for (Preprocessor::macro_iterator I = macro_begin(), E = macro_end();
349 const MacroDirective::DefInfo
350 Def = I->second.findDirectiveAtLoc(Loc, SourceMgr);
351 if (!Def || !Def.getMacroInfo())
353 if (!Def.getMacroInfo()->isObjectLike())
355 if (!MacroDefinitionEquals(Def.getMacroInfo(), Tokens))
357 SourceLocation Location = Def.getLocation();
358 // Choose the macro defined latest.
359 if (BestLocation.isInvalid() ||
360 (Location.isValid() &&
361 SourceMgr.isBeforeInTranslationUnit(BestLocation, Location))) {
362 BestLocation = Location;
363 BestSpelling = I->first->getName();
369 void Preprocessor::recomputeCurLexerKind() {
371 CurLexerKind = CLK_Lexer;
372 else if (CurPTHLexer)
373 CurLexerKind = CLK_PTHLexer;
374 else if (CurTokenLexer)
375 CurLexerKind = CLK_TokenLexer;
377 CurLexerKind = CLK_CachingLexer;
380 bool Preprocessor::SetCodeCompletionPoint(const FileEntry *File,
381 unsigned CompleteLine,
382 unsigned CompleteColumn) {
384 assert(CompleteLine && CompleteColumn && "Starts from 1:1");
385 assert(!CodeCompletionFile && "Already set");
387 using llvm::MemoryBuffer;
389 // Load the actual file's contents.
390 bool Invalid = false;
391 const MemoryBuffer *Buffer = SourceMgr.getMemoryBufferForFile(File, &Invalid);
395 // Find the byte position of the truncation point.
396 const char *Position = Buffer->getBufferStart();
397 for (unsigned Line = 1; Line < CompleteLine; ++Line) {
398 for (; *Position; ++Position) {
399 if (*Position != '\r' && *Position != '\n')
402 // Eat \r\n or \n\r as a single line.
403 if ((Position[1] == '\r' || Position[1] == '\n') &&
404 Position[0] != Position[1])
411 Position += CompleteColumn - 1;
413 // If pointing inside the preamble, adjust the position at the beginning of
414 // the file after the preamble.
415 if (SkipMainFilePreamble.first &&
416 SourceMgr.getFileEntryForID(SourceMgr.getMainFileID()) == File) {
417 if (Position - Buffer->getBufferStart() < SkipMainFilePreamble.first)
418 Position = Buffer->getBufferStart() + SkipMainFilePreamble.first;
421 if (Position > Buffer->getBufferEnd())
422 Position = Buffer->getBufferEnd();
424 CodeCompletionFile = File;
425 CodeCompletionOffset = Position - Buffer->getBufferStart();
427 std::unique_ptr<MemoryBuffer> NewBuffer =
428 MemoryBuffer::getNewUninitMemBuffer(Buffer->getBufferSize() + 1,
429 Buffer->getBufferIdentifier());
430 char *NewBuf = const_cast<char*>(NewBuffer->getBufferStart());
431 char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf);
433 std::copy(Position, Buffer->getBufferEnd(), NewPos+1);
434 SourceMgr.overrideFileContents(File, std::move(NewBuffer));
439 void Preprocessor::CodeCompleteNaturalLanguage() {
441 CodeComplete->CodeCompleteNaturalLanguage();
442 setCodeCompletionReached();
445 /// getSpelling - This method is used to get the spelling of a token into a
446 /// SmallVector. Note that the returned StringRef may not point to the
447 /// supplied buffer if a copy can be avoided.
448 StringRef Preprocessor::getSpelling(const Token &Tok,
449 SmallVectorImpl<char> &Buffer,
450 bool *Invalid) const {
451 // NOTE: this has to be checked *before* testing for an IdentifierInfo.
452 if (Tok.isNot(tok::raw_identifier) && !Tok.hasUCN()) {
453 // Try the fast path.
454 if (const IdentifierInfo *II = Tok.getIdentifierInfo())
455 return II->getName();
458 // Resize the buffer if we need to copy into it.
459 if (Tok.needsCleaning())
460 Buffer.resize(Tok.getLength());
462 const char *Ptr = Buffer.data();
463 unsigned Len = getSpelling(Tok, Ptr, Invalid);
464 return StringRef(Ptr, Len);
467 /// CreateString - Plop the specified string into a scratch buffer and return a
468 /// location for it. If specified, the source location provides a source
469 /// location for the token.
470 void Preprocessor::CreateString(StringRef Str, Token &Tok,
471 SourceLocation ExpansionLocStart,
472 SourceLocation ExpansionLocEnd) {
473 Tok.setLength(Str.size());
476 SourceLocation Loc = ScratchBuf->getToken(Str.data(), Str.size(), DestPtr);
478 if (ExpansionLocStart.isValid())
479 Loc = SourceMgr.createExpansionLoc(Loc, ExpansionLocStart,
480 ExpansionLocEnd, Str.size());
481 Tok.setLocation(Loc);
483 // If this is a raw identifier or a literal token, set the pointer data.
484 if (Tok.is(tok::raw_identifier))
485 Tok.setRawIdentifierData(DestPtr);
486 else if (Tok.isLiteral())
487 Tok.setLiteralData(DestPtr);
490 Module *Preprocessor::getCurrentModule() {
491 if (!getLangOpts().isCompilingModule())
494 return getHeaderSearchInfo().lookupModule(getLangOpts().CurrentModule);
497 //===----------------------------------------------------------------------===//
498 // Preprocessor Initialization Methods
499 //===----------------------------------------------------------------------===//
501 /// EnterMainSourceFile - Enter the specified FileID as the main source file,
502 /// which implicitly adds the builtin defines etc.
503 void Preprocessor::EnterMainSourceFile() {
504 // We do not allow the preprocessor to reenter the main file. Doing so will
505 // cause FileID's to accumulate information from both runs (e.g. #line
506 // information) and predefined macros aren't guaranteed to be set properly.
507 assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!");
508 FileID MainFileID = SourceMgr.getMainFileID();
510 // If MainFileID is loaded it means we loaded an AST file, no need to enter
512 if (!SourceMgr.isLoadedFileID(MainFileID)) {
513 // Enter the main file source buffer.
514 EnterSourceFile(MainFileID, nullptr, SourceLocation());
516 // If we've been asked to skip bytes in the main file (e.g., as part of a
517 // precompiled preamble), do so now.
518 if (SkipMainFilePreamble.first > 0)
519 CurLexer->SkipBytes(SkipMainFilePreamble.first,
520 SkipMainFilePreamble.second);
522 // Tell the header info that the main file was entered. If the file is later
523 // #imported, it won't be re-entered.
524 if (const FileEntry *FE = SourceMgr.getFileEntryForID(MainFileID))
525 HeaderInfo.IncrementIncludeCount(FE);
528 // Preprocess Predefines to populate the initial preprocessor state.
529 std::unique_ptr<llvm::MemoryBuffer> SB =
530 llvm::MemoryBuffer::getMemBufferCopy(Predefines, "<built-in>");
531 assert(SB && "Cannot create predefined source buffer");
532 FileID FID = SourceMgr.createFileID(std::move(SB));
533 assert(FID.isValid() && "Could not create FileID for predefines?");
534 setPredefinesFileID(FID);
536 // Start parsing the predefines.
537 EnterSourceFile(FID, nullptr, SourceLocation());
540 void Preprocessor::replayPreambleConditionalStack() {
541 // Restore the conditional stack from the preamble, if there is one.
542 if (PreambleConditionalStack.isReplaying()) {
543 CurPPLexer->setConditionalLevels(PreambleConditionalStack.getStack());
544 PreambleConditionalStack.doneReplaying();
548 void Preprocessor::EndSourceFile() {
549 // Notify the client that we reached the end of the source file.
551 Callbacks->EndOfMainFile();
554 //===----------------------------------------------------------------------===//
555 // Lexer Event Handling.
556 //===----------------------------------------------------------------------===//
558 /// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the
559 /// identifier information for the token and install it into the token,
560 /// updating the token kind accordingly.
561 IdentifierInfo *Preprocessor::LookUpIdentifierInfo(Token &Identifier) const {
562 assert(!Identifier.getRawIdentifier().empty() && "No raw identifier data!");
564 // Look up this token, see if it is a macro, or if it is a language keyword.
566 if (!Identifier.needsCleaning() && !Identifier.hasUCN()) {
567 // No cleaning needed, just use the characters from the lexed buffer.
568 II = getIdentifierInfo(Identifier.getRawIdentifier());
570 // Cleaning needed, alloca a buffer, clean into it, then use the buffer.
571 SmallString<64> IdentifierBuffer;
572 StringRef CleanedStr = getSpelling(Identifier, IdentifierBuffer);
574 if (Identifier.hasUCN()) {
575 SmallString<64> UCNIdentifierBuffer;
576 expandUCNs(UCNIdentifierBuffer, CleanedStr);
577 II = getIdentifierInfo(UCNIdentifierBuffer);
579 II = getIdentifierInfo(CleanedStr);
583 // Update the token info (identifier info and appropriate token kind).
584 Identifier.setIdentifierInfo(II);
585 if (getLangOpts().MSVCCompat && II->isCPlusPlusOperatorKeyword() &&
586 getSourceManager().isInSystemHeader(Identifier.getLocation()))
587 Identifier.setKind(clang::tok::identifier);
589 Identifier.setKind(II->getTokenID());
594 void Preprocessor::SetPoisonReason(IdentifierInfo *II, unsigned DiagID) {
595 PoisonReasons[II] = DiagID;
598 void Preprocessor::PoisonSEHIdentifiers(bool Poison) {
599 assert(Ident__exception_code && Ident__exception_info);
600 assert(Ident___exception_code && Ident___exception_info);
601 Ident__exception_code->setIsPoisoned(Poison);
602 Ident___exception_code->setIsPoisoned(Poison);
603 Ident_GetExceptionCode->setIsPoisoned(Poison);
604 Ident__exception_info->setIsPoisoned(Poison);
605 Ident___exception_info->setIsPoisoned(Poison);
606 Ident_GetExceptionInfo->setIsPoisoned(Poison);
607 Ident__abnormal_termination->setIsPoisoned(Poison);
608 Ident___abnormal_termination->setIsPoisoned(Poison);
609 Ident_AbnormalTermination->setIsPoisoned(Poison);
612 void Preprocessor::HandlePoisonedIdentifier(Token & Identifier) {
613 assert(Identifier.getIdentifierInfo() &&
614 "Can't handle identifiers without identifier info!");
615 llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it =
616 PoisonReasons.find(Identifier.getIdentifierInfo());
617 if(it == PoisonReasons.end())
618 Diag(Identifier, diag::err_pp_used_poisoned_id);
620 Diag(Identifier,it->second) << Identifier.getIdentifierInfo();
623 /// \brief Returns a diagnostic message kind for reporting a future keyword as
624 /// appropriate for the identifier and specified language.
625 static diag::kind getFutureCompatDiagKind(const IdentifierInfo &II,
626 const LangOptions &LangOpts) {
627 assert(II.isFutureCompatKeyword() && "diagnostic should not be needed");
629 if (LangOpts.CPlusPlus)
630 return llvm::StringSwitch<diag::kind>(II.getName())
631 #define CXX11_KEYWORD(NAME, FLAGS) \
632 .Case(#NAME, diag::warn_cxx11_keyword)
633 #include "clang/Basic/TokenKinds.def"
637 "Keyword not known to come from a newer Standard or proposed Standard");
640 void Preprocessor::updateOutOfDateIdentifier(IdentifierInfo &II) const {
641 assert(II.isOutOfDate() && "not out of date");
642 getExternalSource()->updateOutOfDateIdentifier(II);
645 /// HandleIdentifier - This callback is invoked when the lexer reads an
646 /// identifier. This callback looks up the identifier in the map and/or
647 /// potentially macro expands it or turns it into a named token (like 'for').
649 /// Note that callers of this method are guarded by checking the
650 /// IdentifierInfo's 'isHandleIdentifierCase' bit. If this method changes, the
651 /// IdentifierInfo methods that compute these properties will need to change to
653 bool Preprocessor::HandleIdentifier(Token &Identifier) {
654 assert(Identifier.getIdentifierInfo() &&
655 "Can't handle identifiers without identifier info!");
657 IdentifierInfo &II = *Identifier.getIdentifierInfo();
659 // If the information about this identifier is out of date, update it from
660 // the external source.
661 // We have to treat __VA_ARGS__ in a special way, since it gets
662 // serialized with isPoisoned = true, but our preprocessor may have
663 // unpoisoned it if we're defining a C99 macro.
664 if (II.isOutOfDate()) {
665 bool CurrentIsPoisoned = false;
666 if (&II == Ident__VA_ARGS__)
667 CurrentIsPoisoned = Ident__VA_ARGS__->isPoisoned();
669 updateOutOfDateIdentifier(II);
670 Identifier.setKind(II.getTokenID());
672 if (&II == Ident__VA_ARGS__)
673 II.setIsPoisoned(CurrentIsPoisoned);
676 // If this identifier was poisoned, and if it was not produced from a macro
677 // expansion, emit an error.
678 if (II.isPoisoned() && CurPPLexer) {
679 HandlePoisonedIdentifier(Identifier);
682 // If this is a macro to be expanded, do it.
683 if (MacroDefinition MD = getMacroDefinition(&II)) {
684 auto *MI = MD.getMacroInfo();
685 assert(MI && "macro definition with no macro info?");
686 if (!DisableMacroExpansion) {
687 if (!Identifier.isExpandDisabled() && MI->isEnabled()) {
688 // C99 6.10.3p10: If the preprocessing token immediately after the
689 // macro name isn't a '(', this macro should not be expanded.
690 if (!MI->isFunctionLike() || isNextPPTokenLParen())
691 return HandleMacroExpandedIdentifier(Identifier, MD);
693 // C99 6.10.3.4p2 says that a disabled macro may never again be
694 // expanded, even if it's in a context where it could be expanded in the
696 Identifier.setFlag(Token::DisableExpand);
697 if (MI->isObjectLike() || isNextPPTokenLParen())
698 Diag(Identifier, diag::pp_disabled_macro_expansion);
703 // If this identifier is a keyword in a newer Standard or proposed Standard,
704 // produce a warning. Don't warn if we're not considering macro expansion,
705 // since this identifier might be the name of a macro.
706 // FIXME: This warning is disabled in cases where it shouldn't be, like
707 // "#define constexpr constexpr", "int constexpr;"
708 if (II.isFutureCompatKeyword() && !DisableMacroExpansion) {
709 Diag(Identifier, getFutureCompatDiagKind(II, getLangOpts()))
711 // Don't diagnose this keyword again in this translation unit.
712 II.setIsFutureCompatKeyword(false);
715 // If this is an extension token, diagnose its use.
716 // We avoid diagnosing tokens that originate from macro definitions.
717 // FIXME: This warning is disabled in cases where it shouldn't be,
718 // like "#define TY typeof", "TY(1) x".
719 if (II.isExtensionToken() && !DisableMacroExpansion)
720 Diag(Identifier, diag::ext_token_used);
722 // If this is the 'import' contextual keyword following an '@', note
723 // that the next token indicates a module name.
725 // Note that we do not treat 'import' as a contextual
726 // keyword when we're in a caching lexer, because caching lexers only get
727 // used in contexts where import declarations are disallowed.
729 // Likewise if this is the C++ Modules TS import keyword.
730 if (((LastTokenWasAt && II.isModulesImport()) ||
731 Identifier.is(tok::kw_import)) &&
732 !InMacroArgs && !DisableMacroExpansion &&
733 (getLangOpts().Modules || getLangOpts().DebuggerSupport) &&
734 CurLexerKind != CLK_CachingLexer) {
735 ModuleImportLoc = Identifier.getLocation();
736 ModuleImportPath.clear();
737 ModuleImportExpectsIdentifier = true;
738 CurLexerKind = CLK_LexAfterModuleImport;
743 void Preprocessor::Lex(Token &Result) {
744 // We loop here until a lex function returns a token; this avoids recursion.
747 switch (CurLexerKind) {
749 ReturnedToken = CurLexer->Lex(Result);
752 ReturnedToken = CurPTHLexer->Lex(Result);
755 ReturnedToken = CurTokenLexer->Lex(Result);
757 case CLK_CachingLexer:
759 ReturnedToken = true;
761 case CLK_LexAfterModuleImport:
762 LexAfterModuleImport(Result);
763 ReturnedToken = true;
766 } while (!ReturnedToken);
768 if (Result.is(tok::code_completion))
769 setCodeCompletionIdentifierInfo(Result.getIdentifierInfo());
771 LastTokenWasAt = Result.is(tok::at);
774 /// \brief Lex a token following the 'import' contextual keyword.
776 void Preprocessor::LexAfterModuleImport(Token &Result) {
777 // Figure out what kind of lexer we actually have.
778 recomputeCurLexerKind();
780 // Lex the next token.
783 // The token sequence
785 // import identifier (. identifier)*
787 // indicates a module import directive. We already saw the 'import'
788 // contextual keyword, so now we're looking for the identifiers.
789 if (ModuleImportExpectsIdentifier && Result.getKind() == tok::identifier) {
790 // We expected to see an identifier here, and we did; continue handling
792 ModuleImportPath.push_back(std::make_pair(Result.getIdentifierInfo(),
793 Result.getLocation()));
794 ModuleImportExpectsIdentifier = false;
795 CurLexerKind = CLK_LexAfterModuleImport;
799 // If we're expecting a '.' or a ';', and we got a '.', then wait until we
800 // see the next identifier. (We can also see a '[[' that begins an
801 // attribute-specifier-seq here under the C++ Modules TS.)
802 if (!ModuleImportExpectsIdentifier && Result.getKind() == tok::period) {
803 ModuleImportExpectsIdentifier = true;
804 CurLexerKind = CLK_LexAfterModuleImport;
808 // If we have a non-empty module path, load the named module.
809 if (!ModuleImportPath.empty()) {
810 // Under the Modules TS, the dot is just part of the module name, and not
811 // a real hierarachy separator. Flatten such module names now.
813 // FIXME: Is this the right level to be performing this transformation?
814 std::string FlatModuleName;
815 if (getLangOpts().ModulesTS) {
816 for (auto &Piece : ModuleImportPath) {
817 if (!FlatModuleName.empty())
818 FlatModuleName += ".";
819 FlatModuleName += Piece.first->getName();
821 SourceLocation FirstPathLoc = ModuleImportPath[0].second;
822 ModuleImportPath.clear();
823 ModuleImportPath.push_back(
824 std::make_pair(getIdentifierInfo(FlatModuleName), FirstPathLoc));
827 Module *Imported = nullptr;
828 if (getLangOpts().Modules) {
829 Imported = TheModuleLoader.loadModule(ModuleImportLoc,
832 /*IsIncludeDirective=*/false);
834 makeModuleVisible(Imported, ModuleImportLoc);
836 if (Callbacks && (getLangOpts().Modules || getLangOpts().DebuggerSupport))
837 Callbacks->moduleImport(ModuleImportLoc, ModuleImportPath, Imported);
841 void Preprocessor::makeModuleVisible(Module *M, SourceLocation Loc) {
842 CurSubmoduleState->VisibleModules.setVisible(
843 M, Loc, [](Module *) {},
844 [&](ArrayRef<Module *> Path, Module *Conflict, StringRef Message) {
845 // FIXME: Include the path in the diagnostic.
846 // FIXME: Include the import location for the conflicting module.
847 Diag(ModuleImportLoc, diag::warn_module_conflict)
848 << Path[0]->getFullModuleName()
849 << Conflict->getFullModuleName()
853 // Add this module to the imports list of the currently-built submodule.
854 if (!BuildingSubmoduleStack.empty() && M != BuildingSubmoduleStack.back().M)
855 BuildingSubmoduleStack.back().M->Imports.insert(M);
858 bool Preprocessor::FinishLexStringLiteral(Token &Result, std::string &String,
859 const char *DiagnosticTag,
860 bool AllowMacroExpansion) {
861 // We need at least one string literal.
862 if (Result.isNot(tok::string_literal)) {
863 Diag(Result, diag::err_expected_string_literal)
864 << /*Source='in...'*/0 << DiagnosticTag;
868 // Lex string literal tokens, optionally with macro expansion.
869 SmallVector<Token, 4> StrToks;
871 StrToks.push_back(Result);
873 if (Result.hasUDSuffix())
874 Diag(Result, diag::err_invalid_string_udl);
876 if (AllowMacroExpansion)
879 LexUnexpandedToken(Result);
880 } while (Result.is(tok::string_literal));
882 // Concatenate and parse the strings.
883 StringLiteralParser Literal(StrToks, *this);
884 assert(Literal.isAscii() && "Didn't allow wide strings in");
886 if (Literal.hadError)
889 if (Literal.Pascal) {
890 Diag(StrToks[0].getLocation(), diag::err_expected_string_literal)
891 << /*Source='in...'*/0 << DiagnosticTag;
895 String = Literal.GetString();
899 bool Preprocessor::parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value) {
900 assert(Tok.is(tok::numeric_constant));
901 SmallString<8> IntegerBuffer;
902 bool NumberInvalid = false;
903 StringRef Spelling = getSpelling(Tok, IntegerBuffer, &NumberInvalid);
906 NumericLiteralParser Literal(Spelling, Tok.getLocation(), *this);
907 if (Literal.hadError || !Literal.isIntegerLiteral() || Literal.hasUDSuffix())
909 llvm::APInt APVal(64, 0);
910 if (Literal.GetIntegerValue(APVal))
913 Value = APVal.getLimitedValue();
917 void Preprocessor::addCommentHandler(CommentHandler *Handler) {
918 assert(Handler && "NULL comment handler");
919 assert(std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler) ==
920 CommentHandlers.end() && "Comment handler already registered");
921 CommentHandlers.push_back(Handler);
924 void Preprocessor::removeCommentHandler(CommentHandler *Handler) {
925 std::vector<CommentHandler *>::iterator Pos
926 = std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler);
927 assert(Pos != CommentHandlers.end() && "Comment handler not registered");
928 CommentHandlers.erase(Pos);
931 bool Preprocessor::HandleComment(Token &result, SourceRange Comment) {
932 bool AnyPendingTokens = false;
933 for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(),
934 HEnd = CommentHandlers.end();
936 if ((*H)->HandleComment(*this, Comment))
937 AnyPendingTokens = true;
939 if (!AnyPendingTokens || getCommentRetentionState())
945 ModuleLoader::~ModuleLoader() { }
947 CommentHandler::~CommentHandler() { }
949 CodeCompletionHandler::~CodeCompletionHandler() { }
951 void Preprocessor::createPreprocessingRecord() {
955 Record = new PreprocessingRecord(getSourceManager());
956 addPPCallbacks(std::unique_ptr<PPCallbacks>(Record));