1 //===--- Preprocess.cpp - C Language Family Preprocessor Implementation ---===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file implements the Preprocessor interface.
12 //===----------------------------------------------------------------------===//
14 // Options to support:
15 // -H - Print the name of each header file used.
16 // -d[DNI] - Dump various things.
17 // -fworking-directory - #line's with preprocessor's working dir.
19 // -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD
24 // "Multiple include guards may be useful for:\n"
26 //===----------------------------------------------------------------------===//
28 #include "clang/Lex/Preprocessor.h"
29 #include "clang/Basic/FileManager.h"
30 #include "clang/Basic/FileSystemStatCache.h"
31 #include "clang/Basic/SourceManager.h"
32 #include "clang/Basic/TargetInfo.h"
33 #include "clang/Lex/CodeCompletionHandler.h"
34 #include "clang/Lex/ExternalPreprocessorSource.h"
35 #include "clang/Lex/HeaderSearch.h"
36 #include "clang/Lex/LexDiagnostic.h"
37 #include "clang/Lex/LiteralSupport.h"
38 #include "clang/Lex/MacroArgs.h"
39 #include "clang/Lex/MacroInfo.h"
40 #include "clang/Lex/ModuleLoader.h"
41 #include "clang/Lex/Pragma.h"
42 #include "clang/Lex/PreprocessingRecord.h"
43 #include "clang/Lex/PreprocessorOptions.h"
44 #include "clang/Lex/ScratchBuffer.h"
45 #include "llvm/ADT/APFloat.h"
46 #include "llvm/ADT/STLExtras.h"
47 #include "llvm/ADT/SmallString.h"
48 #include "llvm/ADT/StringExtras.h"
49 #include "llvm/Support/Capacity.h"
50 #include "llvm/Support/ConvertUTF.h"
51 #include "llvm/Support/MemoryBuffer.h"
52 #include "llvm/Support/raw_ostream.h"
53 using namespace clang;
55 //===----------------------------------------------------------------------===//
56 ExternalPreprocessorSource::~ExternalPreprocessorSource() { }
58 Preprocessor::Preprocessor(IntrusiveRefCntPtr<PreprocessorOptions> PPOpts,
59 DiagnosticsEngine &diags, LangOptions &opts,
60 SourceManager &SM, HeaderSearch &Headers,
61 ModuleLoader &TheModuleLoader,
62 IdentifierInfoLookup *IILookup, bool OwnsHeaders,
63 TranslationUnitKind TUKind)
64 : PPOpts(PPOpts), Diags(&diags), LangOpts(opts), Target(nullptr),
65 FileMgr(Headers.getFileMgr()), SourceMgr(SM),
66 ScratchBuf(new ScratchBuffer(SourceMgr)),HeaderInfo(Headers),
67 TheModuleLoader(TheModuleLoader), ExternalSource(nullptr),
68 Identifiers(opts, IILookup),
69 PragmaHandlers(new PragmaNamespace(StringRef())),
70 IncrementalProcessing(false), TUKind(TUKind),
71 CodeComplete(nullptr), CodeCompletionFile(nullptr),
72 CodeCompletionOffset(0), LastTokenWasAt(false),
73 ModuleImportExpectsIdentifier(false), CodeCompletionReached(0),
74 MainFileDir(nullptr), SkipMainFilePreamble(0, true), CurPPLexer(nullptr),
75 CurDirLookup(nullptr), CurLexerKind(CLK_Lexer), CurSubmodule(nullptr),
76 Callbacks(nullptr), MacroArgCache(nullptr), Record(nullptr),
77 MIChainHead(nullptr), DeserialMIChainHead(nullptr) {
78 OwnsHeaderSearch = OwnsHeaders;
80 CounterValue = 0; // __COUNTER__ starts at 0.
83 NumDirectives = NumDefined = NumUndefined = NumPragma = 0;
84 NumIf = NumElse = NumEndif = 0;
85 NumEnteredSourceFiles = 0;
86 NumMacroExpanded = NumFnMacroExpanded = NumBuiltinMacroExpanded = 0;
87 NumFastMacroExpanded = NumTokenPaste = NumFastTokenPaste = 0;
88 MaxIncludeStackDepth = 0;
91 // Default to discarding comments.
93 KeepMacroComments = false;
94 SuppressIncludeNotFoundError = false;
96 // Macro expansion is enabled.
97 DisableMacroExpansion = false;
98 MacroExpansionInDirectivesOverride = false;
100 InMacroArgPreExpansion = false;
101 NumCachedTokenLexers = 0;
102 PragmasEnabled = true;
103 ParsingIfOrElifDirective = false;
104 PreprocessedOutput = false;
108 // We haven't read anything from the external source.
109 ReadMacrosFromExternalSource = false;
111 // "Poison" __VA_ARGS__, which can only appear in the expansion of a macro.
112 // This gets unpoisoned where it is allowed.
113 (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned();
114 SetPoisonReason(Ident__VA_ARGS__,diag::ext_pp_bad_vaargs_use);
116 // Initialize the pragma handlers.
117 RegisterBuiltinPragmas();
119 // Initialize builtin macros like __LINE__ and friends.
120 RegisterBuiltinMacros();
122 if(LangOpts.Borland) {
123 Ident__exception_info = getIdentifierInfo("_exception_info");
124 Ident___exception_info = getIdentifierInfo("__exception_info");
125 Ident_GetExceptionInfo = getIdentifierInfo("GetExceptionInformation");
126 Ident__exception_code = getIdentifierInfo("_exception_code");
127 Ident___exception_code = getIdentifierInfo("__exception_code");
128 Ident_GetExceptionCode = getIdentifierInfo("GetExceptionCode");
129 Ident__abnormal_termination = getIdentifierInfo("_abnormal_termination");
130 Ident___abnormal_termination = getIdentifierInfo("__abnormal_termination");
131 Ident_AbnormalTermination = getIdentifierInfo("AbnormalTermination");
133 Ident__exception_info = Ident__exception_code = nullptr;
134 Ident__abnormal_termination = Ident___exception_info = nullptr;
135 Ident___exception_code = Ident___abnormal_termination = nullptr;
136 Ident_GetExceptionInfo = Ident_GetExceptionCode = nullptr;
137 Ident_AbnormalTermination = nullptr;
141 Preprocessor::~Preprocessor() {
142 assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!");
144 IncludeMacroStack.clear();
146 // Destroy any macro definitions.
147 while (MacroInfoChain *I = MIChainHead) {
148 MIChainHead = I->Next;
149 I->~MacroInfoChain();
152 // Free any cached macro expanders.
153 // This populates MacroArgCache, so all TokenLexers need to be destroyed
154 // before the code below that frees up the MacroArgCache list.
155 std::fill(TokenLexerCache, TokenLexerCache + NumCachedTokenLexers, nullptr);
156 CurTokenLexer.reset();
158 while (DeserializedMacroInfoChain *I = DeserialMIChainHead) {
159 DeserialMIChainHead = I->Next;
160 I->~DeserializedMacroInfoChain();
163 // Free any cached MacroArgs.
164 for (MacroArgs *ArgList = MacroArgCache; ArgList;)
165 ArgList = ArgList->deallocate();
167 // Delete the header search info, if we own it.
168 if (OwnsHeaderSearch)
172 void Preprocessor::Initialize(const TargetInfo &Target) {
173 assert((!this->Target || this->Target == &Target) &&
174 "Invalid override of target information");
175 this->Target = &Target;
177 // Initialize information about built-ins.
178 BuiltinInfo.InitializeTarget(Target);
179 HeaderInfo.setTarget(Target);
182 void Preprocessor::InitializeForModelFile() {
183 NumEnteredSourceFiles = 0;
186 PragmaHandlersBackup = std::move(PragmaHandlers);
187 PragmaHandlers = llvm::make_unique<PragmaNamespace>(StringRef());
188 RegisterBuiltinPragmas();
190 // Reset PredefinesFileID
191 PredefinesFileID = FileID();
194 void Preprocessor::FinalizeForModelFile() {
195 NumEnteredSourceFiles = 1;
197 PragmaHandlers = std::move(PragmaHandlersBackup);
200 void Preprocessor::setPTHManager(PTHManager* pm) {
202 FileMgr.addStatCache(PTH->createStatCache());
205 void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const {
206 llvm::errs() << tok::getTokenName(Tok.getKind()) << " '"
207 << getSpelling(Tok) << "'";
209 if (!DumpFlags) return;
211 llvm::errs() << "\t";
212 if (Tok.isAtStartOfLine())
213 llvm::errs() << " [StartOfLine]";
214 if (Tok.hasLeadingSpace())
215 llvm::errs() << " [LeadingSpace]";
216 if (Tok.isExpandDisabled())
217 llvm::errs() << " [ExpandDisabled]";
218 if (Tok.needsCleaning()) {
219 const char *Start = SourceMgr.getCharacterData(Tok.getLocation());
220 llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength())
224 llvm::errs() << "\tLoc=<";
225 DumpLocation(Tok.getLocation());
229 void Preprocessor::DumpLocation(SourceLocation Loc) const {
233 void Preprocessor::DumpMacro(const MacroInfo &MI) const {
234 llvm::errs() << "MACRO: ";
235 for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) {
236 DumpToken(MI.getReplacementToken(i));
239 llvm::errs() << "\n";
242 void Preprocessor::PrintStats() {
243 llvm::errs() << "\n*** Preprocessor Stats:\n";
244 llvm::errs() << NumDirectives << " directives found:\n";
245 llvm::errs() << " " << NumDefined << " #define.\n";
246 llvm::errs() << " " << NumUndefined << " #undef.\n";
247 llvm::errs() << " #include/#include_next/#import:\n";
248 llvm::errs() << " " << NumEnteredSourceFiles << " source files entered.\n";
249 llvm::errs() << " " << MaxIncludeStackDepth << " max include stack depth\n";
250 llvm::errs() << " " << NumIf << " #if/#ifndef/#ifdef.\n";
251 llvm::errs() << " " << NumElse << " #else/#elif.\n";
252 llvm::errs() << " " << NumEndif << " #endif.\n";
253 llvm::errs() << " " << NumPragma << " #pragma.\n";
254 llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n";
256 llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/"
257 << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, "
258 << NumFastMacroExpanded << " on the fast path.\n";
259 llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
260 << " token paste (##) operations performed, "
261 << NumFastTokenPaste << " on the fast path.\n";
263 llvm::errs() << "\nPreprocessor Memory: " << getTotalMemory() << "B total";
265 llvm::errs() << "\n BumpPtr: " << BP.getTotalMemory();
266 llvm::errs() << "\n Macro Expanded Tokens: "
267 << llvm::capacity_in_bytes(MacroExpandedTokens);
268 llvm::errs() << "\n Predefines Buffer: " << Predefines.capacity();
269 llvm::errs() << "\n Macros: " << llvm::capacity_in_bytes(Macros);
270 llvm::errs() << "\n #pragma push_macro Info: "
271 << llvm::capacity_in_bytes(PragmaPushMacroInfo);
272 llvm::errs() << "\n Poison Reasons: "
273 << llvm::capacity_in_bytes(PoisonReasons);
274 llvm::errs() << "\n Comment Handlers: "
275 << llvm::capacity_in_bytes(CommentHandlers) << "\n";
278 Preprocessor::macro_iterator
279 Preprocessor::macro_begin(bool IncludeExternalMacros) const {
280 if (IncludeExternalMacros && ExternalSource &&
281 !ReadMacrosFromExternalSource) {
282 ReadMacrosFromExternalSource = true;
283 ExternalSource->ReadDefinedMacros();
286 return Macros.begin();
289 size_t Preprocessor::getTotalMemory() const {
290 return BP.getTotalMemory()
291 + llvm::capacity_in_bytes(MacroExpandedTokens)
292 + Predefines.capacity() /* Predefines buffer. */
293 + llvm::capacity_in_bytes(Macros)
294 + llvm::capacity_in_bytes(PragmaPushMacroInfo)
295 + llvm::capacity_in_bytes(PoisonReasons)
296 + llvm::capacity_in_bytes(CommentHandlers);
299 Preprocessor::macro_iterator
300 Preprocessor::macro_end(bool IncludeExternalMacros) const {
301 if (IncludeExternalMacros && ExternalSource &&
302 !ReadMacrosFromExternalSource) {
303 ReadMacrosFromExternalSource = true;
304 ExternalSource->ReadDefinedMacros();
310 /// \brief Compares macro tokens with a specified token value sequence.
311 static bool MacroDefinitionEquals(const MacroInfo *MI,
312 ArrayRef<TokenValue> Tokens) {
313 return Tokens.size() == MI->getNumTokens() &&
314 std::equal(Tokens.begin(), Tokens.end(), MI->tokens_begin());
317 StringRef Preprocessor::getLastMacroWithSpelling(
319 ArrayRef<TokenValue> Tokens) const {
320 SourceLocation BestLocation;
321 StringRef BestSpelling;
322 for (Preprocessor::macro_iterator I = macro_begin(), E = macro_end();
324 if (!I->second->getMacroInfo()->isObjectLike())
326 const MacroDirective::DefInfo
327 Def = I->second->findDirectiveAtLoc(Loc, SourceMgr);
330 if (!MacroDefinitionEquals(Def.getMacroInfo(), Tokens))
332 SourceLocation Location = Def.getLocation();
333 // Choose the macro defined latest.
334 if (BestLocation.isInvalid() ||
335 (Location.isValid() &&
336 SourceMgr.isBeforeInTranslationUnit(BestLocation, Location))) {
337 BestLocation = Location;
338 BestSpelling = I->first->getName();
344 void Preprocessor::recomputeCurLexerKind() {
346 CurLexerKind = CLK_Lexer;
347 else if (CurPTHLexer)
348 CurLexerKind = CLK_PTHLexer;
349 else if (CurTokenLexer)
350 CurLexerKind = CLK_TokenLexer;
352 CurLexerKind = CLK_CachingLexer;
355 bool Preprocessor::SetCodeCompletionPoint(const FileEntry *File,
356 unsigned CompleteLine,
357 unsigned CompleteColumn) {
359 assert(CompleteLine && CompleteColumn && "Starts from 1:1");
360 assert(!CodeCompletionFile && "Already set");
362 using llvm::MemoryBuffer;
364 // Load the actual file's contents.
365 bool Invalid = false;
366 const MemoryBuffer *Buffer = SourceMgr.getMemoryBufferForFile(File, &Invalid);
370 // Find the byte position of the truncation point.
371 const char *Position = Buffer->getBufferStart();
372 for (unsigned Line = 1; Line < CompleteLine; ++Line) {
373 for (; *Position; ++Position) {
374 if (*Position != '\r' && *Position != '\n')
377 // Eat \r\n or \n\r as a single line.
378 if ((Position[1] == '\r' || Position[1] == '\n') &&
379 Position[0] != Position[1])
386 Position += CompleteColumn - 1;
388 // If pointing inside the preamble, adjust the position at the beginning of
389 // the file after the preamble.
390 if (SkipMainFilePreamble.first &&
391 SourceMgr.getFileEntryForID(SourceMgr.getMainFileID()) == File) {
392 if (Position - Buffer->getBufferStart() < SkipMainFilePreamble.first)
393 Position = Buffer->getBufferStart() + SkipMainFilePreamble.first;
396 if (Position > Buffer->getBufferEnd())
397 Position = Buffer->getBufferEnd();
399 CodeCompletionFile = File;
400 CodeCompletionOffset = Position - Buffer->getBufferStart();
402 std::unique_ptr<MemoryBuffer> NewBuffer =
403 MemoryBuffer::getNewUninitMemBuffer(Buffer->getBufferSize() + 1,
404 Buffer->getBufferIdentifier());
405 char *NewBuf = const_cast<char*>(NewBuffer->getBufferStart());
406 char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf);
408 std::copy(Position, Buffer->getBufferEnd(), NewPos+1);
409 SourceMgr.overrideFileContents(File, std::move(NewBuffer));
414 void Preprocessor::CodeCompleteNaturalLanguage() {
416 CodeComplete->CodeCompleteNaturalLanguage();
417 setCodeCompletionReached();
420 /// getSpelling - This method is used to get the spelling of a token into a
421 /// SmallVector. Note that the returned StringRef may not point to the
422 /// supplied buffer if a copy can be avoided.
423 StringRef Preprocessor::getSpelling(const Token &Tok,
424 SmallVectorImpl<char> &Buffer,
425 bool *Invalid) const {
426 // NOTE: this has to be checked *before* testing for an IdentifierInfo.
427 if (Tok.isNot(tok::raw_identifier) && !Tok.hasUCN()) {
428 // Try the fast path.
429 if (const IdentifierInfo *II = Tok.getIdentifierInfo())
430 return II->getName();
433 // Resize the buffer if we need to copy into it.
434 if (Tok.needsCleaning())
435 Buffer.resize(Tok.getLength());
437 const char *Ptr = Buffer.data();
438 unsigned Len = getSpelling(Tok, Ptr, Invalid);
439 return StringRef(Ptr, Len);
442 /// CreateString - Plop the specified string into a scratch buffer and return a
443 /// location for it. If specified, the source location provides a source
444 /// location for the token.
445 void Preprocessor::CreateString(StringRef Str, Token &Tok,
446 SourceLocation ExpansionLocStart,
447 SourceLocation ExpansionLocEnd) {
448 Tok.setLength(Str.size());
451 SourceLocation Loc = ScratchBuf->getToken(Str.data(), Str.size(), DestPtr);
453 if (ExpansionLocStart.isValid())
454 Loc = SourceMgr.createExpansionLoc(Loc, ExpansionLocStart,
455 ExpansionLocEnd, Str.size());
456 Tok.setLocation(Loc);
458 // If this is a raw identifier or a literal token, set the pointer data.
459 if (Tok.is(tok::raw_identifier))
460 Tok.setRawIdentifierData(DestPtr);
461 else if (Tok.isLiteral())
462 Tok.setLiteralData(DestPtr);
465 Module *Preprocessor::getCurrentModule() {
466 if (getLangOpts().CurrentModule.empty())
469 return getHeaderSearchInfo().lookupModule(getLangOpts().CurrentModule);
472 //===----------------------------------------------------------------------===//
473 // Preprocessor Initialization Methods
474 //===----------------------------------------------------------------------===//
477 /// EnterMainSourceFile - Enter the specified FileID as the main source file,
478 /// which implicitly adds the builtin defines etc.
479 void Preprocessor::EnterMainSourceFile() {
480 // We do not allow the preprocessor to reenter the main file. Doing so will
481 // cause FileID's to accumulate information from both runs (e.g. #line
482 // information) and predefined macros aren't guaranteed to be set properly.
483 assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!");
484 FileID MainFileID = SourceMgr.getMainFileID();
486 // If MainFileID is loaded it means we loaded an AST file, no need to enter
488 if (!SourceMgr.isLoadedFileID(MainFileID)) {
489 // Enter the main file source buffer.
490 EnterSourceFile(MainFileID, nullptr, SourceLocation());
492 // If we've been asked to skip bytes in the main file (e.g., as part of a
493 // precompiled preamble), do so now.
494 if (SkipMainFilePreamble.first > 0)
495 CurLexer->SkipBytes(SkipMainFilePreamble.first,
496 SkipMainFilePreamble.second);
498 // Tell the header info that the main file was entered. If the file is later
499 // #imported, it won't be re-entered.
500 if (const FileEntry *FE = SourceMgr.getFileEntryForID(MainFileID))
501 HeaderInfo.IncrementIncludeCount(FE);
504 // Preprocess Predefines to populate the initial preprocessor state.
505 std::unique_ptr<llvm::MemoryBuffer> SB =
506 llvm::MemoryBuffer::getMemBufferCopy(Predefines, "<built-in>");
507 assert(SB && "Cannot create predefined source buffer");
508 FileID FID = SourceMgr.createFileID(std::move(SB));
509 assert(!FID.isInvalid() && "Could not create FileID for predefines?");
510 setPredefinesFileID(FID);
512 // Start parsing the predefines.
513 EnterSourceFile(FID, nullptr, SourceLocation());
516 void Preprocessor::EndSourceFile() {
517 // Notify the client that we reached the end of the source file.
519 Callbacks->EndOfMainFile();
522 //===----------------------------------------------------------------------===//
523 // Lexer Event Handling.
524 //===----------------------------------------------------------------------===//
526 /// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the
527 /// identifier information for the token and install it into the token,
528 /// updating the token kind accordingly.
529 IdentifierInfo *Preprocessor::LookUpIdentifierInfo(Token &Identifier) const {
530 assert(!Identifier.getRawIdentifier().empty() && "No raw identifier data!");
532 // Look up this token, see if it is a macro, or if it is a language keyword.
534 if (!Identifier.needsCleaning() && !Identifier.hasUCN()) {
535 // No cleaning needed, just use the characters from the lexed buffer.
536 II = getIdentifierInfo(Identifier.getRawIdentifier());
538 // Cleaning needed, alloca a buffer, clean into it, then use the buffer.
539 SmallString<64> IdentifierBuffer;
540 StringRef CleanedStr = getSpelling(Identifier, IdentifierBuffer);
542 if (Identifier.hasUCN()) {
543 SmallString<64> UCNIdentifierBuffer;
544 expandUCNs(UCNIdentifierBuffer, CleanedStr);
545 II = getIdentifierInfo(UCNIdentifierBuffer);
547 II = getIdentifierInfo(CleanedStr);
551 // Update the token info (identifier info and appropriate token kind).
552 Identifier.setIdentifierInfo(II);
553 Identifier.setKind(II->getTokenID());
558 void Preprocessor::SetPoisonReason(IdentifierInfo *II, unsigned DiagID) {
559 PoisonReasons[II] = DiagID;
562 void Preprocessor::PoisonSEHIdentifiers(bool Poison) {
563 assert(Ident__exception_code && Ident__exception_info);
564 assert(Ident___exception_code && Ident___exception_info);
565 Ident__exception_code->setIsPoisoned(Poison);
566 Ident___exception_code->setIsPoisoned(Poison);
567 Ident_GetExceptionCode->setIsPoisoned(Poison);
568 Ident__exception_info->setIsPoisoned(Poison);
569 Ident___exception_info->setIsPoisoned(Poison);
570 Ident_GetExceptionInfo->setIsPoisoned(Poison);
571 Ident__abnormal_termination->setIsPoisoned(Poison);
572 Ident___abnormal_termination->setIsPoisoned(Poison);
573 Ident_AbnormalTermination->setIsPoisoned(Poison);
576 void Preprocessor::HandlePoisonedIdentifier(Token & Identifier) {
577 assert(Identifier.getIdentifierInfo() &&
578 "Can't handle identifiers without identifier info!");
579 llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it =
580 PoisonReasons.find(Identifier.getIdentifierInfo());
581 if(it == PoisonReasons.end())
582 Diag(Identifier, diag::err_pp_used_poisoned_id);
584 Diag(Identifier,it->second) << Identifier.getIdentifierInfo();
587 /// HandleIdentifier - This callback is invoked when the lexer reads an
588 /// identifier. This callback looks up the identifier in the map and/or
589 /// potentially macro expands it or turns it into a named token (like 'for').
591 /// Note that callers of this method are guarded by checking the
592 /// IdentifierInfo's 'isHandleIdentifierCase' bit. If this method changes, the
593 /// IdentifierInfo methods that compute these properties will need to change to
595 bool Preprocessor::HandleIdentifier(Token &Identifier) {
596 assert(Identifier.getIdentifierInfo() &&
597 "Can't handle identifiers without identifier info!");
599 IdentifierInfo &II = *Identifier.getIdentifierInfo();
601 // If the information about this identifier is out of date, update it from
602 // the external source.
603 // We have to treat __VA_ARGS__ in a special way, since it gets
604 // serialized with isPoisoned = true, but our preprocessor may have
605 // unpoisoned it if we're defining a C99 macro.
606 if (II.isOutOfDate()) {
607 bool CurrentIsPoisoned = false;
608 if (&II == Ident__VA_ARGS__)
609 CurrentIsPoisoned = Ident__VA_ARGS__->isPoisoned();
611 ExternalSource->updateOutOfDateIdentifier(II);
612 Identifier.setKind(II.getTokenID());
614 if (&II == Ident__VA_ARGS__)
615 II.setIsPoisoned(CurrentIsPoisoned);
618 // If this identifier was poisoned, and if it was not produced from a macro
619 // expansion, emit an error.
620 if (II.isPoisoned() && CurPPLexer) {
621 HandlePoisonedIdentifier(Identifier);
624 // If this is a macro to be expanded, do it.
625 if (MacroDirective *MD = getMacroDirective(&II)) {
626 MacroInfo *MI = MD->getMacroInfo();
627 if (!DisableMacroExpansion) {
628 if (!Identifier.isExpandDisabled() && MI->isEnabled()) {
629 // C99 6.10.3p10: If the preprocessing token immediately after the
630 // macro name isn't a '(', this macro should not be expanded.
631 if (!MI->isFunctionLike() || isNextPPTokenLParen())
632 return HandleMacroExpandedIdentifier(Identifier, MD);
634 // C99 6.10.3.4p2 says that a disabled macro may never again be
635 // expanded, even if it's in a context where it could be expanded in the
637 Identifier.setFlag(Token::DisableExpand);
638 if (MI->isObjectLike() || isNextPPTokenLParen())
639 Diag(Identifier, diag::pp_disabled_macro_expansion);
644 // If this identifier is a keyword in C++11, produce a warning. Don't warn if
645 // we're not considering macro expansion, since this identifier might be the
647 // FIXME: This warning is disabled in cases where it shouldn't be, like
648 // "#define constexpr constexpr", "int constexpr;"
649 if (II.isCXX11CompatKeyword() && !DisableMacroExpansion) {
650 Diag(Identifier, diag::warn_cxx11_keyword) << II.getName();
651 // Don't diagnose this keyword again in this translation unit.
652 II.setIsCXX11CompatKeyword(false);
655 // C++ 2.11p2: If this is an alternative representation of a C++ operator,
656 // then we act as if it is the actual operator and not the textual
657 // representation of it.
658 if (II.isCPlusPlusOperatorKeyword())
659 Identifier.setIdentifierInfo(nullptr);
661 // If this is an extension token, diagnose its use.
662 // We avoid diagnosing tokens that originate from macro definitions.
663 // FIXME: This warning is disabled in cases where it shouldn't be,
664 // like "#define TY typeof", "TY(1) x".
665 if (II.isExtensionToken() && !DisableMacroExpansion)
666 Diag(Identifier, diag::ext_token_used);
668 // If this is the 'import' contextual keyword following an '@', note
669 // that the next token indicates a module name.
671 // Note that we do not treat 'import' as a contextual
672 // keyword when we're in a caching lexer, because caching lexers only get
673 // used in contexts where import declarations are disallowed.
674 if (LastTokenWasAt && II.isModulesImport() && !InMacroArgs &&
675 !DisableMacroExpansion &&
676 (getLangOpts().Modules || getLangOpts().DebuggerSupport) &&
677 CurLexerKind != CLK_CachingLexer) {
678 ModuleImportLoc = Identifier.getLocation();
679 ModuleImportPath.clear();
680 ModuleImportExpectsIdentifier = true;
681 CurLexerKind = CLK_LexAfterModuleImport;
686 void Preprocessor::Lex(Token &Result) {
687 // We loop here until a lex function retuns a token; this avoids recursion.
690 switch (CurLexerKind) {
692 ReturnedToken = CurLexer->Lex(Result);
695 ReturnedToken = CurPTHLexer->Lex(Result);
698 ReturnedToken = CurTokenLexer->Lex(Result);
700 case CLK_CachingLexer:
702 ReturnedToken = true;
704 case CLK_LexAfterModuleImport:
705 LexAfterModuleImport(Result);
706 ReturnedToken = true;
709 } while (!ReturnedToken);
711 LastTokenWasAt = Result.is(tok::at);
715 /// \brief Lex a token following the 'import' contextual keyword.
717 void Preprocessor::LexAfterModuleImport(Token &Result) {
718 // Figure out what kind of lexer we actually have.
719 recomputeCurLexerKind();
721 // Lex the next token.
724 // The token sequence
726 // import identifier (. identifier)*
728 // indicates a module import directive. We already saw the 'import'
729 // contextual keyword, so now we're looking for the identifiers.
730 if (ModuleImportExpectsIdentifier && Result.getKind() == tok::identifier) {
731 // We expected to see an identifier here, and we did; continue handling
733 ModuleImportPath.push_back(std::make_pair(Result.getIdentifierInfo(),
734 Result.getLocation()));
735 ModuleImportExpectsIdentifier = false;
736 CurLexerKind = CLK_LexAfterModuleImport;
740 // If we're expecting a '.' or a ';', and we got a '.', then wait until we
741 // see the next identifier.
742 if (!ModuleImportExpectsIdentifier && Result.getKind() == tok::period) {
743 ModuleImportExpectsIdentifier = true;
744 CurLexerKind = CLK_LexAfterModuleImport;
748 // If we have a non-empty module path, load the named module.
749 if (!ModuleImportPath.empty()) {
750 Module *Imported = nullptr;
751 if (getLangOpts().Modules)
752 Imported = TheModuleLoader.loadModule(ModuleImportLoc,
754 Module::MacrosVisible,
755 /*IsIncludeDirective=*/false);
756 if (Callbacks && (getLangOpts().Modules || getLangOpts().DebuggerSupport))
757 Callbacks->moduleImport(ModuleImportLoc, ModuleImportPath, Imported);
761 bool Preprocessor::FinishLexStringLiteral(Token &Result, std::string &String,
762 const char *DiagnosticTag,
763 bool AllowMacroExpansion) {
764 // We need at least one string literal.
765 if (Result.isNot(tok::string_literal)) {
766 Diag(Result, diag::err_expected_string_literal)
767 << /*Source='in...'*/0 << DiagnosticTag;
771 // Lex string literal tokens, optionally with macro expansion.
772 SmallVector<Token, 4> StrToks;
774 StrToks.push_back(Result);
776 if (Result.hasUDSuffix())
777 Diag(Result, diag::err_invalid_string_udl);
779 if (AllowMacroExpansion)
782 LexUnexpandedToken(Result);
783 } while (Result.is(tok::string_literal));
785 // Concatenate and parse the strings.
786 StringLiteralParser Literal(StrToks, *this);
787 assert(Literal.isAscii() && "Didn't allow wide strings in");
789 if (Literal.hadError)
792 if (Literal.Pascal) {
793 Diag(StrToks[0].getLocation(), diag::err_expected_string_literal)
794 << /*Source='in...'*/0 << DiagnosticTag;
798 String = Literal.GetString();
802 bool Preprocessor::parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value) {
803 assert(Tok.is(tok::numeric_constant));
804 SmallString<8> IntegerBuffer;
805 bool NumberInvalid = false;
806 StringRef Spelling = getSpelling(Tok, IntegerBuffer, &NumberInvalid);
809 NumericLiteralParser Literal(Spelling, Tok.getLocation(), *this);
810 if (Literal.hadError || !Literal.isIntegerLiteral() || Literal.hasUDSuffix())
812 llvm::APInt APVal(64, 0);
813 if (Literal.GetIntegerValue(APVal))
816 Value = APVal.getLimitedValue();
820 void Preprocessor::addCommentHandler(CommentHandler *Handler) {
821 assert(Handler && "NULL comment handler");
822 assert(std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler) ==
823 CommentHandlers.end() && "Comment handler already registered");
824 CommentHandlers.push_back(Handler);
827 void Preprocessor::removeCommentHandler(CommentHandler *Handler) {
828 std::vector<CommentHandler *>::iterator Pos
829 = std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler);
830 assert(Pos != CommentHandlers.end() && "Comment handler not registered");
831 CommentHandlers.erase(Pos);
834 bool Preprocessor::HandleComment(Token &result, SourceRange Comment) {
835 bool AnyPendingTokens = false;
836 for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(),
837 HEnd = CommentHandlers.end();
839 if ((*H)->HandleComment(*this, Comment))
840 AnyPendingTokens = true;
842 if (!AnyPendingTokens || getCommentRetentionState())
848 ModuleLoader::~ModuleLoader() { }
850 CommentHandler::~CommentHandler() { }
852 CodeCompletionHandler::~CodeCompletionHandler() { }
854 void Preprocessor::createPreprocessingRecord() {
858 Record = new PreprocessingRecord(getSourceManager());
859 addPPCallbacks(std::unique_ptr<PPCallbacks>(Record));