//===--- Preprocessor.h - C Language Family Preprocessor --------*- C++ -*-===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// /// /// \file /// \brief Defines the clang::Preprocessor interface. /// //===----------------------------------------------------------------------===// #ifndef LLVM_CLANG_LEX_PREPROCESSOR_H #define LLVM_CLANG_LEX_PREPROCESSOR_H #include "clang/Basic/Builtins.h" #include "clang/Basic/Diagnostic.h" #include "clang/Basic/IdentifierTable.h" #include "clang/Basic/SourceLocation.h" #include "clang/Lex/Lexer.h" #include "clang/Lex/MacroInfo.h" #include "clang/Lex/ModuleMap.h" #include "clang/Lex/PPCallbacks.h" #include "clang/Lex/PTHLexer.h" #include "clang/Lex/PTHManager.h" #include "clang/Lex/TokenLexer.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Support/Allocator.h" #include #include namespace llvm { template class SmallString; } namespace clang { class SourceManager; class ExternalPreprocessorSource; class FileManager; class FileEntry; class HeaderSearch; class PragmaNamespace; class PragmaHandler; class CommentHandler; class ScratchBuffer; class TargetInfo; class PPCallbacks; class CodeCompletionHandler; class DirectoryLookup; class PreprocessingRecord; class ModuleLoader; class PreprocessorOptions; /// \brief Stores token information for comparing actual tokens with /// predefined values. Only handles simple tokens and identifiers. class TokenValue { tok::TokenKind Kind; IdentifierInfo *II; public: TokenValue(tok::TokenKind Kind) : Kind(Kind), II(nullptr) { assert(Kind != tok::raw_identifier && "Raw identifiers are not supported."); assert(Kind != tok::identifier && "Identifiers should be created by TokenValue(IdentifierInfo *)"); assert(!tok::isLiteral(Kind) && "Literals are not supported."); assert(!tok::isAnnotation(Kind) && "Annotations are not supported."); } TokenValue(IdentifierInfo *II) : Kind(tok::identifier), II(II) {} bool operator==(const Token &Tok) const { return Tok.getKind() == Kind && (!II || II == Tok.getIdentifierInfo()); } }; /// \brief Engages in a tight little dance with the lexer to efficiently /// preprocess tokens. /// /// Lexers know only about tokens within a single source file, and don't /// know anything about preprocessor-level issues like the \#include stack, /// token expansion, etc. class Preprocessor : public RefCountedBase { IntrusiveRefCntPtr PPOpts; DiagnosticsEngine *Diags; LangOptions &LangOpts; const TargetInfo *Target; FileManager &FileMgr; SourceManager &SourceMgr; ScratchBuffer *ScratchBuf; HeaderSearch &HeaderInfo; ModuleLoader &TheModuleLoader; /// \brief External source of macros. ExternalPreprocessorSource *ExternalSource; /// An optional PTHManager object used for getting tokens from /// a token cache rather than lexing the original source file. std::unique_ptr PTH; /// A BumpPtrAllocator object used to quickly allocate and release /// objects internal to the Preprocessor. llvm::BumpPtrAllocator BP; /// Identifiers for builtin macros and other builtins. IdentifierInfo *Ident__LINE__, *Ident__FILE__; // __LINE__, __FILE__ IdentifierInfo *Ident__DATE__, *Ident__TIME__; // __DATE__, __TIME__ IdentifierInfo *Ident__INCLUDE_LEVEL__; // __INCLUDE_LEVEL__ IdentifierInfo *Ident__BASE_FILE__; // __BASE_FILE__ IdentifierInfo *Ident__TIMESTAMP__; // __TIMESTAMP__ IdentifierInfo *Ident__COUNTER__; // __COUNTER__ IdentifierInfo *Ident_Pragma, *Ident__pragma; // _Pragma, __pragma IdentifierInfo *Ident__identifier; // __identifier IdentifierInfo *Ident__VA_ARGS__; // __VA_ARGS__ IdentifierInfo *Ident__has_feature; // __has_feature IdentifierInfo *Ident__has_extension; // __has_extension IdentifierInfo *Ident__has_builtin; // __has_builtin IdentifierInfo *Ident__has_attribute; // __has_attribute IdentifierInfo *Ident__has_include; // __has_include IdentifierInfo *Ident__has_include_next; // __has_include_next IdentifierInfo *Ident__has_warning; // __has_warning IdentifierInfo *Ident__is_identifier; // __is_identifier IdentifierInfo *Ident__building_module; // __building_module IdentifierInfo *Ident__MODULE__; // __MODULE__ SourceLocation DATELoc, TIMELoc; unsigned CounterValue; // Next __COUNTER__ value. enum { /// \brief Maximum depth of \#includes. MaxAllowedIncludeStackDepth = 200 }; // State that is set before the preprocessor begins. bool KeepComments : 1; bool KeepMacroComments : 1; bool SuppressIncludeNotFoundError : 1; // State that changes while the preprocessor runs: bool InMacroArgs : 1; // True if parsing fn macro invocation args. /// Whether the preprocessor owns the header search object. bool OwnsHeaderSearch : 1; /// True if macro expansion is disabled. bool DisableMacroExpansion : 1; /// Temporarily disables DisableMacroExpansion (i.e. enables expansion) /// when parsing preprocessor directives. bool MacroExpansionInDirectivesOverride : 1; class ResetMacroExpansionHelper; /// \brief Whether we have already loaded macros from the external source. mutable bool ReadMacrosFromExternalSource : 1; /// \brief True if pragmas are enabled. bool PragmasEnabled : 1; /// \brief True if the current build action is a preprocessing action. bool PreprocessedOutput : 1; /// \brief True if we are currently preprocessing a #if or #elif directive bool ParsingIfOrElifDirective; /// \brief True if we are pre-expanding macro arguments. bool InMacroArgPreExpansion; /// \brief Mapping/lookup information for all identifiers in /// the program, including program keywords. mutable IdentifierTable Identifiers; /// \brief This table contains all the selectors in the program. /// /// Unlike IdentifierTable above, this table *isn't* populated by the /// preprocessor. It is declared/expanded here because its role/lifetime is /// conceptually similar to the IdentifierTable. In addition, the current /// control flow (in clang::ParseAST()), make it convenient to put here. /// /// FIXME: Make sure the lifetime of Identifiers/Selectors *isn't* tied to /// the lifetime of the preprocessor. SelectorTable Selectors; /// \brief Information about builtins. Builtin::Context BuiltinInfo; /// \brief Tracks all of the pragmas that the client registered /// with this preprocessor. PragmaNamespace *PragmaHandlers; /// \brief Tracks all of the comment handlers that the client registered /// with this preprocessor. std::vector CommentHandlers; /// \brief True if we want to ignore EOF token and continue later on (thus /// avoid tearing the Lexer and etc. down). bool IncrementalProcessing; /// The kind of translation unit we are processing. TranslationUnitKind TUKind; /// \brief The code-completion handler. CodeCompletionHandler *CodeComplete; /// \brief The file that we're performing code-completion for, if any. const FileEntry *CodeCompletionFile; /// \brief The offset in file for the code-completion point. unsigned CodeCompletionOffset; /// \brief The location for the code-completion point. This gets instantiated /// when the CodeCompletionFile gets \#include'ed for preprocessing. SourceLocation CodeCompletionLoc; /// \brief The start location for the file of the code-completion point. /// /// This gets instantiated when the CodeCompletionFile gets \#include'ed /// for preprocessing. SourceLocation CodeCompletionFileLoc; /// \brief The source location of the \c import contextual keyword we just /// lexed, if any. SourceLocation ModuleImportLoc; /// \brief The module import path that we're currently processing. SmallVector, 2> ModuleImportPath; /// \brief Whether the last token we lexed was an '@'. bool LastTokenWasAt; /// \brief Whether the module import expects an identifier next. Otherwise, /// it expects a '.' or ';'. bool ModuleImportExpectsIdentifier; /// \brief The source location of the currently-active /// \#pragma clang arc_cf_code_audited begin. SourceLocation PragmaARCCFCodeAuditedLoc; /// \brief True if we hit the code-completion point. bool CodeCompletionReached; /// \brief The number of bytes that we will initially skip when entering the /// main file, along with a flag that indicates whether skipping this number /// of bytes will place the lexer at the start of a line. /// /// This is used when loading a precompiled preamble. std::pair SkipMainFilePreamble; /// \brief The current top of the stack that we're lexing from if /// not expanding a macro and we are lexing directly from source code. /// /// Only one of CurLexer, CurPTHLexer, or CurTokenLexer will be non-null. std::unique_ptr CurLexer; /// \brief The current top of stack that we're lexing from if /// not expanding from a macro and we are lexing from a PTH cache. /// /// Only one of CurLexer, CurPTHLexer, or CurTokenLexer will be non-null. std::unique_ptr CurPTHLexer; /// \brief The current top of the stack what we're lexing from /// if not expanding a macro. /// /// This is an alias for either CurLexer or CurPTHLexer. PreprocessorLexer *CurPPLexer; /// \brief Used to find the current FileEntry, if CurLexer is non-null /// and if applicable. /// /// This allows us to implement \#include_next and find directory-specific /// properties. const DirectoryLookup *CurDirLookup; /// \brief The current macro we are expanding, if we are expanding a macro. /// /// One of CurLexer and CurTokenLexer must be null. std::unique_ptr CurTokenLexer; /// \brief The kind of lexer we're currently working with. enum CurLexerKind { CLK_Lexer, CLK_PTHLexer, CLK_TokenLexer, CLK_CachingLexer, CLK_LexAfterModuleImport } CurLexerKind; /// \brief If the current lexer is for a submodule that is being built, this /// is that submodule. Module *CurSubmodule; /// \brief Keeps track of the stack of files currently /// \#included, and macros currently being expanded from, not counting /// CurLexer/CurTokenLexer. struct IncludeStackInfo { enum CurLexerKind CurLexerKind; Module *TheSubmodule; std::unique_ptr TheLexer; std::unique_ptr ThePTHLexer; PreprocessorLexer *ThePPLexer; std::unique_ptr TheTokenLexer; const DirectoryLookup *TheDirLookup; // The following constructors are completely useless copies of the default // versions, only needed to pacify MSVC. IncludeStackInfo(enum CurLexerKind CurLexerKind, Module *TheSubmodule, std::unique_ptr &&TheLexer, std::unique_ptr &&ThePTHLexer, PreprocessorLexer *ThePPLexer, std::unique_ptr &&TheTokenLexer, const DirectoryLookup *TheDirLookup) : CurLexerKind(std::move(CurLexerKind)), TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)), ThePTHLexer(std::move(ThePTHLexer)), ThePPLexer(std::move(ThePPLexer)), TheTokenLexer(std::move(TheTokenLexer)), TheDirLookup(std::move(TheDirLookup)) {} IncludeStackInfo(IncludeStackInfo &&RHS) : CurLexerKind(std::move(RHS.CurLexerKind)), TheSubmodule(std::move(RHS.TheSubmodule)), TheLexer(std::move(RHS.TheLexer)), ThePTHLexer(std::move(RHS.ThePTHLexer)), ThePPLexer(std::move(RHS.ThePPLexer)), TheTokenLexer(std::move(RHS.TheTokenLexer)), TheDirLookup(std::move(RHS.TheDirLookup)) {} }; std::vector IncludeMacroStack; /// \brief Actions invoked when some preprocessor activity is /// encountered (e.g. a file is \#included, etc). PPCallbacks *Callbacks; struct MacroExpandsInfo { Token Tok; MacroDirective *MD; SourceRange Range; MacroExpandsInfo(Token Tok, MacroDirective *MD, SourceRange Range) : Tok(Tok), MD(MD), Range(Range) { } }; SmallVector DelayedMacroExpandsCallbacks; /// For each IdentifierInfo that was associated with a macro, we /// keep a mapping to the history of all macro definitions and #undefs in /// the reverse order (the latest one is in the head of the list). llvm::DenseMap Macros; friend class ASTReader; /// \brief Macros that we want to warn because they are not used at the end /// of the translation unit. /// /// We store just their SourceLocations instead of /// something like MacroInfo*. The benefit of this is that when we are /// deserializing from PCH, we don't need to deserialize identifier & macros /// just so that we can report that they are unused, we just warn using /// the SourceLocations of this set (that will be filled by the ASTReader). /// We are using SmallPtrSet instead of a vector for faster removal. typedef llvm::SmallPtrSet WarnUnusedMacroLocsTy; WarnUnusedMacroLocsTy WarnUnusedMacroLocs; /// \brief A "freelist" of MacroArg objects that can be /// reused for quick allocation. MacroArgs *MacroArgCache; friend class MacroArgs; /// For each IdentifierInfo used in a \#pragma push_macro directive, /// we keep a MacroInfo stack used to restore the previous macro value. llvm::DenseMap > PragmaPushMacroInfo; // Various statistics we track for performance analysis. unsigned NumDirectives, NumDefined, NumUndefined, NumPragma; unsigned NumIf, NumElse, NumEndif; unsigned NumEnteredSourceFiles, MaxIncludeStackDepth; unsigned NumMacroExpanded, NumFnMacroExpanded, NumBuiltinMacroExpanded; unsigned NumFastMacroExpanded, NumTokenPaste, NumFastTokenPaste; unsigned NumSkipped; /// \brief The predefined macros that preprocessor should use from the /// command line etc. std::string Predefines; /// \brief The file ID for the preprocessor predefines. FileID PredefinesFileID; /// \{ /// \brief Cache of macro expanders to reduce malloc traffic. enum { TokenLexerCacheSize = 8 }; unsigned NumCachedTokenLexers; TokenLexer *TokenLexerCache[TokenLexerCacheSize]; /// \} /// \brief Keeps macro expanded tokens for TokenLexers. // /// Works like a stack; a TokenLexer adds the macro expanded tokens that is /// going to lex in the cache and when it finishes the tokens are removed /// from the end of the cache. SmallVector MacroExpandedTokens; std::vector > MacroExpandingLexersStack; /// \brief A record of the macro definitions and expansions that /// occurred during preprocessing. /// /// This is an optional side structure that can be enabled with /// \c createPreprocessingRecord() prior to preprocessing. PreprocessingRecord *Record; private: // Cached tokens state. typedef SmallVector CachedTokensTy; /// \brief Cached tokens are stored here when we do backtracking or /// lookahead. They are "lexed" by the CachingLex() method. CachedTokensTy CachedTokens; /// \brief The position of the cached token that CachingLex() should /// "lex" next. /// /// If it points beyond the CachedTokens vector, it means that a normal /// Lex() should be invoked. CachedTokensTy::size_type CachedLexPos; /// \brief Stack of backtrack positions, allowing nested backtracks. /// /// The EnableBacktrackAtThisPos() method pushes a position to /// indicate where CachedLexPos should be set when the BackTrack() method is /// invoked (at which point the last position is popped). std::vector BacktrackPositions; struct MacroInfoChain { MacroInfo MI; MacroInfoChain *Next; MacroInfoChain *Prev; }; /// MacroInfos are managed as a chain for easy disposal. This is the head /// of that list. MacroInfoChain *MIChainHead; /// A "freelist" of MacroInfo objects that can be reused for quick /// allocation. MacroInfoChain *MICache; struct DeserializedMacroInfoChain { MacroInfo MI; unsigned OwningModuleID; // MUST be immediately after the MacroInfo object // so it can be accessed by MacroInfo::getOwningModuleID(). DeserializedMacroInfoChain *Next; }; DeserializedMacroInfoChain *DeserialMIChainHead; public: Preprocessor(IntrusiveRefCntPtr PPOpts, DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup = nullptr, bool OwnsHeaderSearch = false, TranslationUnitKind TUKind = TU_Complete); ~Preprocessor(); /// \brief Initialize the preprocessor using information about the target. /// /// \param Target is owned by the caller and must remain valid for the /// lifetime of the preprocessor. void Initialize(const TargetInfo &Target); /// \brief Retrieve the preprocessor options used to initialize this /// preprocessor. PreprocessorOptions &getPreprocessorOpts() const { return *PPOpts; } DiagnosticsEngine &getDiagnostics() const { return *Diags; } void setDiagnostics(DiagnosticsEngine &D) { Diags = &D; } const LangOptions &getLangOpts() const { return LangOpts; } const TargetInfo &getTargetInfo() const { return *Target; } FileManager &getFileManager() const { return FileMgr; } SourceManager &getSourceManager() const { return SourceMgr; } HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; } IdentifierTable &getIdentifierTable() { return Identifiers; } SelectorTable &getSelectorTable() { return Selectors; } Builtin::Context &getBuiltinInfo() { return BuiltinInfo; } llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; } void setPTHManager(PTHManager* pm); PTHManager *getPTHManager() { return PTH.get(); } void setExternalSource(ExternalPreprocessorSource *Source) { ExternalSource = Source; } ExternalPreprocessorSource *getExternalSource() const { return ExternalSource; } /// \brief Retrieve the module loader associated with this preprocessor. ModuleLoader &getModuleLoader() const { return TheModuleLoader; } bool hadModuleLoaderFatalFailure() const { return TheModuleLoader.HadFatalFailure; } /// \brief True if we are currently preprocessing a #if or #elif directive bool isParsingIfOrElifDirective() const { return ParsingIfOrElifDirective; } /// \brief Control whether the preprocessor retains comments in output. void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) { this->KeepComments = KeepComments | KeepMacroComments; this->KeepMacroComments = KeepMacroComments; } bool getCommentRetentionState() const { return KeepComments; } void setPragmasEnabled(bool Enabled) { PragmasEnabled = Enabled; } bool getPragmasEnabled() const { return PragmasEnabled; } void SetSuppressIncludeNotFoundError(bool Suppress) { SuppressIncludeNotFoundError = Suppress; } bool GetSuppressIncludeNotFoundError() { return SuppressIncludeNotFoundError; } /// Sets whether the preprocessor is responsible for producing output or if /// it is producing tokens to be consumed by Parse and Sema. void setPreprocessedOutput(bool IsPreprocessedOutput) { PreprocessedOutput = IsPreprocessedOutput; } /// Returns true if the preprocessor is responsible for generating output, /// false if it is producing tokens to be consumed by Parse and Sema. bool isPreprocessedOutput() const { return PreprocessedOutput; } /// \brief Return true if we are lexing directly from the specified lexer. bool isCurrentLexer(const PreprocessorLexer *L) const { return CurPPLexer == L; } /// \brief Return the current lexer being lexed from. /// /// Note that this ignores any potentially active macro expansions and _Pragma /// expansions going on at the time. PreprocessorLexer *getCurrentLexer() const { return CurPPLexer; } /// \brief Return the current file lexer being lexed from. /// /// Note that this ignores any potentially active macro expansions and _Pragma /// expansions going on at the time. PreprocessorLexer *getCurrentFileLexer() const; /// \brief Returns the FileID for the preprocessor predefines. FileID getPredefinesFileID() const { return PredefinesFileID; } /// \{ /// \brief Accessors for preprocessor callbacks. /// /// Note that this class takes ownership of any PPCallbacks object given to /// it. PPCallbacks *getPPCallbacks() const { return Callbacks; } void addPPCallbacks(PPCallbacks *C) { if (Callbacks) C = new PPChainedCallbacks(C, Callbacks); Callbacks = C; } /// \} /// \brief Given an identifier, return its latest MacroDirective if it is /// \#defined or null if it isn't \#define'd. MacroDirective *getMacroDirective(IdentifierInfo *II) const { if (!II->hasMacroDefinition()) return nullptr; MacroDirective *MD = getMacroDirectiveHistory(II); assert(MD->isDefined() && "Macro is undefined!"); return MD; } const MacroInfo *getMacroInfo(IdentifierInfo *II) const { return const_cast(this)->getMacroInfo(II); } MacroInfo *getMacroInfo(IdentifierInfo *II) { if (MacroDirective *MD = getMacroDirective(II)) return MD->getMacroInfo(); return nullptr; } /// \brief Given an identifier, return the (probably #undef'd) MacroInfo /// representing the most recent macro definition. /// /// One can iterate over all previous macro definitions from the most recent /// one. This should only be called for identifiers that hadMacroDefinition(). MacroDirective *getMacroDirectiveHistory(const IdentifierInfo *II) const; /// \brief Add a directive to the macro directive history for this identifier. void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD); DefMacroDirective *appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI, SourceLocation Loc, bool isImported) { DefMacroDirective *MD = AllocateDefMacroDirective(MI, Loc, isImported); appendMacroDirective(II, MD); return MD; } DefMacroDirective *appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI){ return appendDefMacroDirective(II, MI, MI->getDefinitionLoc(), false); } /// \brief Set a MacroDirective that was loaded from a PCH file. void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *MD); /// \{ /// Iterators for the macro history table. Currently defined macros have /// IdentifierInfo::hasMacroDefinition() set and an empty /// MacroInfo::getUndefLoc() at the head of the list. typedef llvm::DenseMap::const_iterator macro_iterator; macro_iterator macro_begin(bool IncludeExternalMacros = true) const; macro_iterator macro_end(bool IncludeExternalMacros = true) const; /// \} /// \brief Return the name of the macro defined before \p Loc that has /// spelling \p Tokens. If there are multiple macros with same spelling, /// return the last one defined. StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef Tokens) const; const std::string &getPredefines() const { return Predefines; } /// \brief Set the predefines for this Preprocessor. /// /// These predefines are automatically injected when parsing the main file. void setPredefines(const char *P) { Predefines = P; } void setPredefines(const std::string &P) { Predefines = P; } /// Return information about the specified preprocessor /// identifier token. IdentifierInfo *getIdentifierInfo(StringRef Name) const { return &Identifiers.get(Name); } /// \brief Add the specified pragma handler to this preprocessor. /// /// If \p Namespace is non-null, then it is a token required to exist on the /// pragma line before the pragma string starts, e.g. "STDC" or "GCC". void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler); void AddPragmaHandler(PragmaHandler *Handler) { AddPragmaHandler(StringRef(), Handler); } /// \brief Remove the specific pragma handler from this preprocessor. /// /// If \p Namespace is non-null, then it should be the namespace that /// \p Handler was added to. It is an error to remove a handler that /// has not been registered. void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler); void RemovePragmaHandler(PragmaHandler *Handler) { RemovePragmaHandler(StringRef(), Handler); } /// Install empty handlers for all pragmas (making them ignored). void IgnorePragmas(); /// \brief Add the specified comment handler to the preprocessor. void addCommentHandler(CommentHandler *Handler); /// \brief Remove the specified comment handler. /// /// It is an error to remove a handler that has not been registered. void removeCommentHandler(CommentHandler *Handler); /// \brief Set the code completion handler to the given object. void setCodeCompletionHandler(CodeCompletionHandler &Handler) { CodeComplete = &Handler; } /// \brief Retrieve the current code-completion handler. CodeCompletionHandler *getCodeCompletionHandler() const { return CodeComplete; } /// \brief Clear out the code completion handler. void clearCodeCompletionHandler() { CodeComplete = nullptr; } /// \brief Hook used by the lexer to invoke the "natural language" code /// completion point. void CodeCompleteNaturalLanguage(); /// \brief Retrieve the preprocessing record, or NULL if there is no /// preprocessing record. PreprocessingRecord *getPreprocessingRecord() const { return Record; } /// \brief Create a new preprocessing record, which will keep track of /// all macro expansions, macro definitions, etc. void createPreprocessingRecord(); /// \brief Enter the specified FileID as the main source file, /// which implicitly adds the builtin defines etc. void EnterMainSourceFile(); /// \brief Inform the preprocessor callbacks that processing is complete. void EndSourceFile(); /// \brief Add a source file to the top of the include stack and /// start lexing tokens from it instead of the current buffer. /// /// Emits a diagnostic, doesn't enter the file, and returns true on error. bool EnterSourceFile(FileID CurFileID, const DirectoryLookup *Dir, SourceLocation Loc); /// \brief Add a Macro to the top of the include stack and start lexing /// tokens from it instead of the current buffer. /// /// \param Args specifies the tokens input to a function-like macro. /// \param ILEnd specifies the location of the ')' for a function-like macro /// or the identifier for an object-like macro. void EnterMacro(Token &Identifier, SourceLocation ILEnd, MacroInfo *Macro, MacroArgs *Args); /// \brief Add a "macro" context to the top of the include stack, /// which will cause the lexer to start returning the specified tokens. /// /// If \p DisableMacroExpansion is true, tokens lexed from the token stream /// will not be subject to further macro expansion. Otherwise, these tokens /// will be re-macro-expanded when/if expansion is enabled. /// /// If \p OwnsTokens is false, this method assumes that the specified stream /// of tokens has a permanent owner somewhere, so they do not need to be /// copied. If it is true, it assumes the array of tokens is allocated with /// \c new[] and must be freed. void EnterTokenStream(const Token *Toks, unsigned NumToks, bool DisableMacroExpansion, bool OwnsTokens); /// \brief Pop the current lexer/macro exp off the top of the lexer stack. /// /// This should only be used in situations where the current state of the /// top-of-stack lexer is known. void RemoveTopOfLexerStack(); /// From the point that this method is called, and until /// CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor /// keeps track of the lexed tokens so that a subsequent Backtrack() call will /// make the Preprocessor re-lex the same tokens. /// /// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can /// be called multiple times and CommitBacktrackedTokens/Backtrack calls will /// be combined with the EnableBacktrackAtThisPos calls in reverse order. /// /// NOTE: *DO NOT* forget to call either CommitBacktrackedTokens or Backtrack /// at some point after EnableBacktrackAtThisPos. If you don't, caching of /// tokens will continue indefinitely. /// void EnableBacktrackAtThisPos(); /// \brief Disable the last EnableBacktrackAtThisPos call. void CommitBacktrackedTokens(); /// \brief Make Preprocessor re-lex the tokens that were lexed since /// EnableBacktrackAtThisPos() was previously called. void Backtrack(); /// \brief True if EnableBacktrackAtThisPos() was called and /// caching of tokens is on. bool isBacktrackEnabled() const { return !BacktrackPositions.empty(); } /// \brief Lex the next token for this preprocessor. void Lex(Token &Result); void LexAfterModuleImport(Token &Result); /// \brief Lex a string literal, which may be the concatenation of multiple /// string literals and may even come from macro expansion. /// \returns true on success, false if a error diagnostic has been generated. bool LexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion) { if (AllowMacroExpansion) Lex(Result); else LexUnexpandedToken(Result); return FinishLexStringLiteral(Result, String, DiagnosticTag, AllowMacroExpansion); } /// \brief Complete the lexing of a string literal where the first token has /// already been lexed (see LexStringLiteral). bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion); /// \brief Lex a token. If it's a comment, keep lexing until we get /// something not a comment. /// /// This is useful in -E -C mode where comments would foul up preprocessor /// directive handling. void LexNonComment(Token &Result) { do Lex(Result); while (Result.getKind() == tok::comment); } /// \brief Just like Lex, but disables macro expansion of identifier tokens. void LexUnexpandedToken(Token &Result) { // Disable macro expansion. bool OldVal = DisableMacroExpansion; DisableMacroExpansion = true; // Lex the token. Lex(Result); // Reenable it. DisableMacroExpansion = OldVal; } /// \brief Like LexNonComment, but this disables macro expansion of /// identifier tokens. void LexUnexpandedNonComment(Token &Result) { do LexUnexpandedToken(Result); while (Result.getKind() == tok::comment); } /// \brief Parses a simple integer literal to get its numeric value. Floating /// point literals and user defined literals are rejected. Used primarily to /// handle pragmas that accept integer arguments. bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value); /// Disables macro expansion everywhere except for preprocessor directives. void SetMacroExpansionOnlyInDirectives() { DisableMacroExpansion = true; MacroExpansionInDirectivesOverride = true; } /// \brief Peeks ahead N tokens and returns that token without consuming any /// tokens. /// /// LookAhead(0) returns the next token that would be returned by Lex(), /// LookAhead(1) returns the token after it, etc. This returns normal /// tokens after phase 5. As such, it is equivalent to using /// 'Lex', not 'LexUnexpandedToken'. const Token &LookAhead(unsigned N) { if (CachedLexPos + N < CachedTokens.size()) return CachedTokens[CachedLexPos+N]; else return PeekAhead(N+1); } /// \brief When backtracking is enabled and tokens are cached, /// this allows to revert a specific number of tokens. /// /// Note that the number of tokens being reverted should be up to the last /// backtrack position, not more. void RevertCachedTokens(unsigned N) { assert(isBacktrackEnabled() && "Should only be called when tokens are cached for backtracking"); assert(signed(CachedLexPos) - signed(N) >= signed(BacktrackPositions.back()) && "Should revert tokens up to the last backtrack position, not more"); assert(signed(CachedLexPos) - signed(N) >= 0 && "Corrupted backtrack positions ?"); CachedLexPos -= N; } /// \brief Enters a token in the token stream to be lexed next. /// /// If BackTrack() is called afterwards, the token will remain at the /// insertion point. void EnterToken(const Token &Tok) { EnterCachingLexMode(); CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok); } /// We notify the Preprocessor that if it is caching tokens (because /// backtrack is enabled) it should replace the most recent cached tokens /// with the given annotation token. This function has no effect if /// backtracking is not enabled. /// /// Note that the use of this function is just for optimization, so that the /// cached tokens doesn't get re-parsed and re-resolved after a backtrack is /// invoked. void AnnotateCachedTokens(const Token &Tok) { assert(Tok.isAnnotation() && "Expected annotation token"); if (CachedLexPos != 0 && isBacktrackEnabled()) AnnotatePreviousCachedTokens(Tok); } /// Get the location of the last cached token, suitable for setting the end /// location of an annotation token. SourceLocation getLastCachedTokenLocation() const { assert(CachedLexPos != 0); return CachedTokens[CachedLexPos-1].getLocation(); } /// \brief Replace the last token with an annotation token. /// /// Like AnnotateCachedTokens(), this routine replaces an /// already-parsed (and resolved) token with an annotation /// token. However, this routine only replaces the last token with /// the annotation token; it does not affect any other cached /// tokens. This function has no effect if backtracking is not /// enabled. void ReplaceLastTokenWithAnnotation(const Token &Tok) { assert(Tok.isAnnotation() && "Expected annotation token"); if (CachedLexPos != 0 && isBacktrackEnabled()) CachedTokens[CachedLexPos-1] = Tok; } /// Update the current token to represent the provided /// identifier, in order to cache an action performed by typo correction. void TypoCorrectToken(const Token &Tok) { assert(Tok.getIdentifierInfo() && "Expected identifier token"); if (CachedLexPos != 0 && isBacktrackEnabled()) CachedTokens[CachedLexPos-1] = Tok; } /// \brief Recompute the current lexer kind based on the CurLexer/CurPTHLexer/ /// CurTokenLexer pointers. void recomputeCurLexerKind(); /// \brief Returns true if incremental processing is enabled bool isIncrementalProcessingEnabled() const { return IncrementalProcessing; } /// \brief Enables the incremental processing void enableIncrementalProcessing(bool value = true) { IncrementalProcessing = value; } /// \brief Specify the point at which code-completion will be performed. /// /// \param File the file in which code completion should occur. If /// this file is included multiple times, code-completion will /// perform completion the first time it is included. If NULL, this /// function clears out the code-completion point. /// /// \param Line the line at which code completion should occur /// (1-based). /// /// \param Column the column at which code completion should occur /// (1-based). /// /// \returns true if an error occurred, false otherwise. bool SetCodeCompletionPoint(const FileEntry *File, unsigned Line, unsigned Column); /// \brief Determine if we are performing code completion. bool isCodeCompletionEnabled() const { return CodeCompletionFile != nullptr; } /// \brief Returns the location of the code-completion point. /// /// Returns an invalid location if code-completion is not enabled or the file /// containing the code-completion point has not been lexed yet. SourceLocation getCodeCompletionLoc() const { return CodeCompletionLoc; } /// \brief Returns the start location of the file of code-completion point. /// /// Returns an invalid location if code-completion is not enabled or the file /// containing the code-completion point has not been lexed yet. SourceLocation getCodeCompletionFileLoc() const { return CodeCompletionFileLoc; } /// \brief Returns true if code-completion is enabled and we have hit the /// code-completion point. bool isCodeCompletionReached() const { return CodeCompletionReached; } /// \brief Note that we hit the code-completion point. void setCodeCompletionReached() { assert(isCodeCompletionEnabled() && "Code-completion not enabled!"); CodeCompletionReached = true; // Silence any diagnostics that occur after we hit the code-completion. getDiagnostics().setSuppressAllDiagnostics(true); } /// \brief The location of the currently-active \#pragma clang /// arc_cf_code_audited begin. /// /// Returns an invalid location if there is no such pragma active. SourceLocation getPragmaARCCFCodeAuditedLoc() const { return PragmaARCCFCodeAuditedLoc; } /// \brief Set the location of the currently-active \#pragma clang /// arc_cf_code_audited begin. An invalid location ends the pragma. void setPragmaARCCFCodeAuditedLoc(SourceLocation Loc) { PragmaARCCFCodeAuditedLoc = Loc; } /// \brief Instruct the preprocessor to skip part of the main source file. /// /// \param Bytes The number of bytes in the preamble to skip. /// /// \param StartOfLine Whether skipping these bytes puts the lexer at the /// start of a line. void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine) { SkipMainFilePreamble.first = Bytes; SkipMainFilePreamble.second = StartOfLine; } /// Forwarding function for diagnostics. This emits a diagnostic at /// the specified Token's location, translating the token's start /// position in the current buffer into a SourcePosition object for rendering. DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const { return Diags->Report(Loc, DiagID); } DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const { return Diags->Report(Tok.getLocation(), DiagID); } /// Return the 'spelling' of the token at the given /// location; does not go up to the spelling location or down to the /// expansion location. /// /// \param buffer A buffer which will be used only if the token requires /// "cleaning", e.g. if it contains trigraphs or escaped newlines /// \param invalid If non-null, will be set \c true if an error occurs. StringRef getSpelling(SourceLocation loc, SmallVectorImpl &buffer, bool *invalid = nullptr) const { return Lexer::getSpelling(loc, buffer, SourceMgr, LangOpts, invalid); } /// \brief Return the 'spelling' of the Tok token. /// /// The spelling of a token is the characters used to represent the token in /// the source file after trigraph expansion and escaped-newline folding. In /// particular, this wants to get the true, uncanonicalized, spelling of /// things like digraphs, UCNs, etc. /// /// \param Invalid If non-null, will be set \c true if an error occurs. std::string getSpelling(const Token &Tok, bool *Invalid = nullptr) const { return Lexer::getSpelling(Tok, SourceMgr, LangOpts, Invalid); } /// \brief Get the spelling of a token into a preallocated buffer, instead /// of as an std::string. /// /// The caller is required to allocate enough space for the token, which is /// guaranteed to be at least Tok.getLength() bytes long. The length of the /// actual result is returned. /// /// Note that this method may do two possible things: it may either fill in /// the buffer specified with characters, or it may *change the input pointer* /// to point to a constant buffer with the data already in it (avoiding a /// copy). The caller is not allowed to modify the returned buffer pointer /// if an internal buffer is returned. unsigned getSpelling(const Token &Tok, const char *&Buffer, bool *Invalid = nullptr) const { return Lexer::getSpelling(Tok, Buffer, SourceMgr, LangOpts, Invalid); } /// \brief Get the spelling of a token into a SmallVector. /// /// Note that the returned StringRef may not point to the /// supplied buffer if a copy can be avoided. StringRef getSpelling(const Token &Tok, SmallVectorImpl &Buffer, bool *Invalid = nullptr) const; /// \brief Relex the token at the specified location. /// \returns true if there was a failure, false on success. bool getRawToken(SourceLocation Loc, Token &Result, bool IgnoreWhiteSpace = false) { return Lexer::getRawToken(Loc, Result, SourceMgr, LangOpts, IgnoreWhiteSpace); } /// \brief Given a Token \p Tok that is a numeric constant with length 1, /// return the character. char getSpellingOfSingleCharacterNumericConstant(const Token &Tok, bool *Invalid = nullptr) const { assert(Tok.is(tok::numeric_constant) && Tok.getLength() == 1 && "Called on unsupported token"); assert(!Tok.needsCleaning() && "Token can't need cleaning with length 1"); // If the token is carrying a literal data pointer, just use it. if (const char *D = Tok.getLiteralData()) return *D; // Otherwise, fall back on getCharacterData, which is slower, but always // works. return *SourceMgr.getCharacterData(Tok.getLocation(), Invalid); } /// \brief Retrieve the name of the immediate macro expansion. /// /// This routine starts from a source location, and finds the name of the /// macro responsible for its immediate expansion. It looks through any /// intervening macro argument expansions to compute this. It returns a /// StringRef that refers to the SourceManager-owned buffer of the source /// where that macro name is spelled. Thus, the result shouldn't out-live /// the SourceManager. StringRef getImmediateMacroName(SourceLocation Loc) { return Lexer::getImmediateMacroName(Loc, SourceMgr, getLangOpts()); } /// \brief Plop the specified string into a scratch buffer and set the /// specified token's location and length to it. /// /// If specified, the source location provides a location of the expansion /// point of the token. void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart = SourceLocation(), SourceLocation ExpansionLocEnd = SourceLocation()); /// \brief Computes the source location just past the end of the /// token at this source location. /// /// This routine can be used to produce a source location that /// points just past the end of the token referenced by \p Loc, and /// is generally used when a diagnostic needs to point just after a /// token where it expected something different that it received. If /// the returned source location would not be meaningful (e.g., if /// it points into a macro), this routine returns an invalid /// source location. /// /// \param Offset an offset from the end of the token, where the source /// location should refer to. The default offset (0) produces a source /// location pointing just past the end of the token; an offset of 1 produces /// a source location pointing to the last character in the token, etc. SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset = 0) { return Lexer::getLocForEndOfToken(Loc, Offset, SourceMgr, LangOpts); } /// \brief Returns true if the given MacroID location points at the first /// token of the macro expansion. /// /// \param MacroBegin If non-null and function returns true, it is set to /// begin location of the macro. bool isAtStartOfMacroExpansion(SourceLocation loc, SourceLocation *MacroBegin = nullptr) const { return Lexer::isAtStartOfMacroExpansion(loc, SourceMgr, LangOpts, MacroBegin); } /// \brief Returns true if the given MacroID location points at the last /// token of the macro expansion. /// /// \param MacroEnd If non-null and function returns true, it is set to /// end location of the macro. bool isAtEndOfMacroExpansion(SourceLocation loc, SourceLocation *MacroEnd = nullptr) const { return Lexer::isAtEndOfMacroExpansion(loc, SourceMgr, LangOpts, MacroEnd); } /// \brief Print the token to stderr, used for debugging. void DumpToken(const Token &Tok, bool DumpFlags = false) const; void DumpLocation(SourceLocation Loc) const; void DumpMacro(const MacroInfo &MI) const; /// \brief Given a location that specifies the start of a /// token, return a new location that specifies a character within the token. SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Char) const { return Lexer::AdvanceToTokenCharacter(TokStart, Char, SourceMgr, LangOpts); } /// \brief Increment the counters for the number of token paste operations /// performed. /// /// If fast was specified, this is a 'fast paste' case we handled. void IncrementPasteCounter(bool isFast) { if (isFast) ++NumFastTokenPaste; else ++NumTokenPaste; } void PrintStats(); size_t getTotalMemory() const; /// When the macro expander pastes together a comment (/##/) in Microsoft /// mode, this method handles updating the current state, returning the /// token on the next source line. void HandleMicrosoftCommentPaste(Token &Tok); //===--------------------------------------------------------------------===// // Preprocessor callback methods. These are invoked by a lexer as various // directives and events are found. /// Given a tok::raw_identifier token, look up the /// identifier information for the token and install it into the token, /// updating the token kind accordingly. IdentifierInfo *LookUpIdentifierInfo(Token &Identifier) const; private: llvm::DenseMap PoisonReasons; public: /// \brief Specifies the reason for poisoning an identifier. /// /// If that identifier is accessed while poisoned, then this reason will be /// used instead of the default "poisoned" diagnostic. void SetPoisonReason(IdentifierInfo *II, unsigned DiagID); /// \brief Display reason for poisoned identifier. void HandlePoisonedIdentifier(Token & Tok); void MaybeHandlePoisonedIdentifier(Token & Identifier) { if(IdentifierInfo * II = Identifier.getIdentifierInfo()) { if(II->isPoisoned()) { HandlePoisonedIdentifier(Identifier); } } } private: /// Identifiers used for SEH handling in Borland. These are only /// allowed in particular circumstances // __except block IdentifierInfo *Ident__exception_code, *Ident___exception_code, *Ident_GetExceptionCode; // __except filter expression IdentifierInfo *Ident__exception_info, *Ident___exception_info, *Ident_GetExceptionInfo; // __finally IdentifierInfo *Ident__abnormal_termination, *Ident___abnormal_termination, *Ident_AbnormalTermination; const char *getCurLexerEndPos(); public: void PoisonSEHIdentifiers(bool Poison = true); // Borland /// \brief Callback invoked when the lexer reads an identifier and has /// filled in the tokens IdentifierInfo member. /// /// This callback potentially macro expands it or turns it into a named /// token (like 'for'). /// /// \returns true if we actually computed a token, false if we need to /// lex again. bool HandleIdentifier(Token &Identifier); /// \brief Callback invoked when the lexer hits the end of the current file. /// /// This either returns the EOF token and returns true, or /// pops a level off the include stack and returns false, at which point the /// client should call lex again. bool HandleEndOfFile(Token &Result, bool isEndOfMacro = false); /// \brief Callback invoked when the current TokenLexer hits the end of its /// token stream. bool HandleEndOfTokenLexer(Token &Result); /// \brief Callback invoked when the lexer sees a # token at the start of a /// line. /// /// This consumes the directive, modifies the lexer/preprocessor state, and /// advances the lexer(s) so that the next token read is the correct one. void HandleDirective(Token &Result); /// \brief Ensure that the next token is a tok::eod token. /// /// If not, emit a diagnostic and consume up until the eod. /// If \p EnableMacros is true, then we consider macros that expand to zero /// tokens as being ok. void CheckEndOfDirective(const char *Directive, bool EnableMacros = false); /// \brief Read and discard all tokens remaining on the current line until /// the tok::eod token is found. void DiscardUntilEndOfDirective(); /// \brief Returns true if the preprocessor has seen a use of /// __DATE__ or __TIME__ in the file so far. bool SawDateOrTime() const { return DATELoc != SourceLocation() || TIMELoc != SourceLocation(); } unsigned getCounterValue() const { return CounterValue; } void setCounterValue(unsigned V) { CounterValue = V; } /// \brief Retrieves the module that we're currently building, if any. Module *getCurrentModule(); /// \brief Allocate a new MacroInfo object with the provided SourceLocation. MacroInfo *AllocateMacroInfo(SourceLocation L); /// \brief Allocate a new MacroInfo object loaded from an AST file. MacroInfo *AllocateDeserializedMacroInfo(SourceLocation L, unsigned SubModuleID); /// \brief Turn the specified lexer token into a fully checked and spelled /// filename, e.g. as an operand of \#include. /// /// The caller is expected to provide a buffer that is large enough to hold /// the spelling of the filename, but is also expected to handle the case /// when this method decides to use a different buffer. /// /// \returns true if the input filename was in <>'s or false if it was /// in ""'s. bool GetIncludeFilenameSpelling(SourceLocation Loc,StringRef &Filename); /// \brief Given a "foo" or \ reference, look up the indicated file. /// /// Returns null on failure. \p isAngled indicates whether the file /// reference is for system \#include's or not (i.e. using <> instead of ""). const FileEntry *LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, const DirectoryLookup *FromDir, const DirectoryLookup *&CurDir, SmallVectorImpl *SearchPath, SmallVectorImpl *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool SkipCache = false); /// \brief Get the DirectoryLookup structure used to find the current /// FileEntry, if CurLexer is non-null and if applicable. /// /// This allows us to implement \#include_next and find directory-specific /// properties. const DirectoryLookup *GetCurDirLookup() { return CurDirLookup; } /// \brief Return true if we're in the top-level file, not in a \#include. bool isInPrimaryFile() const; /// \brief Handle cases where the \#include name is expanded /// from a macro as multiple tokens, which need to be glued together. /// /// This occurs for code like: /// \code /// \#define FOO /// \#include FOO /// \endcode /// because in this case, "" is returned as 7 tokens, not one. /// /// This code concatenates and consumes tokens up to the '>' token. It /// returns false if the > was found, otherwise it returns true if it finds /// and consumes the EOD marker. bool ConcatenateIncludeName(SmallString<128> &FilenameBuffer, SourceLocation &End); /// \brief Lex an on-off-switch (C99 6.10.6p2) and verify that it is /// followed by EOD. Return true if the token is not a valid on-off-switch. bool LexOnOffSwitch(tok::OnOffSwitch &OOS); bool CheckMacroName(Token &MacroNameTok, char isDefineUndef); private: void PushIncludeMacroStack() { IncludeMacroStack.push_back(IncludeStackInfo( CurLexerKind, CurSubmodule, std::move(CurLexer), std::move(CurPTHLexer), CurPPLexer, std::move(CurTokenLexer), CurDirLookup)); CurPPLexer = nullptr; } void PopIncludeMacroStack() { CurLexer = std::move(IncludeMacroStack.back().TheLexer); CurPTHLexer = std::move(IncludeMacroStack.back().ThePTHLexer); CurPPLexer = IncludeMacroStack.back().ThePPLexer; CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer); CurDirLookup = IncludeMacroStack.back().TheDirLookup; CurSubmodule = IncludeMacroStack.back().TheSubmodule; CurLexerKind = IncludeMacroStack.back().CurLexerKind; IncludeMacroStack.pop_back(); } void PropagateLineStartLeadingSpaceInfo(Token &Result); /// \brief Allocate a new MacroInfo object. MacroInfo *AllocateMacroInfo(); DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI, SourceLocation Loc, bool isImported); UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc); VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc, bool isPublic); /// \brief Release the specified MacroInfo for re-use. /// /// This memory will be reused for allocating new MacroInfo objects. void ReleaseMacroInfo(MacroInfo* MI); /// \brief Lex and validate a macro name, which occurs after a /// \#define or \#undef. /// /// This emits a diagnostic, sets the token kind to eod, /// and discards the rest of the macro line if the macro name is invalid. void ReadMacroName(Token &MacroNameTok, char isDefineUndef = 0); /// The ( starting an argument list of a macro definition has just been read. /// Lex the rest of the arguments and the closing ), updating \p MI with /// what we learn and saving in \p LastTok the last token read. /// Return true if an error occurs parsing the arg list. bool ReadMacroDefinitionArgList(MacroInfo *MI, Token& LastTok); /// We just read a \#if or related directive and decided that the /// subsequent tokens are in the \#if'd out portion of the /// file. Lex the rest of the file, until we see an \#endif. If \p /// FoundNonSkipPortion is true, then we have already emitted code for part of /// this \#if directive, so \#else/\#elif blocks should never be entered. If /// \p FoundElse is false, then \#else directives are ok, if not, then we have /// already seen one so a \#else directive is a duplicate. When this returns, /// the caller can lex the first valid token. void SkipExcludedConditionalBlock(SourceLocation IfTokenLoc, bool FoundNonSkipPortion, bool FoundElse, SourceLocation ElseLoc = SourceLocation()); /// \brief A fast PTH version of SkipExcludedConditionalBlock. void PTHSkipExcludedConditionalBlock(); /// \brief Evaluate an integer constant expression that may occur after a /// \#if or \#elif directive and return it as a bool. /// /// If the expression is equivalent to "!defined(X)" return X in IfNDefMacro. bool EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro); /// \brief Install the standard preprocessor pragmas: /// \#pragma GCC poison/system_header/dependency and \#pragma once. void RegisterBuiltinPragmas(); /// \brief Register builtin macros such as __LINE__ with the identifier table. void RegisterBuiltinMacros(); /// If an identifier token is read that is to be expanded as a macro, handle /// it and return the next token as 'Tok'. If we lexed a token, return true; /// otherwise the caller should lex again. bool HandleMacroExpandedIdentifier(Token &Tok, MacroDirective *MD); /// \brief Cache macro expanded tokens for TokenLexers. // /// Works like a stack; a TokenLexer adds the macro expanded tokens that is /// going to lex in the cache and when it finishes the tokens are removed /// from the end of the cache. Token *cacheMacroExpandedTokens(TokenLexer *tokLexer, ArrayRef tokens); void removeCachedMacroExpandedTokensOfLastLexer(); friend void TokenLexer::ExpandFunctionArguments(); /// Determine whether the next preprocessor token to be /// lexed is a '('. If so, consume the token and return true, if not, this /// method should have no observable side-effect on the lexed tokens. bool isNextPPTokenLParen(); /// After reading "MACRO(", this method is invoked to read all of the formal /// arguments specified for the macro invocation. Returns null on error. MacroArgs *ReadFunctionLikeMacroArgs(Token &MacroName, MacroInfo *MI, SourceLocation &ExpansionEnd); /// \brief If an identifier token is read that is to be expanded /// as a builtin macro, handle it and return the next token as 'Tok'. void ExpandBuiltinMacro(Token &Tok); /// \brief Read a \c _Pragma directive, slice it up, process it, then /// return the first token after the directive. /// This assumes that the \c _Pragma token has just been read into \p Tok. void Handle_Pragma(Token &Tok); /// \brief Like Handle_Pragma except the pragma text is not enclosed within /// a string literal. void HandleMicrosoft__pragma(Token &Tok); /// \brief Add a lexer to the top of the include stack and /// start lexing tokens from it instead of the current buffer. void EnterSourceFileWithLexer(Lexer *TheLexer, const DirectoryLookup *Dir); /// \brief Add a lexer to the top of the include stack and /// start getting tokens from it using the PTH cache. void EnterSourceFileWithPTH(PTHLexer *PL, const DirectoryLookup *Dir); /// \brief Set the FileID for the preprocessor predefines. void setPredefinesFileID(FileID FID) { assert(PredefinesFileID.isInvalid() && "PredefinesFileID already set!"); PredefinesFileID = FID; } /// \brief Returns true if we are lexing from a file and not a /// pragma or a macro. static bool IsFileLexer(const Lexer* L, const PreprocessorLexer* P) { return L ? !L->isPragmaLexer() : P != nullptr; } static bool IsFileLexer(const IncludeStackInfo& I) { return IsFileLexer(I.TheLexer.get(), I.ThePPLexer); } bool IsFileLexer() const { return IsFileLexer(CurLexer.get(), CurPPLexer); } //===--------------------------------------------------------------------===// // Caching stuff. void CachingLex(Token &Result); bool InCachingLexMode() const { // If the Lexer pointers are 0 and IncludeMacroStack is empty, it means // that we are past EOF, not that we are in CachingLex mode. return !CurPPLexer && !CurTokenLexer && !CurPTHLexer && !IncludeMacroStack.empty(); } void EnterCachingLexMode(); void ExitCachingLexMode() { if (InCachingLexMode()) RemoveTopOfLexerStack(); } const Token &PeekAhead(unsigned N); void AnnotatePreviousCachedTokens(const Token &Tok); //===--------------------------------------------------------------------===// /// Handle*Directive - implement the various preprocessor directives. These /// should side-effect the current preprocessor object so that the next call /// to Lex() will return the appropriate token next. void HandleLineDirective(Token &Tok); void HandleDigitDirective(Token &Tok); void HandleUserDiagnosticDirective(Token &Tok, bool isWarning); void HandleIdentSCCSDirective(Token &Tok); void HandleMacroPublicDirective(Token &Tok); void HandleMacroPrivateDirective(Token &Tok); // File inclusion. void HandleIncludeDirective(SourceLocation HashLoc, Token &Tok, const DirectoryLookup *LookupFrom = nullptr, bool isImport = false); void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok); void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok); void HandleImportDirective(SourceLocation HashLoc, Token &Tok); void HandleMicrosoftImportDirective(Token &Tok); // Module inclusion testing. /// \brief Find the module for the source or header file that \p FilenameLoc /// points to. Module *getModuleForLocation(SourceLocation FilenameLoc); // Macro handling. void HandleDefineDirective(Token &Tok, bool ImmediatelyAfterTopLevelIfndef); void HandleUndefDirective(Token &Tok); // Conditional Inclusion. void HandleIfdefDirective(Token &Tok, bool isIfndef, bool ReadAnyTokensBeforeDirective); void HandleIfDirective(Token &Tok, bool ReadAnyTokensBeforeDirective); void HandleEndifDirective(Token &Tok); void HandleElseDirective(Token &Tok); void HandleElifDirective(Token &Tok); // Pragmas. void HandlePragmaDirective(SourceLocation IntroducerLoc, PragmaIntroducerKind Introducer); public: void HandlePragmaOnce(Token &OnceTok); void HandlePragmaMark(); void HandlePragmaPoison(Token &PoisonTok); void HandlePragmaSystemHeader(Token &SysHeaderTok); void HandlePragmaDependency(Token &DependencyTok); void HandlePragmaPushMacro(Token &Tok); void HandlePragmaPopMacro(Token &Tok); void HandlePragmaIncludeAlias(Token &Tok); IdentifierInfo *ParsePragmaPushOrPopMacro(Token &Tok); // Return true and store the first token only if any CommentHandler // has inserted some tokens and getCommentRetentionState() is false. bool HandleComment(Token &Token, SourceRange Comment); /// \brief A macro is used, update information about macros that need unused /// warnings. void markMacroAsUsed(MacroInfo *MI); }; /// \brief Abstract base class that describes a handler that will receive /// source ranges for each of the comments encountered in the source file. class CommentHandler { public: virtual ~CommentHandler(); // The handler shall return true if it has pushed any tokens // to be read using e.g. EnterToken or EnterTokenStream. virtual bool HandleComment(Preprocessor &PP, SourceRange Comment) = 0; }; } // end namespace clang #endif