1 //===--- Preprocessor.h - C Language Family Preprocessor --------*- C++ -*-===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file defines the Preprocessor interface.
12 //===----------------------------------------------------------------------===//
14 #ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
15 #define LLVM_CLANG_LEX_PREPROCESSOR_H
17 #include "clang/Lex/MacroInfo.h"
18 #include "clang/Lex/Lexer.h"
19 #include "clang/Lex/PTHLexer.h"
20 #include "clang/Lex/PPCallbacks.h"
21 #include "clang/Lex/PPMutationListener.h"
22 #include "clang/Lex/TokenLexer.h"
23 #include "clang/Lex/PTHManager.h"
24 #include "clang/Basic/Builtins.h"
25 #include "clang/Basic/Diagnostic.h"
26 #include "clang/Basic/IdentifierTable.h"
27 #include "clang/Basic/SourceLocation.h"
28 #include "llvm/ADT/DenseMap.h"
29 #include "llvm/ADT/IntrusiveRefCntPtr.h"
30 #include "llvm/ADT/SmallPtrSet.h"
31 #include "llvm/ADT/OwningPtr.h"
32 #include "llvm/ADT/SmallVector.h"
33 #include "llvm/ADT/ArrayRef.h"
34 #include "llvm/Support/Allocator.h"
38 template<unsigned InternalLen> class SmallString;
44 class ExternalPreprocessorSource;
48 class PragmaNamespace;
54 class CodeCompletionHandler;
55 class DirectoryLookup;
56 class PreprocessingRecord;
58 class PreprocessorOptions;
60 /// \brief Stores token information for comparing actual tokens with
61 /// predefined values. Only handles simple tokens and identifiers.
67 TokenValue(tok::TokenKind Kind) : Kind(Kind), II(0) {
68 assert(Kind != tok::raw_identifier && "Raw identifiers are not supported.");
69 assert(Kind != tok::identifier &&
70 "Identifiers should be created by TokenValue(IdentifierInfo *)");
71 assert(!tok::isLiteral(Kind) && "Literals are not supported.");
72 assert(!tok::isAnnotation(Kind) && "Annotations are not supported.");
74 TokenValue(IdentifierInfo *II) : Kind(tok::identifier), II(II) {}
75 bool operator==(const Token &Tok) const {
76 return Tok.getKind() == Kind &&
77 (!II || II == Tok.getIdentifierInfo());
81 /// Preprocessor - This object engages in a tight little dance with the lexer to
82 /// efficiently preprocess tokens. Lexers know only about tokens within a
83 /// single source file, and don't know anything about preprocessor-level issues
84 /// like the \#include stack, token expansion, etc.
86 class Preprocessor : public RefCountedBase<Preprocessor> {
87 llvm::IntrusiveRefCntPtr<PreprocessorOptions> PPOpts;
88 DiagnosticsEngine *Diags;
89 LangOptions &LangOpts;
90 const TargetInfo *Target;
92 SourceManager &SourceMgr;
93 ScratchBuffer *ScratchBuf;
94 HeaderSearch &HeaderInfo;
95 ModuleLoader &TheModuleLoader;
97 /// \brief External source of macros.
98 ExternalPreprocessorSource *ExternalSource;
101 /// PTH - An optional PTHManager object used for getting tokens from
102 /// a token cache rather than lexing the original source file.
103 OwningPtr<PTHManager> PTH;
105 /// BP - A BumpPtrAllocator object used to quickly allocate and release
106 /// objects internal to the Preprocessor.
107 llvm::BumpPtrAllocator BP;
109 /// Identifiers for builtin macros and other builtins.
110 IdentifierInfo *Ident__LINE__, *Ident__FILE__; // __LINE__, __FILE__
111 IdentifierInfo *Ident__DATE__, *Ident__TIME__; // __DATE__, __TIME__
112 IdentifierInfo *Ident__INCLUDE_LEVEL__; // __INCLUDE_LEVEL__
113 IdentifierInfo *Ident__BASE_FILE__; // __BASE_FILE__
114 IdentifierInfo *Ident__TIMESTAMP__; // __TIMESTAMP__
115 IdentifierInfo *Ident__COUNTER__; // __COUNTER__
116 IdentifierInfo *Ident_Pragma, *Ident__pragma; // _Pragma, __pragma
117 IdentifierInfo *Ident__VA_ARGS__; // __VA_ARGS__
118 IdentifierInfo *Ident__has_feature; // __has_feature
119 IdentifierInfo *Ident__has_extension; // __has_extension
120 IdentifierInfo *Ident__has_builtin; // __has_builtin
121 IdentifierInfo *Ident__has_attribute; // __has_attribute
122 IdentifierInfo *Ident__has_include; // __has_include
123 IdentifierInfo *Ident__has_include_next; // __has_include_next
124 IdentifierInfo *Ident__has_warning; // __has_warning
125 IdentifierInfo *Ident__building_module; // __building_module
126 IdentifierInfo *Ident__MODULE__; // __MODULE__
128 SourceLocation DATELoc, TIMELoc;
129 unsigned CounterValue; // Next __COUNTER__ value.
132 /// MaxIncludeStackDepth - Maximum depth of \#includes.
133 MaxAllowedIncludeStackDepth = 200
136 // State that is set before the preprocessor begins.
137 bool KeepComments : 1;
138 bool KeepMacroComments : 1;
139 bool SuppressIncludeNotFoundError : 1;
141 // State that changes while the preprocessor runs:
142 bool InMacroArgs : 1; // True if parsing fn macro invocation args.
144 /// Whether the preprocessor owns the header search object.
145 bool OwnsHeaderSearch : 1;
147 /// DisableMacroExpansion - True if macro expansion is disabled.
148 bool DisableMacroExpansion : 1;
150 /// MacroExpansionInDirectivesOverride - Temporarily disables
151 /// DisableMacroExpansion (i.e. enables expansion) when parsing preprocessor
153 bool MacroExpansionInDirectivesOverride : 1;
155 class ResetMacroExpansionHelper;
157 /// \brief Whether we have already loaded macros from the external source.
158 mutable bool ReadMacrosFromExternalSource : 1;
160 /// \brief True if pragmas are enabled.
161 bool PragmasEnabled : 1;
163 /// \brief True if we are pre-expanding macro arguments.
164 bool InMacroArgPreExpansion;
166 /// Identifiers - This is mapping/lookup information for all identifiers in
167 /// the program, including program keywords.
168 mutable IdentifierTable Identifiers;
170 /// Selectors - This table contains all the selectors in the program. Unlike
171 /// IdentifierTable above, this table *isn't* populated by the preprocessor.
172 /// It is declared/expanded here because it's role/lifetime is
173 /// conceptually similar the IdentifierTable. In addition, the current control
174 /// flow (in clang::ParseAST()), make it convenient to put here.
175 /// FIXME: Make sure the lifetime of Identifiers/Selectors *isn't* tied to
176 /// the lifetime of the preprocessor.
177 SelectorTable Selectors;
179 /// BuiltinInfo - Information about builtins.
180 Builtin::Context BuiltinInfo;
182 /// PragmaHandlers - This tracks all of the pragmas that the client registered
183 /// with this preprocessor.
184 PragmaNamespace *PragmaHandlers;
186 /// \brief Tracks all of the comment handlers that the client registered
187 /// with this preprocessor.
188 std::vector<CommentHandler *> CommentHandlers;
190 /// \brief True if we want to ignore EOF token and continue later on (thus
191 /// avoid tearing the Lexer and etc. down).
192 bool IncrementalProcessing;
194 /// \brief The code-completion handler.
195 CodeCompletionHandler *CodeComplete;
197 /// \brief The file that we're performing code-completion for, if any.
198 const FileEntry *CodeCompletionFile;
200 /// \brief The offset in file for the code-completion point.
201 unsigned CodeCompletionOffset;
203 /// \brief The location for the code-completion point. This gets instantiated
204 /// when the CodeCompletionFile gets \#include'ed for preprocessing.
205 SourceLocation CodeCompletionLoc;
207 /// \brief The start location for the file of the code-completion point.
209 /// This gets instantiated when the CodeCompletionFile gets \#include'ed
210 /// for preprocessing.
211 SourceLocation CodeCompletionFileLoc;
213 /// \brief The source location of the 'import' contextual keyword we just
215 SourceLocation ModuleImportLoc;
217 /// \brief The module import path that we're currently processing.
218 llvm::SmallVector<std::pair<IdentifierInfo *, SourceLocation>, 2>
221 /// \brief Whether the module import expectes an identifier next. Otherwise,
222 /// it expects a '.' or ';'.
223 bool ModuleImportExpectsIdentifier;
225 /// \brief The source location of the currently-active
226 /// #pragma clang arc_cf_code_audited begin.
227 SourceLocation PragmaARCCFCodeAuditedLoc;
229 /// \brief True if we hit the code-completion point.
230 bool CodeCompletionReached;
232 /// \brief The number of bytes that we will initially skip when entering the
233 /// main file, which is used when loading a precompiled preamble, along
234 /// with a flag that indicates whether skipping this number of bytes will
235 /// place the lexer at the start of a line.
236 std::pair<unsigned, bool> SkipMainFilePreamble;
238 /// CurLexer - This is the current top of the stack that we're lexing from if
239 /// not expanding a macro and we are lexing directly from source code.
240 /// Only one of CurLexer, CurPTHLexer, or CurTokenLexer will be non-null.
241 OwningPtr<Lexer> CurLexer;
243 /// CurPTHLexer - This is the current top of stack that we're lexing from if
244 /// not expanding from a macro and we are lexing from a PTH cache.
245 /// Only one of CurLexer, CurPTHLexer, or CurTokenLexer will be non-null.
246 OwningPtr<PTHLexer> CurPTHLexer;
248 /// CurPPLexer - This is the current top of the stack what we're lexing from
249 /// if not expanding a macro. This is an alias for either CurLexer or
251 PreprocessorLexer *CurPPLexer;
253 /// CurLookup - The DirectoryLookup structure used to find the current
254 /// FileEntry, if CurLexer is non-null and if applicable. This allows us to
255 /// implement \#include_next and find directory-specific properties.
256 const DirectoryLookup *CurDirLookup;
258 /// CurTokenLexer - This is the current macro we are expanding, if we are
259 /// expanding a macro. One of CurLexer and CurTokenLexer must be null.
260 OwningPtr<TokenLexer> CurTokenLexer;
262 /// \brief The kind of lexer we're currently working with.
268 CLK_LexAfterModuleImport
271 /// IncludeMacroStack - This keeps track of the stack of files currently
272 /// \#included, and macros currently being expanded from, not counting
273 /// CurLexer/CurTokenLexer.
274 struct IncludeStackInfo {
275 enum CurLexerKind CurLexerKind;
277 PTHLexer *ThePTHLexer;
278 PreprocessorLexer *ThePPLexer;
279 TokenLexer *TheTokenLexer;
280 const DirectoryLookup *TheDirLookup;
282 IncludeStackInfo(enum CurLexerKind K, Lexer *L, PTHLexer* P,
283 PreprocessorLexer* PPL,
284 TokenLexer* TL, const DirectoryLookup *D)
285 : CurLexerKind(K), TheLexer(L), ThePTHLexer(P), ThePPLexer(PPL),
286 TheTokenLexer(TL), TheDirLookup(D) {}
288 std::vector<IncludeStackInfo> IncludeMacroStack;
290 /// Callbacks - These are actions invoked when some preprocessor activity is
291 /// encountered (e.g. a file is \#included, etc).
292 PPCallbacks *Callbacks;
294 /// \brief Listener whose actions are invoked when an entity in the
295 /// preprocessor (e.g., a macro) that was loaded from an AST file is
297 PPMutationListener *Listener;
299 struct MacroExpandsInfo {
303 MacroExpandsInfo(Token Tok, MacroInfo *MI, SourceRange Range)
304 : Tok(Tok), MI(MI), Range(Range) { }
306 SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks;
308 /// Macros - For each IdentifierInfo that was associated with a macro, we
309 /// keep a mapping to the history of all macro definitions and #undefs in
310 /// the reverse order (the latest one is in the head of the list).
311 llvm::DenseMap<IdentifierInfo*, MacroInfo*> Macros;
312 friend class ASTReader;
314 /// \brief Macros that we want to warn because they are not used at the end
315 /// of the translation unit; we store just their SourceLocations instead
316 /// something like MacroInfo*. The benefit of this is that when we are
317 /// deserializing from PCH, we don't need to deserialize identifier & macros
318 /// just so that we can report that they are unused, we just warn using
319 /// the SourceLocations of this set (that will be filled by the ASTReader).
320 /// We are using SmallPtrSet instead of a vector for faster removal.
321 typedef llvm::SmallPtrSet<SourceLocation, 32> WarnUnusedMacroLocsTy;
322 WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
324 /// MacroArgCache - This is a "freelist" of MacroArg objects that can be
325 /// reused for quick allocation.
326 MacroArgs *MacroArgCache;
327 friend class MacroArgs;
329 /// PragmaPushMacroInfo - For each IdentifierInfo used in a #pragma
330 /// push_macro directive, we keep a MacroInfo stack used to restore
331 /// previous macro value.
332 llvm::DenseMap<IdentifierInfo*, std::vector<MacroInfo*> > PragmaPushMacroInfo;
334 // Various statistics we track for performance analysis.
335 unsigned NumDirectives, NumIncluded, NumDefined, NumUndefined, NumPragma;
336 unsigned NumIf, NumElse, NumEndif;
337 unsigned NumEnteredSourceFiles, MaxIncludeStackDepth;
338 unsigned NumMacroExpanded, NumFnMacroExpanded, NumBuiltinMacroExpanded;
339 unsigned NumFastMacroExpanded, NumTokenPaste, NumFastTokenPaste;
342 /// Predefines - This string is the predefined macros that preprocessor
343 /// should use from the command line etc.
344 std::string Predefines;
346 /// TokenLexerCache - Cache macro expanders to reduce malloc traffic.
347 enum { TokenLexerCacheSize = 8 };
348 unsigned NumCachedTokenLexers;
349 TokenLexer *TokenLexerCache[TokenLexerCacheSize];
351 /// \brief Keeps macro expanded tokens for TokenLexers.
353 /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
354 /// going to lex in the cache and when it finishes the tokens are removed
355 /// from the end of the cache.
356 SmallVector<Token, 16> MacroExpandedTokens;
357 std::vector<std::pair<TokenLexer *, size_t> > MacroExpandingLexersStack;
359 /// \brief A record of the macro definitions and expansions that
360 /// occurred during preprocessing.
362 /// This is an optional side structure that can be enabled with
363 /// \c createPreprocessingRecord() prior to preprocessing.
364 PreprocessingRecord *Record;
366 private: // Cached tokens state.
367 typedef SmallVector<Token, 1> CachedTokensTy;
369 /// CachedTokens - Cached tokens are stored here when we do backtracking or
370 /// lookahead. They are "lexed" by the CachingLex() method.
371 CachedTokensTy CachedTokens;
373 /// CachedLexPos - The position of the cached token that CachingLex() should
374 /// "lex" next. If it points beyond the CachedTokens vector, it means that
375 /// a normal Lex() should be invoked.
376 CachedTokensTy::size_type CachedLexPos;
378 /// BacktrackPositions - Stack of backtrack positions, allowing nested
379 /// backtracks. The EnableBacktrackAtThisPos() method pushes a position to
380 /// indicate where CachedLexPos should be set when the BackTrack() method is
381 /// invoked (at which point the last position is popped).
382 std::vector<CachedTokensTy::size_type> BacktrackPositions;
384 struct MacroInfoChain {
386 MacroInfoChain *Next;
387 MacroInfoChain *Prev;
390 /// MacroInfos are managed as a chain for easy disposal. This is the head
392 MacroInfoChain *MIChainHead;
394 /// MICache - A "freelist" of MacroInfo objects that can be reused for quick
396 MacroInfoChain *MICache;
399 Preprocessor(llvm::IntrusiveRefCntPtr<PreprocessorOptions> PPOpts,
400 DiagnosticsEngine &diags, LangOptions &opts,
401 const TargetInfo *target,
402 SourceManager &SM, HeaderSearch &Headers,
403 ModuleLoader &TheModuleLoader,
404 IdentifierInfoLookup *IILookup = 0,
405 bool OwnsHeaderSearch = false,
406 bool DelayInitialization = false,
407 bool IncrProcessing = false);
411 /// \brief Initialize the preprocessor, if the constructor did not already
412 /// perform the initialization.
414 /// \param Target Information about the target.
415 void Initialize(const TargetInfo &Target);
417 /// \brief Retrieve the preprocessor options used to initialize this
419 PreprocessorOptions &getPreprocessorOpts() const { return *PPOpts; }
421 DiagnosticsEngine &getDiagnostics() const { return *Diags; }
422 void setDiagnostics(DiagnosticsEngine &D) { Diags = &D; }
424 const LangOptions &getLangOpts() const { return LangOpts; }
425 const TargetInfo &getTargetInfo() const { return *Target; }
426 FileManager &getFileManager() const { return FileMgr; }
427 SourceManager &getSourceManager() const { return SourceMgr; }
428 HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; }
430 IdentifierTable &getIdentifierTable() { return Identifiers; }
431 SelectorTable &getSelectorTable() { return Selectors; }
432 Builtin::Context &getBuiltinInfo() { return BuiltinInfo; }
433 llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; }
435 void setPTHManager(PTHManager* pm);
437 PTHManager *getPTHManager() { return PTH.get(); }
439 void setExternalSource(ExternalPreprocessorSource *Source) {
440 ExternalSource = Source;
443 ExternalPreprocessorSource *getExternalSource() const {
444 return ExternalSource;
447 /// \brief Retrieve the module loader associated with this preprocessor.
448 ModuleLoader &getModuleLoader() const { return TheModuleLoader; }
450 /// SetCommentRetentionState - Control whether or not the preprocessor retains
451 /// comments in output.
452 void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) {
453 this->KeepComments = KeepComments | KeepMacroComments;
454 this->KeepMacroComments = KeepMacroComments;
457 bool getCommentRetentionState() const { return KeepComments; }
459 void setPragmasEnabled(bool Enabled) { PragmasEnabled = Enabled; }
460 bool getPragmasEnabled() const { return PragmasEnabled; }
462 void SetSuppressIncludeNotFoundError(bool Suppress) {
463 SuppressIncludeNotFoundError = Suppress;
466 bool GetSuppressIncludeNotFoundError() {
467 return SuppressIncludeNotFoundError;
470 /// isCurrentLexer - Return true if we are lexing directly from the specified
472 bool isCurrentLexer(const PreprocessorLexer *L) const {
473 return CurPPLexer == L;
476 /// getCurrentLexer - Return the current lexer being lexed from. Note
477 /// that this ignores any potentially active macro expansions and _Pragma
478 /// expansions going on at the time.
479 PreprocessorLexer *getCurrentLexer() const { return CurPPLexer; }
481 /// getCurrentFileLexer - Return the current file lexer being lexed from.
482 /// Note that this ignores any potentially active macro expansions and _Pragma
483 /// expansions going on at the time.
484 PreprocessorLexer *getCurrentFileLexer() const;
486 /// getPPCallbacks/addPPCallbacks - Accessors for preprocessor callbacks.
487 /// Note that this class takes ownership of any PPCallbacks object given to
489 PPCallbacks *getPPCallbacks() const { return Callbacks; }
490 void addPPCallbacks(PPCallbacks *C) {
492 C = new PPChainedCallbacks(C, Callbacks);
496 /// \brief Attach an preprocessor mutation listener to the preprocessor.
498 /// The preprocessor mutation listener provides the ability to track
499 /// modifications to the preprocessor entities committed after they were
500 /// initially created.
501 void setPPMutationListener(PPMutationListener *Listener) {
502 this->Listener = Listener;
505 /// \brief Retrieve a pointer to the preprocessor mutation listener
506 /// associated with this preprocessor, if any.
507 PPMutationListener *getPPMutationListener() const { return Listener; }
509 /// \brief Given an identifier, return the MacroInfo it is \#defined to
510 /// or null if it isn't \#define'd.
511 MacroInfo *getMacroInfo(IdentifierInfo *II) const {
512 if (!II->hasMacroDefinition())
515 MacroInfo *MI = getMacroInfoHistory(II);
516 assert(MI->getUndefLoc().isInvalid() && "Macro is undefined!");
520 /// \brief Given an identifier, return the (probably #undef'd) MacroInfo
521 /// representing the most recent macro definition. One can iterate over all
522 /// previous macro definitions from it. This method should only be called for
523 /// identifiers that hadMacroDefinition().
524 MacroInfo *getMacroInfoHistory(IdentifierInfo *II) const;
526 /// \brief Specify a macro for this identifier.
527 void setMacroInfo(IdentifierInfo *II, MacroInfo *MI);
528 /// \brief Add a MacroInfo that was loaded from an AST file.
529 void addLoadedMacroInfo(IdentifierInfo *II, MacroInfo *MI,
530 MacroInfo *Hint = 0);
531 /// \brief Make the given MacroInfo, that was loaded from an AST file and
532 /// previously hidden, visible.
533 void makeLoadedMacroInfoVisible(IdentifierInfo *II, MacroInfo *MI);
534 /// \brief Undefine a macro for this identifier.
535 void clearMacroInfo(IdentifierInfo *II);
537 /// macro_iterator/macro_begin/macro_end - This allows you to walk the macro
538 /// history table. Currently defined macros have
539 /// IdentifierInfo::hasMacroDefinition() set and an empty
540 /// MacroInfo::getUndefLoc() at the head of the list.
541 typedef llvm::DenseMap<IdentifierInfo*,
542 MacroInfo*>::const_iterator macro_iterator;
543 macro_iterator macro_begin(bool IncludeExternalMacros = true) const;
544 macro_iterator macro_end(bool IncludeExternalMacros = true) const;
546 /// \brief Return the name of the macro defined before \p Loc that has
547 /// spelling \p Tokens. If there are multiple macros with same spelling,
548 /// return the last one defined.
549 StringRef getLastMacroWithSpelling(SourceLocation Loc,
550 ArrayRef<TokenValue> Tokens) const;
552 const std::string &getPredefines() const { return Predefines; }
553 /// setPredefines - Set the predefines for this Preprocessor. These
554 /// predefines are automatically injected when parsing the main file.
555 void setPredefines(const char *P) { Predefines = P; }
556 void setPredefines(const std::string &P) { Predefines = P; }
558 /// Return information about the specified preprocessor
559 /// identifier token.
560 IdentifierInfo *getIdentifierInfo(StringRef Name) const {
561 return &Identifiers.get(Name);
564 /// AddPragmaHandler - Add the specified pragma handler to the preprocessor.
565 /// If 'Namespace' is non-null, then it is a token required to exist on the
566 /// pragma line before the pragma string starts, e.g. "STDC" or "GCC".
567 void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler);
568 void AddPragmaHandler(PragmaHandler *Handler) {
569 AddPragmaHandler(StringRef(), Handler);
572 /// RemovePragmaHandler - Remove the specific pragma handler from
573 /// the preprocessor. If \p Namespace is non-null, then it should
574 /// be the namespace that \p Handler was added to. It is an error
575 /// to remove a handler that has not been registered.
576 void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler);
577 void RemovePragmaHandler(PragmaHandler *Handler) {
578 RemovePragmaHandler(StringRef(), Handler);
581 /// \brief Add the specified comment handler to the preprocessor.
582 void addCommentHandler(CommentHandler *Handler);
584 /// \brief Remove the specified comment handler.
586 /// It is an error to remove a handler that has not been registered.
587 void removeCommentHandler(CommentHandler *Handler);
589 /// \brief Set the code completion handler to the given object.
590 void setCodeCompletionHandler(CodeCompletionHandler &Handler) {
591 CodeComplete = &Handler;
594 /// \brief Retrieve the current code-completion handler.
595 CodeCompletionHandler *getCodeCompletionHandler() const {
599 /// \brief Clear out the code completion handler.
600 void clearCodeCompletionHandler() {
604 /// \brief Hook used by the lexer to invoke the "natural language" code
605 /// completion point.
606 void CodeCompleteNaturalLanguage();
608 /// \brief Retrieve the preprocessing record, or NULL if there is no
609 /// preprocessing record.
610 PreprocessingRecord *getPreprocessingRecord() const { return Record; }
612 /// \brief Create a new preprocessing record, which will keep track of
613 /// all macro expansions, macro definitions, etc.
614 void createPreprocessingRecord(bool RecordConditionalDirectives);
616 /// EnterMainSourceFile - Enter the specified FileID as the main source file,
617 /// which implicitly adds the builtin defines etc.
618 void EnterMainSourceFile();
620 /// EndSourceFile - Inform the preprocessor callbacks that processing is
622 void EndSourceFile();
624 /// EnterSourceFile - Add a source file to the top of the include stack and
625 /// start lexing tokens from it instead of the current buffer. Emit an error
626 /// and don't enter the file on error.
627 void EnterSourceFile(FileID CurFileID, const DirectoryLookup *Dir,
630 /// EnterMacro - Add a Macro to the top of the include stack and start lexing
631 /// tokens from it instead of the current buffer. Args specifies the
632 /// tokens input to a function-like macro.
634 /// ILEnd specifies the location of the ')' for a function-like macro or the
635 /// identifier for an object-like macro.
636 void EnterMacro(Token &Identifier, SourceLocation ILEnd, MacroInfo *Macro,
639 /// EnterTokenStream - Add a "macro" context to the top of the include stack,
640 /// which will cause the lexer to start returning the specified tokens.
642 /// If DisableMacroExpansion is true, tokens lexed from the token stream will
643 /// not be subject to further macro expansion. Otherwise, these tokens will
644 /// be re-macro-expanded when/if expansion is enabled.
646 /// If OwnsTokens is false, this method assumes that the specified stream of
647 /// tokens has a permanent owner somewhere, so they do not need to be copied.
648 /// If it is true, it assumes the array of tokens is allocated with new[] and
651 void EnterTokenStream(const Token *Toks, unsigned NumToks,
652 bool DisableMacroExpansion, bool OwnsTokens);
654 /// RemoveTopOfLexerStack - Pop the current lexer/macro exp off the top of the
655 /// lexer stack. This should only be used in situations where the current
656 /// state of the top-of-stack lexer is known.
657 void RemoveTopOfLexerStack();
659 /// EnableBacktrackAtThisPos - From the point that this method is called, and
660 /// until CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
661 /// keeps track of the lexed tokens so that a subsequent Backtrack() call will
662 /// make the Preprocessor re-lex the same tokens.
664 /// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
665 /// be called multiple times and CommitBacktrackedTokens/Backtrack calls will
666 /// be combined with the EnableBacktrackAtThisPos calls in reverse order.
668 /// NOTE: *DO NOT* forget to call either CommitBacktrackedTokens or Backtrack
669 /// at some point after EnableBacktrackAtThisPos. If you don't, caching of
670 /// tokens will continue indefinitely.
672 void EnableBacktrackAtThisPos();
674 /// CommitBacktrackedTokens - Disable the last EnableBacktrackAtThisPos call.
675 void CommitBacktrackedTokens();
677 /// Backtrack - Make Preprocessor re-lex the tokens that were lexed since
678 /// EnableBacktrackAtThisPos() was previously called.
681 /// isBacktrackEnabled - True if EnableBacktrackAtThisPos() was called and
682 /// caching of tokens is on.
683 bool isBacktrackEnabled() const { return !BacktrackPositions.empty(); }
685 /// Lex - To lex a token from the preprocessor, just pull a token from the
686 /// current lexer or macro object.
687 void Lex(Token &Result) {
688 switch (CurLexerKind) {
689 case CLK_Lexer: CurLexer->Lex(Result); break;
690 case CLK_PTHLexer: CurPTHLexer->Lex(Result); break;
691 case CLK_TokenLexer: CurTokenLexer->Lex(Result); break;
692 case CLK_CachingLexer: CachingLex(Result); break;
693 case CLK_LexAfterModuleImport: LexAfterModuleImport(Result); break;
697 void LexAfterModuleImport(Token &Result);
699 /// LexNonComment - Lex a token. If it's a comment, keep lexing until we get
700 /// something not a comment. This is useful in -E -C mode where comments
701 /// would foul up preprocessor directive handling.
702 void LexNonComment(Token &Result) {
705 while (Result.getKind() == tok::comment);
708 /// LexUnexpandedToken - This is just like Lex, but this disables macro
709 /// expansion of identifier tokens.
710 void LexUnexpandedToken(Token &Result) {
711 // Disable macro expansion.
712 bool OldVal = DisableMacroExpansion;
713 DisableMacroExpansion = true;
718 DisableMacroExpansion = OldVal;
721 /// LexUnexpandedNonComment - Like LexNonComment, but this disables macro
722 /// expansion of identifier tokens.
723 void LexUnexpandedNonComment(Token &Result) {
725 LexUnexpandedToken(Result);
726 while (Result.getKind() == tok::comment);
729 /// Disables macro expansion everywhere except for preprocessor directives.
730 void SetMacroExpansionOnlyInDirectives() {
731 DisableMacroExpansion = true;
732 MacroExpansionInDirectivesOverride = true;
735 /// LookAhead - This peeks ahead N tokens and returns that token without
736 /// consuming any tokens. LookAhead(0) returns the next token that would be
737 /// returned by Lex(), LookAhead(1) returns the token after it, etc. This
738 /// returns normal tokens after phase 5. As such, it is equivalent to using
739 /// 'Lex', not 'LexUnexpandedToken'.
740 const Token &LookAhead(unsigned N) {
741 if (CachedLexPos + N < CachedTokens.size())
742 return CachedTokens[CachedLexPos+N];
744 return PeekAhead(N+1);
747 /// RevertCachedTokens - When backtracking is enabled and tokens are cached,
748 /// this allows to revert a specific number of tokens.
749 /// Note that the number of tokens being reverted should be up to the last
750 /// backtrack position, not more.
751 void RevertCachedTokens(unsigned N) {
752 assert(isBacktrackEnabled() &&
753 "Should only be called when tokens are cached for backtracking");
754 assert(signed(CachedLexPos) - signed(N) >= signed(BacktrackPositions.back())
755 && "Should revert tokens up to the last backtrack position, not more");
756 assert(signed(CachedLexPos) - signed(N) >= 0 &&
757 "Corrupted backtrack positions ?");
761 /// EnterToken - Enters a token in the token stream to be lexed next. If
762 /// BackTrack() is called afterwards, the token will remain at the insertion
764 void EnterToken(const Token &Tok) {
765 EnterCachingLexMode();
766 CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok);
769 /// AnnotateCachedTokens - We notify the Preprocessor that if it is caching
770 /// tokens (because backtrack is enabled) it should replace the most recent
771 /// cached tokens with the given annotation token. This function has no effect
772 /// if backtracking is not enabled.
774 /// Note that the use of this function is just for optimization; so that the
775 /// cached tokens doesn't get re-parsed and re-resolved after a backtrack is
777 void AnnotateCachedTokens(const Token &Tok) {
778 assert(Tok.isAnnotation() && "Expected annotation token");
779 if (CachedLexPos != 0 && isBacktrackEnabled())
780 AnnotatePreviousCachedTokens(Tok);
783 /// \brief Replace the last token with an annotation token.
785 /// Like AnnotateCachedTokens(), this routine replaces an
786 /// already-parsed (and resolved) token with an annotation
787 /// token. However, this routine only replaces the last token with
788 /// the annotation token; it does not affect any other cached
789 /// tokens. This function has no effect if backtracking is not
791 void ReplaceLastTokenWithAnnotation(const Token &Tok) {
792 assert(Tok.isAnnotation() && "Expected annotation token");
793 if (CachedLexPos != 0 && isBacktrackEnabled())
794 CachedTokens[CachedLexPos-1] = Tok;
797 /// TypoCorrectToken - Update the current token to represent the provided
798 /// identifier, in order to cache an action performed by typo correction.
799 void TypoCorrectToken(const Token &Tok) {
800 assert(Tok.getIdentifierInfo() && "Expected identifier token");
801 if (CachedLexPos != 0 && isBacktrackEnabled())
802 CachedTokens[CachedLexPos-1] = Tok;
805 /// \brief Recompute the current lexer kind based on the CurLexer/CurPTHLexer/
806 /// CurTokenLexer pointers.
807 void recomputeCurLexerKind();
809 /// \brief Returns true if incremental processing is enabled
810 bool isIncrementalProcessingEnabled() const { return IncrementalProcessing; }
812 /// \brief Enables the incremental processing
813 void enableIncrementalProcessing(bool value = true) {
814 IncrementalProcessing = value;
817 /// \brief Specify the point at which code-completion will be performed.
819 /// \param File the file in which code completion should occur. If
820 /// this file is included multiple times, code-completion will
821 /// perform completion the first time it is included. If NULL, this
822 /// function clears out the code-completion point.
824 /// \param Line the line at which code completion should occur
827 /// \param Column the column at which code completion should occur
830 /// \returns true if an error occurred, false otherwise.
831 bool SetCodeCompletionPoint(const FileEntry *File,
832 unsigned Line, unsigned Column);
834 /// \brief Determine if we are performing code completion.
835 bool isCodeCompletionEnabled() const { return CodeCompletionFile != 0; }
837 /// \brief Returns the location of the code-completion point.
838 /// Returns an invalid location if code-completion is not enabled or the file
839 /// containing the code-completion point has not been lexed yet.
840 SourceLocation getCodeCompletionLoc() const { return CodeCompletionLoc; }
842 /// \brief Returns the start location of the file of code-completion point.
843 /// Returns an invalid location if code-completion is not enabled or the file
844 /// containing the code-completion point has not been lexed yet.
845 SourceLocation getCodeCompletionFileLoc() const {
846 return CodeCompletionFileLoc;
849 /// \brief Returns true if code-completion is enabled and we have hit the
850 /// code-completion point.
851 bool isCodeCompletionReached() const { return CodeCompletionReached; }
853 /// \brief Note that we hit the code-completion point.
854 void setCodeCompletionReached() {
855 assert(isCodeCompletionEnabled() && "Code-completion not enabled!");
856 CodeCompletionReached = true;
857 // Silence any diagnostics that occur after we hit the code-completion.
858 getDiagnostics().setSuppressAllDiagnostics(true);
861 /// \brief The location of the currently-active \#pragma clang
862 /// arc_cf_code_audited begin. Returns an invalid location if there
863 /// is no such pragma active.
864 SourceLocation getPragmaARCCFCodeAuditedLoc() const {
865 return PragmaARCCFCodeAuditedLoc;
868 /// \brief Set the location of the currently-active \#pragma clang
869 /// arc_cf_code_audited begin. An invalid location ends the pragma.
870 void setPragmaARCCFCodeAuditedLoc(SourceLocation Loc) {
871 PragmaARCCFCodeAuditedLoc = Loc;
874 /// \brief Instruct the preprocessor to skip part of the main source file.
876 /// \param Bytes The number of bytes in the preamble to skip.
878 /// \param StartOfLine Whether skipping these bytes puts the lexer at the
880 void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine) {
881 SkipMainFilePreamble.first = Bytes;
882 SkipMainFilePreamble.second = StartOfLine;
885 /// Diag - Forwarding function for diagnostics. This emits a diagnostic at
886 /// the specified Token's location, translating the token's start
887 /// position in the current buffer into a SourcePosition object for rendering.
888 DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const {
889 return Diags->Report(Loc, DiagID);
892 DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const {
893 return Diags->Report(Tok.getLocation(), DiagID);
896 /// getSpelling() - Return the 'spelling' of the token at the given
897 /// location; does not go up to the spelling location or down to the
898 /// expansion location.
900 /// \param buffer A buffer which will be used only if the token requires
901 /// "cleaning", e.g. if it contains trigraphs or escaped newlines
902 /// \param invalid If non-null, will be set \c true if an error occurs.
903 StringRef getSpelling(SourceLocation loc,
904 SmallVectorImpl<char> &buffer,
905 bool *invalid = 0) const {
906 return Lexer::getSpelling(loc, buffer, SourceMgr, LangOpts, invalid);
909 /// getSpelling() - Return the 'spelling' of the Tok token. The spelling of a
910 /// token is the characters used to represent the token in the source file
911 /// after trigraph expansion and escaped-newline folding. In particular, this
912 /// wants to get the true, uncanonicalized, spelling of things like digraphs
915 /// \param Invalid If non-null, will be set \c true if an error occurs.
916 std::string getSpelling(const Token &Tok, bool *Invalid = 0) const {
917 return Lexer::getSpelling(Tok, SourceMgr, LangOpts, Invalid);
920 /// getSpelling - This method is used to get the spelling of a token into a
921 /// preallocated buffer, instead of as an std::string. The caller is required
922 /// to allocate enough space for the token, which is guaranteed to be at least
923 /// Tok.getLength() bytes long. The length of the actual result is returned.
925 /// Note that this method may do two possible things: it may either fill in
926 /// the buffer specified with characters, or it may *change the input pointer*
927 /// to point to a constant buffer with the data already in it (avoiding a
928 /// copy). The caller is not allowed to modify the returned buffer pointer
929 /// if an internal buffer is returned.
930 unsigned getSpelling(const Token &Tok, const char *&Buffer,
931 bool *Invalid = 0) const {
932 return Lexer::getSpelling(Tok, Buffer, SourceMgr, LangOpts, Invalid);
935 /// getSpelling - This method is used to get the spelling of a token into a
936 /// SmallVector. Note that the returned StringRef may not point to the
937 /// supplied buffer if a copy can be avoided.
938 StringRef getSpelling(const Token &Tok,
939 SmallVectorImpl<char> &Buffer,
940 bool *Invalid = 0) const;
942 /// getSpellingOfSingleCharacterNumericConstant - Tok is a numeric constant
943 /// with length 1, return the character.
944 char getSpellingOfSingleCharacterNumericConstant(const Token &Tok,
945 bool *Invalid = 0) const {
946 assert(Tok.is(tok::numeric_constant) &&
947 Tok.getLength() == 1 && "Called on unsupported token");
948 assert(!Tok.needsCleaning() && "Token can't need cleaning with length 1");
950 // If the token is carrying a literal data pointer, just use it.
951 if (const char *D = Tok.getLiteralData())
954 // Otherwise, fall back on getCharacterData, which is slower, but always
956 return *SourceMgr.getCharacterData(Tok.getLocation(), Invalid);
959 /// \brief Retrieve the name of the immediate macro expansion.
961 /// This routine starts from a source location, and finds the name of the macro
962 /// responsible for its immediate expansion. It looks through any intervening
963 /// macro argument expansions to compute this. It returns a StringRef which
964 /// refers to the SourceManager-owned buffer of the source where that macro
965 /// name is spelled. Thus, the result shouldn't out-live the SourceManager.
966 StringRef getImmediateMacroName(SourceLocation Loc) {
967 return Lexer::getImmediateMacroName(Loc, SourceMgr, getLangOpts());
970 /// CreateString - Plop the specified string into a scratch buffer and set the
971 /// specified token's location and length to it. If specified, the source
972 /// location provides a location of the expansion point of the token.
973 void CreateString(StringRef Str, Token &Tok,
974 SourceLocation ExpansionLocStart = SourceLocation(),
975 SourceLocation ExpansionLocEnd = SourceLocation());
977 /// \brief Computes the source location just past the end of the
978 /// token at this source location.
980 /// This routine can be used to produce a source location that
981 /// points just past the end of the token referenced by \p Loc, and
982 /// is generally used when a diagnostic needs to point just after a
983 /// token where it expected something different that it received. If
984 /// the returned source location would not be meaningful (e.g., if
985 /// it points into a macro), this routine returns an invalid
988 /// \param Offset an offset from the end of the token, where the source
989 /// location should refer to. The default offset (0) produces a source
990 /// location pointing just past the end of the token; an offset of 1 produces
991 /// a source location pointing to the last character in the token, etc.
992 SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset = 0) {
993 return Lexer::getLocForEndOfToken(Loc, Offset, SourceMgr, LangOpts);
996 /// \brief Returns true if the given MacroID location points at the first
997 /// token of the macro expansion.
999 /// \param MacroBegin If non-null and function returns true, it is set to
1000 /// begin location of the macro.
1001 bool isAtStartOfMacroExpansion(SourceLocation loc,
1002 SourceLocation *MacroBegin = 0) const {
1003 return Lexer::isAtStartOfMacroExpansion(loc, SourceMgr, LangOpts,
1007 /// \brief Returns true if the given MacroID location points at the last
1008 /// token of the macro expansion.
1010 /// \param MacroEnd If non-null and function returns true, it is set to
1011 /// end location of the macro.
1012 bool isAtEndOfMacroExpansion(SourceLocation loc,
1013 SourceLocation *MacroEnd = 0) const {
1014 return Lexer::isAtEndOfMacroExpansion(loc, SourceMgr, LangOpts, MacroEnd);
1017 /// DumpToken - Print the token to stderr, used for debugging.
1019 void DumpToken(const Token &Tok, bool DumpFlags = false) const;
1020 void DumpLocation(SourceLocation Loc) const;
1021 void DumpMacro(const MacroInfo &MI) const;
1023 /// AdvanceToTokenCharacter - Given a location that specifies the start of a
1024 /// token, return a new location that specifies a character within the token.
1025 SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart,
1026 unsigned Char) const {
1027 return Lexer::AdvanceToTokenCharacter(TokStart, Char, SourceMgr, LangOpts);
1030 /// IncrementPasteCounter - Increment the counters for the number of token
1031 /// paste operations performed. If fast was specified, this is a 'fast paste'
1032 /// case we handled.
1034 void IncrementPasteCounter(bool isFast) {
1036 ++NumFastTokenPaste;
1043 size_t getTotalMemory() const;
1045 /// HandleMicrosoftCommentPaste - When the macro expander pastes together a
1046 /// comment (/##/) in microsoft mode, this method handles updating the current
1047 /// state, returning the token on the next source line.
1048 void HandleMicrosoftCommentPaste(Token &Tok);
1050 //===--------------------------------------------------------------------===//
1051 // Preprocessor callback methods. These are invoked by a lexer as various
1052 // directives and events are found.
1054 /// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the
1055 /// identifier information for the token and install it into the token,
1056 /// updating the token kind accordingly.
1057 IdentifierInfo *LookUpIdentifierInfo(Token &Identifier) const;
1060 llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
1064 // SetPoisonReason - Call this function to indicate the reason for
1065 // poisoning an identifier. If that identifier is accessed while
1066 // poisoned, then this reason will be used instead of the default
1067 // "poisoned" diagnostic.
1068 void SetPoisonReason(IdentifierInfo *II, unsigned DiagID);
1070 // HandlePoisonedIdentifier - Display reason for poisoned
1072 void HandlePoisonedIdentifier(Token & Tok);
1074 void MaybeHandlePoisonedIdentifier(Token & Identifier) {
1075 if(IdentifierInfo * II = Identifier.getIdentifierInfo()) {
1076 if(II->isPoisoned()) {
1077 HandlePoisonedIdentifier(Identifier);
1083 /// Identifiers used for SEH handling in Borland. These are only
1084 /// allowed in particular circumstances
1086 IdentifierInfo *Ident__exception_code,
1087 *Ident___exception_code,
1088 *Ident_GetExceptionCode;
1089 // __except filter expression
1090 IdentifierInfo *Ident__exception_info,
1091 *Ident___exception_info,
1092 *Ident_GetExceptionInfo;
1094 IdentifierInfo *Ident__abnormal_termination,
1095 *Ident___abnormal_termination,
1096 *Ident_AbnormalTermination;
1098 void PoisonSEHIdentifiers(bool Poison = true); // Borland
1100 /// HandleIdentifier - This callback is invoked when the lexer reads an
1101 /// identifier and has filled in the tokens IdentifierInfo member. This
1102 /// callback potentially macro expands it or turns it into a named token (like
1104 void HandleIdentifier(Token &Identifier);
1107 /// HandleEndOfFile - This callback is invoked when the lexer hits the end of
1108 /// the current file. This either returns the EOF token and returns true, or
1109 /// pops a level off the include stack and returns false, at which point the
1110 /// client should call lex again.
1111 bool HandleEndOfFile(Token &Result, bool isEndOfMacro = false);
1113 /// HandleEndOfTokenLexer - This callback is invoked when the current
1114 /// TokenLexer hits the end of its token stream.
1115 bool HandleEndOfTokenLexer(Token &Result);
1117 /// HandleDirective - This callback is invoked when the lexer sees a # token
1118 /// at the start of a line. This consumes the directive, modifies the
1119 /// lexer/preprocessor state, and advances the lexer(s) so that the next token
1120 /// read is the correct one.
1121 void HandleDirective(Token &Result);
1123 /// CheckEndOfDirective - Ensure that the next token is a tok::eod token. If
1124 /// not, emit a diagnostic and consume up until the eod. If EnableMacros is
1125 /// true, then we consider macros that expand to zero tokens as being ok.
1126 void CheckEndOfDirective(const char *Directive, bool EnableMacros = false);
1128 /// DiscardUntilEndOfDirective - Read and discard all tokens remaining on the
1129 /// current line until the tok::eod token is found.
1130 void DiscardUntilEndOfDirective();
1132 /// SawDateOrTime - This returns true if the preprocessor has seen a use of
1133 /// __DATE__ or __TIME__ in the file so far.
1134 bool SawDateOrTime() const {
1135 return DATELoc != SourceLocation() || TIMELoc != SourceLocation();
1137 unsigned getCounterValue() const { return CounterValue; }
1138 void setCounterValue(unsigned V) { CounterValue = V; }
1140 /// \brief Retrieves the module that we're currently building, if any.
1141 Module *getCurrentModule();
1143 /// \brief Allocate a new MacroInfo object with the provided SourceLocation.
1144 MacroInfo *AllocateMacroInfo(SourceLocation L);
1146 /// \brief Allocate a new MacroInfo object which is clone of \p MI.
1147 MacroInfo *CloneMacroInfo(const MacroInfo &MI);
1149 /// \brief Turn the specified lexer token into a fully checked and spelled
1150 /// filename, e.g. as an operand of \#include.
1152 /// The caller is expected to provide a buffer that is large enough to hold
1153 /// the spelling of the filename, but is also expected to handle the case
1154 /// when this method decides to use a different buffer.
1156 /// \returns true if the input filename was in <>'s or false if it was
1158 bool GetIncludeFilenameSpelling(SourceLocation Loc,StringRef &Filename);
1160 /// \brief Given a "foo" or \<foo> reference, look up the indicated file.
1162 /// Returns null on failure. \p isAngled indicates whether the file
1163 /// reference is for system \#include's or not (i.e. using <> instead of "").
1164 const FileEntry *LookupFile(StringRef Filename,
1165 bool isAngled, const DirectoryLookup *FromDir,
1166 const DirectoryLookup *&CurDir,
1167 SmallVectorImpl<char> *SearchPath,
1168 SmallVectorImpl<char> *RelativePath,
1169 Module **SuggestedModule,
1170 bool SkipCache = false);
1172 /// GetCurLookup - The DirectoryLookup structure used to find the current
1173 /// FileEntry, if CurLexer is non-null and if applicable. This allows us to
1174 /// implement \#include_next and find directory-specific properties.
1175 const DirectoryLookup *GetCurDirLookup() { return CurDirLookup; }
1177 /// \brief Return true if we're in the top-level file, not in a \#include.
1178 bool isInPrimaryFile() const;
1180 /// ConcatenateIncludeName - Handle cases where the \#include name is expanded
1181 /// from a macro as multiple tokens, which need to be glued together. This
1182 /// occurs for code like:
1184 /// \#define FOO <x/y.h>
1187 /// because in this case, "<x/y.h>" is returned as 7 tokens, not one.
1189 /// This code concatenates and consumes tokens up to the '>' token. It
1190 /// returns false if the > was found, otherwise it returns true if it finds
1191 /// and consumes the EOD marker.
1192 bool ConcatenateIncludeName(SmallString<128> &FilenameBuffer,
1193 SourceLocation &End);
1195 /// LexOnOffSwitch - Lex an on-off-switch (C99 6.10.6p2) and verify that it is
1196 /// followed by EOD. Return true if the token is not a valid on-off-switch.
1197 bool LexOnOffSwitch(tok::OnOffSwitch &OOS);
1201 void PushIncludeMacroStack() {
1202 IncludeMacroStack.push_back(IncludeStackInfo(CurLexerKind,
1206 CurTokenLexer.take(),
1211 void PopIncludeMacroStack() {
1212 CurLexer.reset(IncludeMacroStack.back().TheLexer);
1213 CurPTHLexer.reset(IncludeMacroStack.back().ThePTHLexer);
1214 CurPPLexer = IncludeMacroStack.back().ThePPLexer;
1215 CurTokenLexer.reset(IncludeMacroStack.back().TheTokenLexer);
1216 CurDirLookup = IncludeMacroStack.back().TheDirLookup;
1217 CurLexerKind = IncludeMacroStack.back().CurLexerKind;
1218 IncludeMacroStack.pop_back();
1221 /// \brief Allocate a new MacroInfo object.
1222 MacroInfo *AllocateMacroInfo();
1224 /// \brief Release the specified MacroInfo for re-use.
1226 /// This memory will be reused for allocating new MacroInfo objects.
1227 void ReleaseMacroInfo(MacroInfo* MI);
1229 /// ReadMacroName - Lex and validate a macro name, which occurs after a
1230 /// \#define or \#undef. This emits a diagnostic, sets the token kind to eod,
1231 /// and discards the rest of the macro line if the macro name is invalid.
1232 void ReadMacroName(Token &MacroNameTok, char isDefineUndef = 0);
1234 /// ReadMacroDefinitionArgList - The ( starting an argument list of a macro
1235 /// definition has just been read. Lex the rest of the arguments and the
1236 /// closing ), updating MI with what we learn and saving in LastTok the
1237 /// last token read.
1238 /// Return true if an error occurs parsing the arg list.
1239 bool ReadMacroDefinitionArgList(MacroInfo *MI, Token& LastTok);
1241 /// We just read a \#if or related directive and decided that the
1242 /// subsequent tokens are in the \#if'd out portion of the
1243 /// file. Lex the rest of the file, until we see an \#endif. If \p
1244 /// FoundNonSkipPortion is true, then we have already emitted code for part of
1245 /// this \#if directive, so \#else/\#elif blocks should never be entered. If
1246 /// \p FoundElse is false, then \#else directives are ok, if not, then we have
1247 /// already seen one so a \#else directive is a duplicate. When this returns,
1248 /// the caller can lex the first valid token.
1249 void SkipExcludedConditionalBlock(SourceLocation IfTokenLoc,
1250 bool FoundNonSkipPortion, bool FoundElse,
1251 SourceLocation ElseLoc = SourceLocation());
1253 /// \brief A fast PTH version of SkipExcludedConditionalBlock.
1254 void PTHSkipExcludedConditionalBlock();
1256 /// EvaluateDirectiveExpression - Evaluate an integer constant expression that
1257 /// may occur after a #if or #elif directive and return it as a bool. If the
1258 /// expression is equivalent to "!defined(X)" return X in IfNDefMacro.
1259 bool EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro);
1261 /// RegisterBuiltinPragmas - Install the standard preprocessor pragmas:
1262 /// \#pragma GCC poison/system_header/dependency and \#pragma once.
1263 void RegisterBuiltinPragmas();
1265 /// \brief Register builtin macros such as __LINE__ with the identifier table.
1266 void RegisterBuiltinMacros();
1268 /// HandleMacroExpandedIdentifier - If an identifier token is read that is to
1269 /// be expanded as a macro, handle it and return the next token as 'Tok'. If
1270 /// the macro should not be expanded return true, otherwise return false.
1271 bool HandleMacroExpandedIdentifier(Token &Tok, MacroInfo *MI);
1273 /// \brief Cache macro expanded tokens for TokenLexers.
1275 /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
1276 /// going to lex in the cache and when it finishes the tokens are removed
1277 /// from the end of the cache.
1278 Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
1279 ArrayRef<Token> tokens);
1280 void removeCachedMacroExpandedTokensOfLastLexer();
1281 friend void TokenLexer::ExpandFunctionArguments();
1283 /// isNextPPTokenLParen - Determine whether the next preprocessor token to be
1284 /// lexed is a '('. If so, consume the token and return true, if not, this
1285 /// method should have no observable side-effect on the lexed tokens.
1286 bool isNextPPTokenLParen();
1288 /// ReadFunctionLikeMacroArgs - After reading "MACRO(", this method is
1289 /// invoked to read all of the formal arguments specified for the macro
1290 /// invocation. This returns null on error.
1291 MacroArgs *ReadFunctionLikeMacroArgs(Token &MacroName, MacroInfo *MI,
1292 SourceLocation &ExpansionEnd);
1294 /// ExpandBuiltinMacro - If an identifier token is read that is to be expanded
1295 /// as a builtin macro, handle it and return the next token as 'Tok'.
1296 void ExpandBuiltinMacro(Token &Tok);
1298 /// Handle_Pragma - Read a _Pragma directive, slice it up, process it, then
1299 /// return the first token after the directive. The _Pragma token has just
1300 /// been read into 'Tok'.
1301 void Handle_Pragma(Token &Tok);
1303 /// HandleMicrosoft__pragma - Like Handle_Pragma except the pragma text
1304 /// is not enclosed within a string literal.
1305 void HandleMicrosoft__pragma(Token &Tok);
1307 /// EnterSourceFileWithLexer - Add a lexer to the top of the include stack and
1308 /// start lexing tokens from it instead of the current buffer.
1309 void EnterSourceFileWithLexer(Lexer *TheLexer, const DirectoryLookup *Dir);
1311 /// EnterSourceFileWithPTH - Add a lexer to the top of the include stack and
1312 /// start getting tokens from it using the PTH cache.
1313 void EnterSourceFileWithPTH(PTHLexer *PL, const DirectoryLookup *Dir);
1315 /// IsFileLexer - Returns true if we are lexing from a file and not a
1316 /// pragma or a macro.
1317 static bool IsFileLexer(const Lexer* L, const PreprocessorLexer* P) {
1318 return L ? !L->isPragmaLexer() : P != 0;
1321 static bool IsFileLexer(const IncludeStackInfo& I) {
1322 return IsFileLexer(I.TheLexer, I.ThePPLexer);
1325 bool IsFileLexer() const {
1326 return IsFileLexer(CurLexer.get(), CurPPLexer);
1329 //===--------------------------------------------------------------------===//
1331 void CachingLex(Token &Result);
1332 bool InCachingLexMode() const {
1333 // If the Lexer pointers are 0 and IncludeMacroStack is empty, it means
1334 // that we are past EOF, not that we are in CachingLex mode.
1335 return CurPPLexer == 0 && CurTokenLexer == 0 && CurPTHLexer == 0 &&
1336 !IncludeMacroStack.empty();
1338 void EnterCachingLexMode();
1339 void ExitCachingLexMode() {
1340 if (InCachingLexMode())
1341 RemoveTopOfLexerStack();
1343 const Token &PeekAhead(unsigned N);
1344 void AnnotatePreviousCachedTokens(const Token &Tok);
1346 //===--------------------------------------------------------------------===//
1347 /// Handle*Directive - implement the various preprocessor directives. These
1348 /// should side-effect the current preprocessor object so that the next call
1349 /// to Lex() will return the appropriate token next.
1350 void HandleLineDirective(Token &Tok);
1351 void HandleDigitDirective(Token &Tok);
1352 void HandleUserDiagnosticDirective(Token &Tok, bool isWarning);
1353 void HandleIdentSCCSDirective(Token &Tok);
1354 void HandleMacroPublicDirective(Token &Tok);
1355 void HandleMacroPrivateDirective(Token &Tok);
1358 void HandleIncludeDirective(SourceLocation HashLoc,
1360 const DirectoryLookup *LookupFrom = 0,
1361 bool isImport = false);
1362 void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
1363 void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);
1364 void HandleImportDirective(SourceLocation HashLoc, Token &Tok);
1365 void HandleMicrosoftImportDirective(Token &Tok);
1368 void HandleDefineDirective(Token &Tok);
1369 void HandleUndefDirective(Token &Tok);
1370 void UndefineMacro(IdentifierInfo *II, MacroInfo *MI,
1371 SourceLocation UndefLoc);
1373 // Conditional Inclusion.
1374 void HandleIfdefDirective(Token &Tok, bool isIfndef,
1375 bool ReadAnyTokensBeforeDirective);
1376 void HandleIfDirective(Token &Tok, bool ReadAnyTokensBeforeDirective);
1377 void HandleEndifDirective(Token &Tok);
1378 void HandleElseDirective(Token &Tok);
1379 void HandleElifDirective(Token &Tok);
1382 void HandlePragmaDirective(unsigned Introducer);
1384 void HandlePragmaOnce(Token &OnceTok);
1385 void HandlePragmaMark();
1386 void HandlePragmaPoison(Token &PoisonTok);
1387 void HandlePragmaSystemHeader(Token &SysHeaderTok);
1388 void HandlePragmaDependency(Token &DependencyTok);
1389 void HandlePragmaComment(Token &CommentTok);
1390 void HandlePragmaMessage(Token &MessageTok);
1391 void HandlePragmaPushMacro(Token &Tok);
1392 void HandlePragmaPopMacro(Token &Tok);
1393 void HandlePragmaIncludeAlias(Token &Tok);
1394 IdentifierInfo *ParsePragmaPushOrPopMacro(Token &Tok);
1396 // Return true and store the first token only if any CommentHandler
1397 // has inserted some tokens and getCommentRetentionState() is false.
1398 bool HandleComment(Token &Token, SourceRange Comment);
1400 /// \brief A macro is used, update information about macros that need unused
1402 void markMacroAsUsed(MacroInfo *MI);
1405 /// \brief Abstract base class that describes a handler that will receive
1406 /// source ranges for each of the comments encountered in the source file.
1407 class CommentHandler {
1409 virtual ~CommentHandler();
1411 // The handler shall return true if it has pushed any tokens
1412 // to be read using e.g. EnterToken or EnterTokenStream.
1413 virtual bool HandleComment(Preprocessor &PP, SourceRange Comment) = 0;
1416 } // end namespace clang