1 //===--- TokenLexer.cpp - Lex from a token stream -------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file implements the TokenLexer interface.
12 //===----------------------------------------------------------------------===//
14 #include "clang/Lex/TokenLexer.h"
15 #include "clang/Basic/SourceManager.h"
16 #include "clang/Lex/LexDiagnostic.h"
17 #include "clang/Lex/MacroArgs.h"
18 #include "clang/Lex/MacroInfo.h"
19 #include "clang/Lex/Preprocessor.h"
20 #include "llvm/ADT/SmallString.h"
22 using namespace clang;
24 /// Create a TokenLexer for the specified macro with the specified actual
25 /// arguments. Note that this ctor takes ownership of the ActualArgs pointer.
26 void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI,
28 // If the client is reusing a TokenLexer, make sure to free any memory
29 // associated with it.
36 ExpandLocStart = Tok.getLocation();
38 AtStartOfLine = Tok.isAtStartOfLine();
39 HasLeadingSpace = Tok.hasLeadingSpace();
40 NextTokGetsSpace = false;
41 Tokens = &*Macro->tokens_begin();
43 DisableMacroExpansion = false;
44 NumTokens = Macro->tokens_end()-Macro->tokens_begin();
45 MacroExpansionStart = SourceLocation();
47 SourceManager &SM = PP.getSourceManager();
48 MacroStartSLocOffset = SM.getNextLocalOffset();
51 assert(Tokens[0].getLocation().isValid());
52 assert((Tokens[0].getLocation().isFileID() || Tokens[0].is(tok::comment)) &&
53 "Macro defined in macro?");
54 assert(ExpandLocStart.isValid());
56 // Reserve a source location entry chunk for the length of the macro
57 // definition. Tokens that get lexed directly from the definition will
58 // have their locations pointing inside this chunk. This is to avoid
59 // creating separate source location entries for each token.
60 MacroDefStart = SM.getExpansionLoc(Tokens[0].getLocation());
61 MacroDefLength = Macro->getDefinitionLength(SM);
62 MacroExpansionStart = SM.createExpansionLoc(MacroDefStart,
68 // If this is a function-like macro, expand the arguments and change
69 // Tokens to point to the expanded tokens.
70 if (Macro->isFunctionLike() && Macro->getNumParams())
71 ExpandFunctionArguments();
73 // Mark the macro as currently disabled, so that it is not recursively
74 // expanded. The macro must be disabled only after argument pre-expansion of
75 // function-like macro arguments occurs.
76 Macro->DisableMacro();
79 /// Create a TokenLexer for the specified token stream. This does not
80 /// take ownership of the specified token vector.
81 void TokenLexer::Init(const Token *TokArray, unsigned NumToks,
82 bool disableMacroExpansion, bool ownsTokens) {
83 // If the client is reusing a TokenLexer, make sure to free any memory
84 // associated with it.
90 OwnsTokens = ownsTokens;
91 DisableMacroExpansion = disableMacroExpansion;
94 ExpandLocStart = ExpandLocEnd = SourceLocation();
95 AtStartOfLine = false;
96 HasLeadingSpace = false;
97 NextTokGetsSpace = false;
98 MacroExpansionStart = SourceLocation();
100 // Set HasLeadingSpace/AtStartOfLine so that the first token will be
101 // returned unmodified.
103 AtStartOfLine = TokArray[0].isAtStartOfLine();
104 HasLeadingSpace = TokArray[0].hasLeadingSpace();
108 void TokenLexer::destroy() {
109 // If this was a function-like macro that actually uses its arguments, delete
110 // the expanded tokens.
117 // TokenLexer owns its formal arguments.
118 if (ActualArgs) ActualArgs->destroy(PP);
121 bool TokenLexer::MaybeRemoveCommaBeforeVaArgs(
122 SmallVectorImpl<Token> &ResultToks, bool HasPasteOperator, MacroInfo *Macro,
123 unsigned MacroArgNo, Preprocessor &PP) {
124 // Is the macro argument __VA_ARGS__?
125 if (!Macro->isVariadic() || MacroArgNo != Macro->getNumParams()-1)
128 // In Microsoft-compatibility mode, a comma is removed in the expansion
129 // of " ... , __VA_ARGS__ " if __VA_ARGS__ is empty. This extension is
130 // not supported by gcc.
131 if (!HasPasteOperator && !PP.getLangOpts().MSVCCompat)
134 // GCC removes the comma in the expansion of " ... , ## __VA_ARGS__ " if
135 // __VA_ARGS__ is empty, but not in strict C99 mode where there are no
136 // named arguments, where it remains. In all other modes, including C99
137 // with GNU extensions, it is removed regardless of named arguments.
138 // Microsoft also appears to support this extension, unofficially.
139 if (PP.getLangOpts().C99 && !PP.getLangOpts().GNUMode
140 && Macro->getNumParams() < 2)
143 // Is a comma available to be removed?
144 if (ResultToks.empty() || !ResultToks.back().is(tok::comma))
147 // Issue an extension diagnostic for the paste operator.
148 if (HasPasteOperator)
149 PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma);
152 ResultToks.pop_back();
154 if (!ResultToks.empty()) {
155 // If the comma was right after another paste (e.g. "X##,##__VA_ARGS__"),
156 // then removal of the comma should produce a placemarker token (in C99
157 // terms) which we model by popping off the previous ##, giving us a plain
158 // "X" when __VA_ARGS__ is empty.
159 if (ResultToks.back().is(tok::hashhash))
160 ResultToks.pop_back();
162 // Remember that this comma was elided.
163 ResultToks.back().setFlag(Token::CommaAfterElided);
166 // Never add a space, even if the comma, ##, or arg had a space.
167 NextTokGetsSpace = false;
171 /// Expand the arguments of a function-like macro so that we can quickly
172 /// return preexpanded tokens from Tokens.
173 void TokenLexer::ExpandFunctionArguments() {
174 SmallVector<Token, 128> ResultToks;
176 // Loop through 'Tokens', expanding them into ResultToks. Keep
177 // track of whether we change anything. If not, no need to keep them. If so,
178 // we install the newly expanded sequence as the new 'Tokens' list.
179 bool MadeChange = false;
181 for (unsigned i = 0, e = NumTokens; i != e; ++i) {
182 // If we found the stringify operator, get the argument stringified. The
183 // preprocessor already verified that the following token is a macro name
184 // when the #define was parsed.
185 const Token &CurTok = Tokens[i];
186 // We don't want a space for the next token after a paste
187 // operator. In valid code, the token will get smooshed onto the
188 // preceding one anyway. In assembler-with-cpp mode, invalid
189 // pastes are allowed through: in this case, we do not want the
190 // extra whitespace to be added. For example, we want ". ## foo"
191 // -> ".foo" not ". foo".
192 if (i != 0 && !Tokens[i-1].is(tok::hashhash) && CurTok.hasLeadingSpace())
193 NextTokGetsSpace = true;
195 if (CurTok.isOneOf(tok::hash, tok::hashat)) {
196 int ArgNo = Macro->getParameterNum(Tokens[i+1].getIdentifierInfo());
197 assert(ArgNo != -1 && "Token following # is not an argument?");
199 SourceLocation ExpansionLocStart =
200 getExpansionLocForMacroDefLoc(CurTok.getLocation());
201 SourceLocation ExpansionLocEnd =
202 getExpansionLocForMacroDefLoc(Tokens[i+1].getLocation());
205 if (CurTok.is(tok::hash)) // Stringify
206 Res = ActualArgs->getStringifiedArgument(ArgNo, PP,
210 // 'charify': don't bother caching these.
211 Res = MacroArgs::StringifyArgument(ActualArgs->getUnexpArgument(ArgNo),
216 Res.setFlag(Token::StringifiedInMacro);
218 // The stringified/charified string leading space flag gets set to match
219 // the #/#@ operator.
220 if (NextTokGetsSpace)
221 Res.setFlag(Token::LeadingSpace);
223 ResultToks.push_back(Res);
225 ++i; // Skip arg name.
226 NextTokGetsSpace = false;
230 // Find out if there is a paste (##) operator before or after the token.
231 bool NonEmptyPasteBefore =
232 !ResultToks.empty() && ResultToks.back().is(tok::hashhash);
233 bool PasteBefore = i != 0 && Tokens[i-1].is(tok::hashhash);
234 bool PasteAfter = i+1 != e && Tokens[i+1].is(tok::hashhash);
235 assert(!NonEmptyPasteBefore || PasteBefore);
237 // Otherwise, if this is not an argument token, just add the token to the
239 IdentifierInfo *II = CurTok.getIdentifierInfo();
240 int ArgNo = II ? Macro->getParameterNum(II) : -1;
242 // This isn't an argument, just add it.
243 ResultToks.push_back(CurTok);
245 if (NextTokGetsSpace) {
246 ResultToks.back().setFlag(Token::LeadingSpace);
247 NextTokGetsSpace = false;
248 } else if (PasteBefore && !NonEmptyPasteBefore)
249 ResultToks.back().clearFlag(Token::LeadingSpace);
254 // An argument is expanded somehow, the result is different than the
258 // Otherwise, this is a use of the argument.
260 // In Microsoft mode, remove the comma before __VA_ARGS__ to ensure there
261 // are no trailing commas if __VA_ARGS__ is empty.
262 if (!PasteBefore && ActualArgs->isVarargsElidedUse() &&
263 MaybeRemoveCommaBeforeVaArgs(ResultToks,
264 /*HasPasteOperator=*/false,
268 // If it is not the LHS/RHS of a ## operator, we must pre-expand the
269 // argument and substitute the expanded tokens into the result. This is
271 if (!PasteBefore && !PasteAfter) {
272 const Token *ResultArgToks;
274 // Only preexpand the argument if it could possibly need it. This
275 // avoids some work in common cases.
276 const Token *ArgTok = ActualArgs->getUnexpArgument(ArgNo);
277 if (ActualArgs->ArgNeedsPreexpansion(ArgTok, PP))
278 ResultArgToks = &ActualArgs->getPreExpArgument(ArgNo, Macro, PP)[0];
280 ResultArgToks = ArgTok; // Use non-preexpanded tokens.
282 // If the arg token expanded into anything, append it.
283 if (ResultArgToks->isNot(tok::eof)) {
284 size_t FirstResult = ResultToks.size();
285 unsigned NumToks = MacroArgs::getArgLength(ResultArgToks);
286 ResultToks.append(ResultArgToks, ResultArgToks+NumToks);
288 // In Microsoft-compatibility mode, we follow MSVC's preprocessing
289 // behavior by not considering single commas from nested macro
290 // expansions as argument separators. Set a flag on the token so we can
291 // test for this later when the macro expansion is processed.
292 if (PP.getLangOpts().MSVCCompat && NumToks == 1 &&
293 ResultToks.back().is(tok::comma))
294 ResultToks.back().setFlag(Token::IgnoredComma);
296 // If the '##' came from expanding an argument, turn it into 'unknown'
298 for (Token &Tok : llvm::make_range(ResultToks.begin() + FirstResult,
300 if (Tok.is(tok::hashhash))
301 Tok.setKind(tok::unknown);
304 if(ExpandLocStart.isValid()) {
305 updateLocForMacroArgTokens(CurTok.getLocation(),
306 ResultToks.begin()+FirstResult,
310 // If any tokens were substituted from the argument, the whitespace
311 // before the first token should match the whitespace of the arg
313 ResultToks[FirstResult].setFlagValue(Token::LeadingSpace,
315 ResultToks[FirstResult].setFlagValue(Token::StartOfLine, false);
316 NextTokGetsSpace = false;
321 // Okay, we have a token that is either the LHS or RHS of a paste (##)
322 // argument. It gets substituted as its non-pre-expanded tokens.
323 const Token *ArgToks = ActualArgs->getUnexpArgument(ArgNo);
324 unsigned NumToks = MacroArgs::getArgLength(ArgToks);
325 if (NumToks) { // Not an empty argument?
326 bool VaArgsPseudoPaste = false;
327 // If this is the GNU ", ## __VA_ARGS__" extension, and we just learned
328 // that __VA_ARGS__ expands to multiple tokens, avoid a pasting error when
329 // the expander trys to paste ',' with the first token of the __VA_ARGS__
331 if (NonEmptyPasteBefore && ResultToks.size() >= 2 &&
332 ResultToks[ResultToks.size()-2].is(tok::comma) &&
333 (unsigned)ArgNo == Macro->getNumParams()-1 &&
334 Macro->isVariadic()) {
335 VaArgsPseudoPaste = true;
336 // Remove the paste operator, report use of the extension.
337 PP.Diag(ResultToks.pop_back_val().getLocation(), diag::ext_paste_comma);
340 ResultToks.append(ArgToks, ArgToks+NumToks);
342 // If the '##' came from expanding an argument, turn it into 'unknown'
344 for (Token &Tok : llvm::make_range(ResultToks.end() - NumToks,
346 if (Tok.is(tok::hashhash))
347 Tok.setKind(tok::unknown);
350 if (ExpandLocStart.isValid()) {
351 updateLocForMacroArgTokens(CurTok.getLocation(),
352 ResultToks.end()-NumToks, ResultToks.end());
355 // Transfer the leading whitespace information from the token
356 // (the macro argument) onto the first token of the
357 // expansion. Note that we don't do this for the GNU
358 // pseudo-paste extension ", ## __VA_ARGS__".
359 if (!VaArgsPseudoPaste) {
360 ResultToks[ResultToks.size() - NumToks].setFlagValue(Token::StartOfLine,
362 ResultToks[ResultToks.size() - NumToks].setFlagValue(
363 Token::LeadingSpace, NextTokGetsSpace);
366 NextTokGetsSpace = false;
370 // If an empty argument is on the LHS or RHS of a paste, the standard (C99
371 // 6.10.3.3p2,3) calls for a bunch of placemarker stuff to occur. We
372 // implement this by eating ## operators when a LHS or RHS expands to
375 // Discard the argument token and skip (don't copy to the expansion
376 // buffer) the paste operator after it.
381 // If this is on the RHS of a paste operator, we've already copied the
382 // paste operator to the ResultToks list, unless the LHS was empty too.
385 if (NonEmptyPasteBefore) {
386 assert(ResultToks.back().is(tok::hashhash));
387 ResultToks.pop_back();
390 // If this is the __VA_ARGS__ token, and if the argument wasn't provided,
391 // and if the macro had at least one real argument, and if the token before
392 // the ## was a comma, remove the comma. This is a GCC extension which is
393 // disabled when using -std=c99.
394 if (ActualArgs->isVarargsElidedUse())
395 MaybeRemoveCommaBeforeVaArgs(ResultToks,
396 /*HasPasteOperator=*/true,
400 // If anything changed, install this as the new Tokens list.
402 assert(!OwnsTokens && "This would leak if we already own the token list");
403 // This is deleted in the dtor.
404 NumTokens = ResultToks.size();
405 // The tokens will be added to Preprocessor's cache and will be removed
406 // when this TokenLexer finishes lexing them.
407 Tokens = PP.cacheMacroExpandedTokens(this, ResultToks);
409 // The preprocessor cache of macro expanded tokens owns these tokens,not us.
414 /// \brief Checks if two tokens form wide string literal.
415 static bool isWideStringLiteralFromMacro(const Token &FirstTok,
416 const Token &SecondTok) {
417 return FirstTok.is(tok::identifier) &&
418 FirstTok.getIdentifierInfo()->isStr("L") && SecondTok.isLiteral() &&
419 SecondTok.stringifiedInMacro();
422 /// Lex - Lex and return a token from this macro stream.
424 bool TokenLexer::Lex(Token &Tok) {
425 // Lexing off the end of the macro, pop this macro off the expansion stack.
427 // If this is a macro (not a token stream), mark the macro enabled now
428 // that it is no longer being expanded.
429 if (Macro) Macro->EnableMacro();
432 Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
433 Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace || NextTokGetsSpace);
435 Tok.setFlag(Token::LeadingEmptyMacro);
436 return PP.HandleEndOfTokenLexer(Tok);
439 SourceManager &SM = PP.getSourceManager();
441 // If this is the first token of the expanded result, we inherit spacing
443 bool isFirstToken = CurToken == 0;
445 // Get the next token to return.
446 Tok = Tokens[CurToken++];
448 bool TokenIsFromPaste = false;
450 // If this token is followed by a token paste (##) operator, paste the tokens!
451 // Note that ## is a normal token when not expanding a macro.
452 if (!isAtEnd() && Macro &&
453 (Tokens[CurToken].is(tok::hashhash) ||
454 // Special processing of L#x macros in -fms-compatibility mode.
455 // Microsoft compiler is able to form a wide string literal from
456 // 'L#macro_arg' construct in a function-like macro.
457 (PP.getLangOpts().MSVCCompat &&
458 isWideStringLiteralFromMacro(Tok, Tokens[CurToken])))) {
459 // When handling the microsoft /##/ extension, the final token is
460 // returned by PasteTokens, not the pasted token.
461 if (PasteTokens(Tok))
464 TokenIsFromPaste = true;
467 // The token's current location indicate where the token was lexed from. We
468 // need this information to compute the spelling of the token, but any
469 // diagnostics for the expanded token should appear as if they came from
470 // ExpansionLoc. Pull this information together into a new SourceLocation
471 // that captures all of this.
472 if (ExpandLocStart.isValid() && // Don't do this for token streams.
473 // Check that the token's location was not already set properly.
474 SM.isBeforeInSLocAddrSpace(Tok.getLocation(), MacroStartSLocOffset)) {
475 SourceLocation instLoc;
476 if (Tok.is(tok::comment)) {
477 instLoc = SM.createExpansionLoc(Tok.getLocation(),
482 instLoc = getExpansionLocForMacroDefLoc(Tok.getLocation());
485 Tok.setLocation(instLoc);
488 // If this is the first token, set the lexical properties of the token to
489 // match the lexical properties of the macro identifier.
491 Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
492 Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace);
494 // If this is not the first token, we may still need to pass through
495 // leading whitespace if we've expanded a macro.
496 if (AtStartOfLine) Tok.setFlag(Token::StartOfLine);
497 if (HasLeadingSpace) Tok.setFlag(Token::LeadingSpace);
499 AtStartOfLine = false;
500 HasLeadingSpace = false;
502 // Handle recursive expansion!
503 if (!Tok.isAnnotation() && Tok.getIdentifierInfo() != nullptr) {
504 // Change the kind of this identifier to the appropriate token kind, e.g.
505 // turning "for" into a keyword.
506 IdentifierInfo *II = Tok.getIdentifierInfo();
507 Tok.setKind(II->getTokenID());
509 // If this identifier was poisoned and from a paste, emit an error. This
510 // won't be handled by Preprocessor::HandleIdentifier because this is coming
511 // from a macro expansion.
512 if (II->isPoisoned() && TokenIsFromPaste) {
513 PP.HandlePoisonedIdentifier(Tok);
516 if (!DisableMacroExpansion && II->isHandleIdentifierCase())
517 return PP.HandleIdentifier(Tok);
520 // Otherwise, return a normal token.
524 /// PasteTokens - Tok is the LHS of a ## operator, and CurToken is the ##
525 /// operator. Read the ## and RHS, and paste the LHS/RHS together. If there
526 /// are more ## after it, chomp them iteratively. Return the result as Tok.
527 /// If this returns true, the caller should immediately return the token.
528 bool TokenLexer::PasteTokens(Token &Tok) {
529 // MSVC: If previous token was pasted, this must be a recovery from an invalid
530 // paste operation. Ignore spaces before this token to mimic MSVC output.
531 // Required for generating valid UUID strings in some MS headers.
532 if (PP.getLangOpts().MicrosoftExt && (CurToken >= 2) &&
533 Tokens[CurToken - 2].is(tok::hashhash))
534 Tok.clearFlag(Token::LeadingSpace);
536 SmallString<128> Buffer;
537 const char *ResultTokStrPtr = nullptr;
538 SourceLocation StartLoc = Tok.getLocation();
539 SourceLocation PasteOpLoc;
541 // Consume the ## operator if any.
542 PasteOpLoc = Tokens[CurToken].getLocation();
543 if (Tokens[CurToken].is(tok::hashhash))
545 assert(!isAtEnd() && "No token on the RHS of a paste operator!");
547 // Get the RHS token.
548 const Token &RHS = Tokens[CurToken];
550 // Allocate space for the result token. This is guaranteed to be enough for
552 Buffer.resize(Tok.getLength() + RHS.getLength());
554 // Get the spelling of the LHS token in Buffer.
555 const char *BufPtr = &Buffer[0];
556 bool Invalid = false;
557 unsigned LHSLen = PP.getSpelling(Tok, BufPtr, &Invalid);
558 if (BufPtr != &Buffer[0]) // Really, we want the chars in Buffer!
559 memcpy(&Buffer[0], BufPtr, LHSLen);
563 BufPtr = Buffer.data() + LHSLen;
564 unsigned RHSLen = PP.getSpelling(RHS, BufPtr, &Invalid);
567 if (RHSLen && BufPtr != &Buffer[LHSLen])
568 // Really, we want the chars in Buffer!
569 memcpy(&Buffer[LHSLen], BufPtr, RHSLen);
571 // Trim excess space.
572 Buffer.resize(LHSLen+RHSLen);
574 // Plop the pasted result (including the trailing newline and null) into a
575 // scratch buffer where we can lex it.
577 ResultTokTmp.startToken();
579 // Claim that the tmp token is a string_literal so that we can get the
580 // character pointer back from CreateString in getLiteralData().
581 ResultTokTmp.setKind(tok::string_literal);
582 PP.CreateString(Buffer, ResultTokTmp);
583 SourceLocation ResultTokLoc = ResultTokTmp.getLocation();
584 ResultTokStrPtr = ResultTokTmp.getLiteralData();
586 // Lex the resultant pasted token into Result.
589 if (Tok.isAnyIdentifier() && RHS.isAnyIdentifier()) {
590 // Common paste case: identifier+identifier = identifier. Avoid creating
591 // a lexer and other overhead.
592 PP.IncrementPasteCounter(true);
594 Result.setKind(tok::raw_identifier);
595 Result.setRawIdentifierData(ResultTokStrPtr);
596 Result.setLocation(ResultTokLoc);
597 Result.setLength(LHSLen+RHSLen);
599 PP.IncrementPasteCounter(false);
601 assert(ResultTokLoc.isFileID() &&
602 "Should be a raw location into scratch buffer");
603 SourceManager &SourceMgr = PP.getSourceManager();
604 FileID LocFileID = SourceMgr.getFileID(ResultTokLoc);
606 bool Invalid = false;
607 const char *ScratchBufStart
608 = SourceMgr.getBufferData(LocFileID, &Invalid).data();
612 // Make a lexer to lex this string from. Lex just this one token.
613 // Make a lexer object so that we lex and expand the paste result.
614 Lexer TL(SourceMgr.getLocForStartOfFile(LocFileID),
615 PP.getLangOpts(), ScratchBufStart,
616 ResultTokStrPtr, ResultTokStrPtr+LHSLen+RHSLen);
618 // Lex a token in raw mode. This way it won't look up identifiers
619 // automatically, lexing off the end will return an eof token, and
620 // warnings are disabled. This returns true if the result token is the
622 bool isInvalid = !TL.LexFromRawLexer(Result);
624 // If we got an EOF token, we didn't form even ONE token. For example, we
625 // did "/ ## /" to get "//".
626 isInvalid |= Result.is(tok::eof);
628 // If pasting the two tokens didn't form a full new token, this is an
629 // error. This occurs with "x ## +" and other stuff. Return with Tok
630 // unmodified and with RHS as the next token to lex.
632 // Explicitly convert the token location to have proper expansion
633 // information so that the user knows where it came from.
634 SourceManager &SM = PP.getSourceManager();
636 SM.createExpansionLoc(PasteOpLoc, ExpandLocStart, ExpandLocEnd, 2);
638 // Test for the Microsoft extension of /##/ turning into // here on the
640 if (PP.getLangOpts().MicrosoftExt && Tok.is(tok::slash) &&
641 RHS.is(tok::slash)) {
642 HandleMicrosoftCommentPaste(Tok, Loc);
646 // Do not emit the error when preprocessing assembler code.
647 if (!PP.getLangOpts().AsmPreprocessor) {
648 // If we're in microsoft extensions mode, downgrade this from a hard
649 // error to an extension that defaults to an error. This allows
651 PP.Diag(Loc, PP.getLangOpts().MicrosoftExt ? diag::ext_pp_bad_paste_ms
652 : diag::err_pp_bad_paste)
656 // An error has occurred so exit loop.
660 // Turn ## into 'unknown' to avoid # ## # from looking like a paste
662 if (Result.is(tok::hashhash))
663 Result.setKind(tok::unknown);
666 // Transfer properties of the LHS over the Result.
667 Result.setFlagValue(Token::StartOfLine , Tok.isAtStartOfLine());
668 Result.setFlagValue(Token::LeadingSpace, Tok.hasLeadingSpace());
670 // Finally, replace LHS with the result, consume the RHS, and iterate.
673 } while (!isAtEnd() && Tokens[CurToken].is(tok::hashhash));
675 SourceLocation EndLoc = Tokens[CurToken - 1].getLocation();
677 // The token's current location indicate where the token was lexed from. We
678 // need this information to compute the spelling of the token, but any
679 // diagnostics for the expanded token should appear as if the token was
680 // expanded from the full ## expression. Pull this information together into
681 // a new SourceLocation that captures all of this.
682 SourceManager &SM = PP.getSourceManager();
683 if (StartLoc.isFileID())
684 StartLoc = getExpansionLocForMacroDefLoc(StartLoc);
685 if (EndLoc.isFileID())
686 EndLoc = getExpansionLocForMacroDefLoc(EndLoc);
687 FileID MacroFID = SM.getFileID(MacroExpansionStart);
688 while (SM.getFileID(StartLoc) != MacroFID)
689 StartLoc = SM.getImmediateExpansionRange(StartLoc).first;
690 while (SM.getFileID(EndLoc) != MacroFID)
691 EndLoc = SM.getImmediateExpansionRange(EndLoc).second;
693 Tok.setLocation(SM.createExpansionLoc(Tok.getLocation(), StartLoc, EndLoc,
696 // Now that we got the result token, it will be subject to expansion. Since
697 // token pasting re-lexes the result token in raw mode, identifier information
698 // isn't looked up. As such, if the result is an identifier, look up id info.
699 if (Tok.is(tok::raw_identifier)) {
700 // Look up the identifier info for the token. We disabled identifier lookup
701 // by saying we're skipping contents, so we need to do this manually.
702 PP.LookUpIdentifierInfo(Tok);
707 /// isNextTokenLParen - If the next token lexed will pop this macro off the
708 /// expansion stack, return 2. If the next unexpanded token is a '(', return
709 /// 1, otherwise return 0.
710 unsigned TokenLexer::isNextTokenLParen() const {
714 return Tokens[CurToken].is(tok::l_paren);
717 /// isParsingPreprocessorDirective - Return true if we are in the middle of a
718 /// preprocessor directive.
719 bool TokenLexer::isParsingPreprocessorDirective() const {
720 return Tokens[NumTokens-1].is(tok::eod) && !isAtEnd();
723 /// HandleMicrosoftCommentPaste - In microsoft compatibility mode, /##/ pastes
724 /// together to form a comment that comments out everything in the current
725 /// macro, other active macros, and anything left on the current physical
726 /// source line of the expanded buffer. Handle this by returning the
727 /// first token on the next line.
728 void TokenLexer::HandleMicrosoftCommentPaste(Token &Tok, SourceLocation OpLoc) {
729 PP.Diag(OpLoc, diag::ext_comment_paste_microsoft);
731 // We 'comment out' the rest of this macro by just ignoring the rest of the
732 // tokens that have not been lexed yet, if any.
734 // Since this must be a macro, mark the macro enabled now that it is no longer
736 assert(Macro && "Token streams can't paste comments");
737 Macro->EnableMacro();
739 PP.HandleMicrosoftCommentPaste(Tok);
742 /// \brief If \arg loc is a file ID and points inside the current macro
743 /// definition, returns the appropriate source location pointing at the
744 /// macro expansion source location entry, otherwise it returns an invalid
747 TokenLexer::getExpansionLocForMacroDefLoc(SourceLocation loc) const {
748 assert(ExpandLocStart.isValid() && MacroExpansionStart.isValid() &&
749 "Not appropriate for token streams");
750 assert(loc.isValid() && loc.isFileID());
752 SourceManager &SM = PP.getSourceManager();
753 assert(SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength) &&
754 "Expected loc to come from the macro definition");
756 unsigned relativeOffset = 0;
757 SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength, &relativeOffset);
758 return MacroExpansionStart.getLocWithOffset(relativeOffset);
761 /// \brief Finds the tokens that are consecutive (from the same FileID)
762 /// creates a single SLocEntry, and assigns SourceLocations to each token that
763 /// point to that SLocEntry. e.g for
764 /// assert(foo == bar);
765 /// There will be a single SLocEntry for the "foo == bar" chunk and locations
766 /// for the 'foo', '==', 'bar' tokens will point inside that chunk.
768 /// \arg begin_tokens will be updated to a position past all the found
769 /// consecutive tokens.
770 static void updateConsecutiveMacroArgTokens(SourceManager &SM,
771 SourceLocation InstLoc,
772 Token *&begin_tokens,
773 Token * end_tokens) {
774 assert(begin_tokens < end_tokens);
776 SourceLocation FirstLoc = begin_tokens->getLocation();
777 SourceLocation CurLoc = FirstLoc;
779 // Compare the source location offset of tokens and group together tokens that
780 // are close, even if their locations point to different FileIDs. e.g.
782 // |bar | foo | cake | (3 tokens from 3 consecutive FileIDs)
784 // |bar foo cake| (one SLocEntry chunk for all tokens)
786 // we can perform this "merge" since the token's spelling location depends
787 // on the relative offset.
789 Token *NextTok = begin_tokens + 1;
790 for (; NextTok < end_tokens; ++NextTok) {
791 SourceLocation NextLoc = NextTok->getLocation();
792 if (CurLoc.isFileID() != NextLoc.isFileID())
793 break; // Token from different kind of FileID.
796 if (!SM.isInSameSLocAddrSpace(CurLoc, NextLoc, &RelOffs))
797 break; // Token from different local/loaded location.
798 // Check that token is not before the previous token or more than 50
799 // "characters" away.
800 if (RelOffs < 0 || RelOffs > 50)
803 if (CurLoc.isMacroID() && !SM.isWrittenInSameFile(CurLoc, NextLoc))
804 break; // Token from a different macro.
809 // For the consecutive tokens, find the length of the SLocEntry to contain
811 Token &LastConsecutiveTok = *(NextTok-1);
813 SM.isInSameSLocAddrSpace(FirstLoc, LastConsecutiveTok.getLocation(),
815 unsigned FullLength = LastRelOffs + LastConsecutiveTok.getLength();
817 // Create a macro expansion SLocEntry that will "contain" all of the tokens.
818 SourceLocation Expansion =
819 SM.createMacroArgExpansionLoc(FirstLoc, InstLoc,FullLength);
821 // Change the location of the tokens from the spelling location to the new
822 // expanded location.
823 for (; begin_tokens < NextTok; ++begin_tokens) {
824 Token &Tok = *begin_tokens;
826 SM.isInSameSLocAddrSpace(FirstLoc, Tok.getLocation(), &RelOffs);
827 Tok.setLocation(Expansion.getLocWithOffset(RelOffs));
831 /// \brief Creates SLocEntries and updates the locations of macro argument
832 /// tokens to their new expanded locations.
834 /// \param ArgIdDefLoc the location of the macro argument id inside the macro
836 /// \param Tokens the macro argument tokens to update.
837 void TokenLexer::updateLocForMacroArgTokens(SourceLocation ArgIdSpellLoc,
840 SourceManager &SM = PP.getSourceManager();
842 SourceLocation InstLoc =
843 getExpansionLocForMacroDefLoc(ArgIdSpellLoc);
845 while (begin_tokens < end_tokens) {
846 // If there's only one token just create a SLocEntry for it.
847 if (end_tokens - begin_tokens == 1) {
848 Token &Tok = *begin_tokens;
849 Tok.setLocation(SM.createMacroArgExpansionLoc(Tok.getLocation(),
855 updateConsecutiveMacroArgTokens(SM, InstLoc, begin_tokens, end_tokens);
859 void TokenLexer::PropagateLineStartLeadingSpaceInfo(Token &Result) {
860 AtStartOfLine = Result.isAtStartOfLine();
861 HasLeadingSpace = Result.hasLeadingSpace();