1 //===--- CommentLexer.cpp -------------------------------------------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 #include "clang/AST/CommentLexer.h"
10 #include "clang/AST/CommentCommandTraits.h"
11 #include "clang/AST/CommentDiagnostic.h"
12 #include "clang/Basic/CharInfo.h"
13 #include "llvm/ADT/StringExtras.h"
14 #include "llvm/ADT/StringSwitch.h"
15 #include "llvm/Support/ConvertUTF.h"
16 #include "llvm/Support/ErrorHandling.h"
21 void Token::dump(const Lexer &L, const SourceManager &SM) const {
22 llvm::errs() << "comments::Token Kind=" << Kind << " ";
23 Loc.print(llvm::errs(), SM);
24 llvm::errs() << " " << Length << " \"" << L.getSpelling(*this, SM) << "\"\n";
27 static inline bool isHTMLNamedCharacterReferenceCharacter(char C) {
31 static inline bool isHTMLDecimalCharacterReferenceCharacter(char C) {
35 static inline bool isHTMLHexCharacterReferenceCharacter(char C) {
39 static inline StringRef convertCodePointToUTF8(
40 llvm::BumpPtrAllocator &Allocator,
42 char *Resolved = Allocator.Allocate<char>(UNI_MAX_UTF8_BYTES_PER_CODE_POINT);
43 char *ResolvedPtr = Resolved;
44 if (llvm::ConvertCodePointToUTF8(CodePoint, ResolvedPtr))
45 return StringRef(Resolved, ResolvedPtr - Resolved);
52 #include "clang/AST/CommentHTMLTags.inc"
53 #include "clang/AST/CommentHTMLNamedCharacterReferences.inc"
55 } // end anonymous namespace
57 StringRef Lexer::resolveHTMLNamedCharacterReference(StringRef Name) const {
58 // Fast path, first check a few most widely used named character references.
59 return llvm::StringSwitch<StringRef>(Name)
66 .Default(translateHTMLNamedCharacterReferenceToUTF8(Name));
69 StringRef Lexer::resolveHTMLDecimalCharacterReference(StringRef Name) const {
70 unsigned CodePoint = 0;
71 for (unsigned i = 0, e = Name.size(); i != e; ++i) {
72 assert(isHTMLDecimalCharacterReferenceCharacter(Name[i]));
74 CodePoint += Name[i] - '0';
76 return convertCodePointToUTF8(Allocator, CodePoint);
79 StringRef Lexer::resolveHTMLHexCharacterReference(StringRef Name) const {
80 unsigned CodePoint = 0;
81 for (unsigned i = 0, e = Name.size(); i != e; ++i) {
83 const char C = Name[i];
84 assert(isHTMLHexCharacterReferenceCharacter(C));
85 CodePoint += llvm::hexDigitValue(C);
87 return convertCodePointToUTF8(Allocator, CodePoint);
90 void Lexer::skipLineStartingDecorations() {
91 // This function should be called only for C comments
92 assert(CommentState == LCS_InsideCComment);
94 if (BufferPtr == CommentEnd)
102 const char *NewBufferPtr = BufferPtr;
104 if (NewBufferPtr == CommentEnd)
107 char C = *NewBufferPtr;
108 while (isHorizontalWhitespace(C)) {
110 if (NewBufferPtr == CommentEnd)
115 BufferPtr = NewBufferPtr + 1;
125 /// Returns pointer to the first newline character in the string.
126 const char *findNewline(const char *BufferPtr, const char *BufferEnd) {
127 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
128 if (isVerticalWhitespace(*BufferPtr))
134 const char *skipNewline(const char *BufferPtr, const char *BufferEnd) {
135 if (BufferPtr == BufferEnd)
138 if (*BufferPtr == '\n')
141 assert(*BufferPtr == '\r');
143 if (BufferPtr != BufferEnd && *BufferPtr == '\n')
149 const char *skipNamedCharacterReference(const char *BufferPtr,
150 const char *BufferEnd) {
151 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
152 if (!isHTMLNamedCharacterReferenceCharacter(*BufferPtr))
158 const char *skipDecimalCharacterReference(const char *BufferPtr,
159 const char *BufferEnd) {
160 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
161 if (!isHTMLDecimalCharacterReferenceCharacter(*BufferPtr))
167 const char *skipHexCharacterReference(const char *BufferPtr,
168 const char *BufferEnd) {
169 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
170 if (!isHTMLHexCharacterReferenceCharacter(*BufferPtr))
176 bool isHTMLIdentifierStartingCharacter(char C) {
180 bool isHTMLIdentifierCharacter(char C) {
181 return isAlphanumeric(C);
184 const char *skipHTMLIdentifier(const char *BufferPtr, const char *BufferEnd) {
185 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
186 if (!isHTMLIdentifierCharacter(*BufferPtr))
192 /// Skip HTML string quoted in single or double quotes. Escaping quotes inside
195 /// Returns pointer to closing quote.
196 const char *skipHTMLQuotedString(const char *BufferPtr, const char *BufferEnd)
198 const char Quote = *BufferPtr;
199 assert(Quote == '\"' || Quote == '\'');
202 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
203 const char C = *BufferPtr;
204 if (C == Quote && BufferPtr[-1] != '\\')
210 const char *skipWhitespace(const char *BufferPtr, const char *BufferEnd) {
211 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
212 if (!isWhitespace(*BufferPtr))
218 bool isWhitespace(const char *BufferPtr, const char *BufferEnd) {
219 return skipWhitespace(BufferPtr, BufferEnd) == BufferEnd;
222 bool isCommandNameStartCharacter(char C) {
226 bool isCommandNameCharacter(char C) {
227 return isAlphanumeric(C);
230 const char *skipCommandName(const char *BufferPtr, const char *BufferEnd) {
231 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
232 if (!isCommandNameCharacter(*BufferPtr))
238 /// Return the one past end pointer for BCPL comments.
239 /// Handles newlines escaped with backslash or trigraph for backslahs.
240 const char *findBCPLCommentEnd(const char *BufferPtr, const char *BufferEnd) {
241 const char *CurPtr = BufferPtr;
242 while (CurPtr != BufferEnd) {
243 while (!isVerticalWhitespace(*CurPtr)) {
245 if (CurPtr == BufferEnd)
248 // We found a newline, check if it is escaped.
249 const char *EscapePtr = CurPtr - 1;
250 while(isHorizontalWhitespace(*EscapePtr))
253 if (*EscapePtr == '\\' ||
254 (EscapePtr - 2 >= BufferPtr && EscapePtr[0] == '/' &&
255 EscapePtr[-1] == '?' && EscapePtr[-2] == '?')) {
256 // We found an escaped newline.
257 CurPtr = skipNewline(CurPtr, BufferEnd);
259 return CurPtr; // Not an escaped newline.
264 /// Return the one past end pointer for C comments.
265 /// Very dumb, does not handle escaped newlines or trigraphs.
266 const char *findCCommentEnd(const char *BufferPtr, const char *BufferEnd) {
267 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
268 if (*BufferPtr == '*') {
269 assert(BufferPtr + 1 != BufferEnd);
270 if (*(BufferPtr + 1) == '/')
274 llvm_unreachable("buffer end hit before '*/' was seen");
277 } // end anonymous namespace
279 void Lexer::formTokenWithChars(Token &Result, const char *TokEnd,
280 tok::TokenKind Kind) {
281 const unsigned TokLen = TokEnd - BufferPtr;
282 Result.setLocation(getSourceLocation(BufferPtr));
283 Result.setKind(Kind);
284 Result.setLength(TokLen);
286 Result.TextPtr = "<UNSET>";
292 void Lexer::lexCommentText(Token &T) {
293 assert(CommentState == LCS_InsideBCPLComment ||
294 CommentState == LCS_InsideCComment);
296 // Handles lexing non-command text, i.e. text and newline.
297 auto HandleNonCommandToken = [&]() -> void {
298 assert(State == LS_Normal);
300 const char *TokenPtr = BufferPtr;
301 assert(TokenPtr < CommentEnd);
305 TokenPtr = skipNewline(TokenPtr, CommentEnd);
306 formTokenWithChars(T, TokenPtr, tok::newline);
308 if (CommentState == LCS_InsideCComment)
309 skipLineStartingDecorations();
313 StringRef TokStartSymbols = ParseCommands ? "\n\r\\@&<" : "\n\r";
314 size_t End = StringRef(TokenPtr, CommentEnd - TokenPtr)
315 .find_first_of(TokStartSymbols);
316 if (End != StringRef::npos)
319 TokenPtr = CommentEnd;
320 formTextToken(T, TokenPtr);
327 return HandleNonCommandToken();
332 case LS_VerbatimBlockFirstLine:
333 lexVerbatimBlockFirstLine(T);
335 case LS_VerbatimBlockBody:
336 lexVerbatimBlockBody(T);
338 case LS_VerbatimLineText:
339 lexVerbatimLineText(T);
341 case LS_HTMLStartTag:
349 assert(State == LS_Normal);
350 const char *TokenPtr = BufferPtr;
351 assert(TokenPtr < CommentEnd);
355 // Commands that start with a backslash and commands that start with
356 // 'at' have equivalent semantics. But we keep information about the
357 // exact syntax in AST for comments.
358 tok::TokenKind CommandKind =
359 (*TokenPtr == '@') ? tok::at_command : tok::backslash_command;
361 if (TokenPtr == CommentEnd) {
362 formTextToken(T, TokenPtr);
370 case '\\': case '@': case '&': case '$':
371 case '#': case '<': case '>': case '%':
372 case '\"': case '.': case ':':
373 // This is one of \\ \@ \& \$ etc escape sequences.
375 if (C == ':' && TokenPtr != CommentEnd && *TokenPtr == ':') {
376 // This is the \:: escape sequence.
379 StringRef UnescapedText(BufferPtr + 1, TokenPtr - (BufferPtr + 1));
380 formTokenWithChars(T, TokenPtr, tok::text);
381 T.setText(UnescapedText);
385 // Don't make zero-length commands.
386 if (!isCommandNameStartCharacter(*TokenPtr)) {
387 formTextToken(T, TokenPtr);
391 TokenPtr = skipCommandName(TokenPtr, CommentEnd);
392 unsigned Length = TokenPtr - (BufferPtr + 1);
394 // Hardcoded support for lexing LaTeX formula commands
395 // \f$ \f[ \f] \f{ \f} as a single command.
396 if (Length == 1 && TokenPtr[-1] == 'f' && TokenPtr != CommentEnd) {
398 if (C == '$' || C == '[' || C == ']' || C == '{' || C == '}') {
404 StringRef CommandName(BufferPtr + 1, Length);
406 const CommandInfo *Info = Traits.getCommandInfoOrNULL(CommandName);
408 if ((Info = Traits.getTypoCorrectCommandInfo(CommandName))) {
409 StringRef CorrectedName = Info->Name;
410 SourceLocation Loc = getSourceLocation(BufferPtr);
411 SourceLocation EndLoc = getSourceLocation(TokenPtr);
412 SourceRange FullRange = SourceRange(Loc, EndLoc);
413 SourceRange CommandRange(Loc.getLocWithOffset(1), EndLoc);
414 Diag(Loc, diag::warn_correct_comment_command_name)
415 << FullRange << CommandName << CorrectedName
416 << FixItHint::CreateReplacement(CommandRange, CorrectedName);
418 formTokenWithChars(T, TokenPtr, tok::unknown_command);
419 T.setUnknownCommandName(CommandName);
420 Diag(T.getLocation(), diag::warn_unknown_comment_command_name)
421 << SourceRange(T.getLocation(), T.getEndLocation());
425 if (Info->IsVerbatimBlockCommand) {
426 setupAndLexVerbatimBlock(T, TokenPtr, *BufferPtr, Info);
429 if (Info->IsVerbatimLineCommand) {
430 setupAndLexVerbatimLine(T, TokenPtr, Info);
433 formTokenWithChars(T, TokenPtr, CommandKind);
434 T.setCommandID(Info->getID());
439 lexHTMLCharacterReference(T);
444 if (TokenPtr == CommentEnd) {
445 formTextToken(T, TokenPtr);
448 const char C = *TokenPtr;
449 if (isHTMLIdentifierStartingCharacter(C))
450 setupAndLexHTMLStartTag(T);
452 setupAndLexHTMLEndTag(T);
454 formTextToken(T, TokenPtr);
459 return HandleNonCommandToken();
463 void Lexer::setupAndLexVerbatimBlock(Token &T,
464 const char *TextBegin,
465 char Marker, const CommandInfo *Info) {
466 assert(Info->IsVerbatimBlockCommand);
468 VerbatimBlockEndCommandName.clear();
469 VerbatimBlockEndCommandName.append(Marker == '\\' ? "\\" : "@");
470 VerbatimBlockEndCommandName.append(Info->EndCommandName);
472 formTokenWithChars(T, TextBegin, tok::verbatim_block_begin);
473 T.setVerbatimBlockID(Info->getID());
475 // If there is a newline following the verbatim opening command, skip the
476 // newline so that we don't create an tok::verbatim_block_line with empty
478 if (BufferPtr != CommentEnd &&
479 isVerticalWhitespace(*BufferPtr)) {
480 BufferPtr = skipNewline(BufferPtr, CommentEnd);
481 State = LS_VerbatimBlockBody;
485 State = LS_VerbatimBlockFirstLine;
488 void Lexer::lexVerbatimBlockFirstLine(Token &T) {
490 assert(BufferPtr < CommentEnd);
492 // FIXME: It would be better to scan the text once, finding either the block
493 // end command or newline.
495 // Extract current line.
496 const char *Newline = findNewline(BufferPtr, CommentEnd);
497 StringRef Line(BufferPtr, Newline - BufferPtr);
499 // Look for end command in current line.
500 size_t Pos = Line.find(VerbatimBlockEndCommandName);
502 const char *NextLine;
503 if (Pos == StringRef::npos) {
504 // Current line is completely verbatim.
506 NextLine = skipNewline(Newline, CommentEnd);
507 } else if (Pos == 0) {
508 // Current line contains just an end command.
509 const char *End = BufferPtr + VerbatimBlockEndCommandName.size();
510 StringRef Name(BufferPtr + 1, End - (BufferPtr + 1));
511 formTokenWithChars(T, End, tok::verbatim_block_end);
512 T.setVerbatimBlockID(Traits.getCommandInfo(Name)->getID());
516 // There is some text, followed by end command. Extract text first.
517 TextEnd = BufferPtr + Pos;
519 // If there is only whitespace before end command, skip whitespace.
520 if (isWhitespace(BufferPtr, TextEnd)) {
526 StringRef Text(BufferPtr, TextEnd - BufferPtr);
527 formTokenWithChars(T, NextLine, tok::verbatim_block_line);
528 T.setVerbatimBlockText(Text);
530 State = LS_VerbatimBlockBody;
533 void Lexer::lexVerbatimBlockBody(Token &T) {
534 assert(State == LS_VerbatimBlockBody);
536 if (CommentState == LCS_InsideCComment)
537 skipLineStartingDecorations();
539 if (BufferPtr == CommentEnd) {
540 formTokenWithChars(T, BufferPtr, tok::verbatim_block_line);
541 T.setVerbatimBlockText("");
545 lexVerbatimBlockFirstLine(T);
548 void Lexer::setupAndLexVerbatimLine(Token &T, const char *TextBegin,
549 const CommandInfo *Info) {
550 assert(Info->IsVerbatimLineCommand);
551 formTokenWithChars(T, TextBegin, tok::verbatim_line_name);
552 T.setVerbatimLineID(Info->getID());
554 State = LS_VerbatimLineText;
557 void Lexer::lexVerbatimLineText(Token &T) {
558 assert(State == LS_VerbatimLineText);
560 // Extract current line.
561 const char *Newline = findNewline(BufferPtr, CommentEnd);
562 StringRef Text(BufferPtr, Newline - BufferPtr);
563 formTokenWithChars(T, Newline, tok::verbatim_line_text);
564 T.setVerbatimLineText(Text);
569 void Lexer::lexHTMLCharacterReference(Token &T) {
570 const char *TokenPtr = BufferPtr;
571 assert(*TokenPtr == '&');
573 if (TokenPtr == CommentEnd) {
574 formTextToken(T, TokenPtr);
578 bool isNamed = false;
579 bool isDecimal = false;
581 if (isHTMLNamedCharacterReferenceCharacter(C)) {
583 TokenPtr = skipNamedCharacterReference(TokenPtr, CommentEnd);
585 } else if (C == '#') {
587 if (TokenPtr == CommentEnd) {
588 formTextToken(T, TokenPtr);
592 if (isHTMLDecimalCharacterReferenceCharacter(C)) {
594 TokenPtr = skipDecimalCharacterReference(TokenPtr, CommentEnd);
596 } else if (C == 'x' || C == 'X') {
599 TokenPtr = skipHexCharacterReference(TokenPtr, CommentEnd);
601 formTextToken(T, TokenPtr);
605 formTextToken(T, TokenPtr);
608 if (NamePtr == TokenPtr || TokenPtr == CommentEnd ||
610 formTextToken(T, TokenPtr);
613 StringRef Name(NamePtr, TokenPtr - NamePtr);
614 TokenPtr++; // Skip semicolon.
617 Resolved = resolveHTMLNamedCharacterReference(Name);
619 Resolved = resolveHTMLDecimalCharacterReference(Name);
621 Resolved = resolveHTMLHexCharacterReference(Name);
623 if (Resolved.empty()) {
624 formTextToken(T, TokenPtr);
627 formTokenWithChars(T, TokenPtr, tok::text);
631 void Lexer::setupAndLexHTMLStartTag(Token &T) {
632 assert(BufferPtr[0] == '<' &&
633 isHTMLIdentifierStartingCharacter(BufferPtr[1]));
634 const char *TagNameEnd = skipHTMLIdentifier(BufferPtr + 2, CommentEnd);
635 StringRef Name(BufferPtr + 1, TagNameEnd - (BufferPtr + 1));
636 if (!isHTMLTagName(Name)) {
637 formTextToken(T, TagNameEnd);
641 formTokenWithChars(T, TagNameEnd, tok::html_start_tag);
642 T.setHTMLTagStartName(Name);
644 BufferPtr = skipWhitespace(BufferPtr, CommentEnd);
646 const char C = *BufferPtr;
647 if (BufferPtr != CommentEnd &&
648 (C == '>' || C == '/' || isHTMLIdentifierStartingCharacter(C)))
649 State = LS_HTMLStartTag;
652 void Lexer::lexHTMLStartTag(Token &T) {
653 assert(State == LS_HTMLStartTag);
655 const char *TokenPtr = BufferPtr;
657 if (isHTMLIdentifierCharacter(C)) {
658 TokenPtr = skipHTMLIdentifier(TokenPtr, CommentEnd);
659 StringRef Ident(BufferPtr, TokenPtr - BufferPtr);
660 formTokenWithChars(T, TokenPtr, tok::html_ident);
661 T.setHTMLIdent(Ident);
666 formTokenWithChars(T, TokenPtr, tok::html_equals);
670 const char *OpenQuote = TokenPtr;
671 TokenPtr = skipHTMLQuotedString(TokenPtr, CommentEnd);
672 const char *ClosingQuote = TokenPtr;
673 if (TokenPtr != CommentEnd) // Skip closing quote.
675 formTokenWithChars(T, TokenPtr, tok::html_quoted_string);
676 T.setHTMLQuotedString(StringRef(OpenQuote + 1,
677 ClosingQuote - (OpenQuote + 1)));
682 formTokenWithChars(T, TokenPtr, tok::html_greater);
687 if (TokenPtr != CommentEnd && *TokenPtr == '>') {
689 formTokenWithChars(T, TokenPtr, tok::html_slash_greater);
691 formTextToken(T, TokenPtr);
698 // Now look ahead and return to normal state if we don't see any HTML tokens
700 BufferPtr = skipWhitespace(BufferPtr, CommentEnd);
701 if (BufferPtr == CommentEnd) {
707 if (!isHTMLIdentifierStartingCharacter(C) &&
708 C != '=' && C != '\"' && C != '\'' && C != '>') {
714 void Lexer::setupAndLexHTMLEndTag(Token &T) {
715 assert(BufferPtr[0] == '<' && BufferPtr[1] == '/');
717 const char *TagNameBegin = skipWhitespace(BufferPtr + 2, CommentEnd);
718 const char *TagNameEnd = skipHTMLIdentifier(TagNameBegin, CommentEnd);
719 StringRef Name(TagNameBegin, TagNameEnd - TagNameBegin);
720 if (!isHTMLTagName(Name)) {
721 formTextToken(T, TagNameEnd);
725 const char *End = skipWhitespace(TagNameEnd, CommentEnd);
727 formTokenWithChars(T, End, tok::html_end_tag);
728 T.setHTMLTagEndName(Name);
730 if (BufferPtr != CommentEnd && *BufferPtr == '>')
731 State = LS_HTMLEndTag;
734 void Lexer::lexHTMLEndTag(Token &T) {
735 assert(BufferPtr != CommentEnd && *BufferPtr == '>');
737 formTokenWithChars(T, BufferPtr + 1, tok::html_greater);
741 Lexer::Lexer(llvm::BumpPtrAllocator &Allocator, DiagnosticsEngine &Diags,
742 const CommandTraits &Traits, SourceLocation FileLoc,
743 const char *BufferStart, const char *BufferEnd,
745 : Allocator(Allocator), Diags(Diags), Traits(Traits),
746 BufferStart(BufferStart), BufferEnd(BufferEnd), FileLoc(FileLoc),
747 BufferPtr(BufferStart), CommentState(LCS_BeforeComment), State(LS_Normal),
748 ParseCommands(ParseCommands) {}
750 void Lexer::lex(Token &T) {
752 switch (CommentState) {
753 case LCS_BeforeComment:
754 if (BufferPtr == BufferEnd) {
755 formTokenWithChars(T, BufferPtr, tok::eof);
759 assert(*BufferPtr == '/');
760 BufferPtr++; // Skip first slash.
762 case '/': { // BCPL comment.
763 BufferPtr++; // Skip second slash.
765 if (BufferPtr != BufferEnd) {
766 // Skip Doxygen magic marker, if it is present.
767 // It might be missing because of a typo //< or /*<, or because we
768 // merged this non-Doxygen comment into a bunch of Doxygen comments
769 // around it: /** ... */ /* ... */ /** ... */
770 const char C = *BufferPtr;
771 if (C == '/' || C == '!')
775 // Skip less-than symbol that marks trailing comments.
776 // Skip it even if the comment is not a Doxygen one, because //< and /*<
777 // are frequent typos.
778 if (BufferPtr != BufferEnd && *BufferPtr == '<')
781 CommentState = LCS_InsideBCPLComment;
782 if (State != LS_VerbatimBlockBody && State != LS_VerbatimBlockFirstLine)
784 CommentEnd = findBCPLCommentEnd(BufferPtr, BufferEnd);
787 case '*': { // C comment.
788 BufferPtr++; // Skip star.
790 // Skip Doxygen magic marker.
791 const char C = *BufferPtr;
792 if ((C == '*' && *(BufferPtr + 1) != '/') || C == '!')
795 // Skip less-than symbol that marks trailing comments.
796 if (BufferPtr != BufferEnd && *BufferPtr == '<')
799 CommentState = LCS_InsideCComment;
801 CommentEnd = findCCommentEnd(BufferPtr, BufferEnd);
805 llvm_unreachable("second character of comment should be '/' or '*'");
808 case LCS_BetweenComments: {
809 // Consecutive comments are extracted only if there is only whitespace
810 // between them. So we can search for the start of the next comment.
811 const char *EndWhitespace = BufferPtr;
812 while(EndWhitespace != BufferEnd && *EndWhitespace != '/')
815 // Turn any whitespace between comments (and there is only whitespace
816 // between them -- guaranteed by comment extraction) into a newline. We
817 // have two newlines between C comments in total (first one was synthesized
819 formTokenWithChars(T, EndWhitespace, tok::newline);
821 CommentState = LCS_BeforeComment;
825 case LCS_InsideBCPLComment:
826 case LCS_InsideCComment:
827 if (BufferPtr != CommentEnd) {
831 // Skip C comment closing sequence.
832 if (CommentState == LCS_InsideCComment) {
833 assert(BufferPtr[0] == '*' && BufferPtr[1] == '/');
835 assert(BufferPtr <= BufferEnd);
837 // Synthenize newline just after the C comment, regardless if there is
838 // actually a newline.
839 formTokenWithChars(T, BufferPtr, tok::newline);
841 CommentState = LCS_BetweenComments;
844 // Don't synthesized a newline after BCPL comment.
845 CommentState = LCS_BetweenComments;
852 StringRef Lexer::getSpelling(const Token &Tok,
853 const SourceManager &SourceMgr) const {
854 SourceLocation Loc = Tok.getLocation();
855 std::pair<FileID, unsigned> LocInfo = SourceMgr.getDecomposedLoc(Loc);
857 bool InvalidTemp = false;
858 StringRef File = SourceMgr.getBufferData(LocInfo.first, &InvalidTemp);
862 const char *Begin = File.data() + LocInfo.second;
863 return StringRef(Begin, Tok.getLength());
866 } // end namespace comments
867 } // end namespace clang