1 #include "clang/AST/CommentLexer.h"
2 #include "clang/AST/CommentCommandTraits.h"
3 #include "clang/AST/CommentDiagnostic.h"
4 #include "clang/Basic/CharInfo.h"
5 #include "llvm/ADT/StringExtras.h"
6 #include "llvm/ADT/StringSwitch.h"
7 #include "llvm/Support/ConvertUTF.h"
8 #include "llvm/Support/ErrorHandling.h"
13 void Token::dump(const Lexer &L, const SourceManager &SM) const {
14 llvm::errs() << "comments::Token Kind=" << Kind << " ";
16 llvm::errs() << " " << Length << " \"" << L.getSpelling(*this, SM) << "\"\n";
19 static inline bool isHTMLNamedCharacterReferenceCharacter(char C) {
23 static inline bool isHTMLDecimalCharacterReferenceCharacter(char C) {
27 static inline bool isHTMLHexCharacterReferenceCharacter(char C) {
31 static inline StringRef convertCodePointToUTF8(
32 llvm::BumpPtrAllocator &Allocator,
34 char *Resolved = Allocator.Allocate<char>(UNI_MAX_UTF8_BYTES_PER_CODE_POINT);
35 char *ResolvedPtr = Resolved;
36 if (llvm::ConvertCodePointToUTF8(CodePoint, ResolvedPtr))
37 return StringRef(Resolved, ResolvedPtr - Resolved);
44 #include "clang/AST/CommentHTMLTags.inc"
45 #include "clang/AST/CommentHTMLNamedCharacterReferences.inc"
47 } // unnamed namespace
49 StringRef Lexer::resolveHTMLNamedCharacterReference(StringRef Name) const {
50 // Fast path, first check a few most widely used named character references.
51 return llvm::StringSwitch<StringRef>(Name)
58 .Default(translateHTMLNamedCharacterReferenceToUTF8(Name));
61 StringRef Lexer::resolveHTMLDecimalCharacterReference(StringRef Name) const {
62 unsigned CodePoint = 0;
63 for (unsigned i = 0, e = Name.size(); i != e; ++i) {
64 assert(isHTMLDecimalCharacterReferenceCharacter(Name[i]));
66 CodePoint += Name[i] - '0';
68 return convertCodePointToUTF8(Allocator, CodePoint);
71 StringRef Lexer::resolveHTMLHexCharacterReference(StringRef Name) const {
72 unsigned CodePoint = 0;
73 for (unsigned i = 0, e = Name.size(); i != e; ++i) {
75 const char C = Name[i];
76 assert(isHTMLHexCharacterReferenceCharacter(C));
77 CodePoint += llvm::hexDigitValue(C);
79 return convertCodePointToUTF8(Allocator, CodePoint);
82 void Lexer::skipLineStartingDecorations() {
83 // This function should be called only for C comments
84 assert(CommentState == LCS_InsideCComment);
86 if (BufferPtr == CommentEnd)
94 const char *NewBufferPtr = BufferPtr;
96 if (NewBufferPtr == CommentEnd)
99 char C = *NewBufferPtr;
100 while (isHorizontalWhitespace(C)) {
102 if (NewBufferPtr == CommentEnd)
107 BufferPtr = NewBufferPtr + 1;
117 /// Returns pointer to the first newline character in the string.
118 const char *findNewline(const char *BufferPtr, const char *BufferEnd) {
119 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
120 if (isVerticalWhitespace(*BufferPtr))
126 const char *skipNewline(const char *BufferPtr, const char *BufferEnd) {
127 if (BufferPtr == BufferEnd)
130 if (*BufferPtr == '\n')
133 assert(*BufferPtr == '\r');
135 if (BufferPtr != BufferEnd && *BufferPtr == '\n')
141 const char *skipNamedCharacterReference(const char *BufferPtr,
142 const char *BufferEnd) {
143 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
144 if (!isHTMLNamedCharacterReferenceCharacter(*BufferPtr))
150 const char *skipDecimalCharacterReference(const char *BufferPtr,
151 const char *BufferEnd) {
152 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
153 if (!isHTMLDecimalCharacterReferenceCharacter(*BufferPtr))
159 const char *skipHexCharacterReference(const char *BufferPtr,
160 const char *BufferEnd) {
161 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
162 if (!isHTMLHexCharacterReferenceCharacter(*BufferPtr))
168 bool isHTMLIdentifierStartingCharacter(char C) {
172 bool isHTMLIdentifierCharacter(char C) {
173 return isAlphanumeric(C);
176 const char *skipHTMLIdentifier(const char *BufferPtr, const char *BufferEnd) {
177 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
178 if (!isHTMLIdentifierCharacter(*BufferPtr))
184 /// Skip HTML string quoted in single or double quotes. Escaping quotes inside
187 /// Returns pointer to closing quote.
188 const char *skipHTMLQuotedString(const char *BufferPtr, const char *BufferEnd)
190 const char Quote = *BufferPtr;
191 assert(Quote == '\"' || Quote == '\'');
194 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
195 const char C = *BufferPtr;
196 if (C == Quote && BufferPtr[-1] != '\\')
202 const char *skipWhitespace(const char *BufferPtr, const char *BufferEnd) {
203 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
204 if (!isWhitespace(*BufferPtr))
210 bool isWhitespace(const char *BufferPtr, const char *BufferEnd) {
211 return skipWhitespace(BufferPtr, BufferEnd) == BufferEnd;
214 bool isCommandNameStartCharacter(char C) {
218 bool isCommandNameCharacter(char C) {
219 return isAlphanumeric(C);
222 const char *skipCommandName(const char *BufferPtr, const char *BufferEnd) {
223 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
224 if (!isCommandNameCharacter(*BufferPtr))
230 /// Return the one past end pointer for BCPL comments.
231 /// Handles newlines escaped with backslash or trigraph for backslahs.
232 const char *findBCPLCommentEnd(const char *BufferPtr, const char *BufferEnd) {
233 const char *CurPtr = BufferPtr;
234 while (CurPtr != BufferEnd) {
235 while (!isVerticalWhitespace(*CurPtr)) {
237 if (CurPtr == BufferEnd)
240 // We found a newline, check if it is escaped.
241 const char *EscapePtr = CurPtr - 1;
242 while(isHorizontalWhitespace(*EscapePtr))
245 if (*EscapePtr == '\\' ||
246 (EscapePtr - 2 >= BufferPtr && EscapePtr[0] == '/' &&
247 EscapePtr[-1] == '?' && EscapePtr[-2] == '?')) {
248 // We found an escaped newline.
249 CurPtr = skipNewline(CurPtr, BufferEnd);
251 return CurPtr; // Not an escaped newline.
256 /// Return the one past end pointer for C comments.
257 /// Very dumb, does not handle escaped newlines or trigraphs.
258 const char *findCCommentEnd(const char *BufferPtr, const char *BufferEnd) {
259 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
260 if (*BufferPtr == '*') {
261 assert(BufferPtr + 1 != BufferEnd);
262 if (*(BufferPtr + 1) == '/')
266 llvm_unreachable("buffer end hit before '*/' was seen");
269 } // unnamed namespace
271 void Lexer::lexCommentText(Token &T) {
272 assert(CommentState == LCS_InsideBCPLComment ||
273 CommentState == LCS_InsideCComment);
278 case LS_VerbatimBlockFirstLine:
279 lexVerbatimBlockFirstLine(T);
281 case LS_VerbatimBlockBody:
282 lexVerbatimBlockBody(T);
284 case LS_VerbatimLineText:
285 lexVerbatimLineText(T);
287 case LS_HTMLStartTag:
295 assert(State == LS_Normal);
297 const char *TokenPtr = BufferPtr;
298 assert(TokenPtr < CommentEnd);
299 while (TokenPtr != CommentEnd) {
303 // Commands that start with a backslash and commands that start with
304 // 'at' have equivalent semantics. But we keep information about the
305 // exact syntax in AST for comments.
306 tok::TokenKind CommandKind =
307 (*TokenPtr == '@') ? tok::at_command : tok::backslash_command;
309 if (TokenPtr == CommentEnd) {
310 formTextToken(T, TokenPtr);
318 case '\\': case '@': case '&': case '$':
319 case '#': case '<': case '>': case '%':
320 case '\"': case '.': case ':':
321 // This is one of \\ \@ \& \$ etc escape sequences.
323 if (C == ':' && TokenPtr != CommentEnd && *TokenPtr == ':') {
324 // This is the \:: escape sequence.
327 StringRef UnescapedText(BufferPtr + 1, TokenPtr - (BufferPtr + 1));
328 formTokenWithChars(T, TokenPtr, tok::text);
329 T.setText(UnescapedText);
333 // Don't make zero-length commands.
334 if (!isCommandNameStartCharacter(*TokenPtr)) {
335 formTextToken(T, TokenPtr);
339 TokenPtr = skipCommandName(TokenPtr, CommentEnd);
340 unsigned Length = TokenPtr - (BufferPtr + 1);
342 // Hardcoded support for lexing LaTeX formula commands
343 // \f$ \f[ \f] \f{ \f} as a single command.
344 if (Length == 1 && TokenPtr[-1] == 'f' && TokenPtr != CommentEnd) {
346 if (C == '$' || C == '[' || C == ']' || C == '{' || C == '}') {
352 const StringRef CommandName(BufferPtr + 1, Length);
354 const CommandInfo *Info = Traits.getCommandInfoOrNULL(CommandName);
356 if ((Info = Traits.getTypoCorrectCommandInfo(CommandName))) {
357 StringRef CorrectedName = Info->Name;
358 SourceLocation Loc = getSourceLocation(BufferPtr);
359 SourceRange CommandRange(Loc.getLocWithOffset(1),
360 getSourceLocation(TokenPtr));
361 Diag(Loc, diag::warn_correct_comment_command_name)
362 << CommandName << CorrectedName
363 << FixItHint::CreateReplacement(CommandRange, CorrectedName);
365 formTokenWithChars(T, TokenPtr, tok::unknown_command);
366 T.setUnknownCommandName(CommandName);
367 Diag(T.getLocation(), diag::warn_unknown_comment_command_name);
371 if (Info->IsVerbatimBlockCommand) {
372 setupAndLexVerbatimBlock(T, TokenPtr, *BufferPtr, Info);
375 if (Info->IsVerbatimLineCommand) {
376 setupAndLexVerbatimLine(T, TokenPtr, Info);
379 formTokenWithChars(T, TokenPtr, CommandKind);
380 T.setCommandID(Info->getID());
385 lexHTMLCharacterReference(T);
390 if (TokenPtr == CommentEnd) {
391 formTextToken(T, TokenPtr);
394 const char C = *TokenPtr;
395 if (isHTMLIdentifierStartingCharacter(C))
396 setupAndLexHTMLStartTag(T);
398 setupAndLexHTMLEndTag(T);
400 formTextToken(T, TokenPtr);
407 TokenPtr = skipNewline(TokenPtr, CommentEnd);
408 formTokenWithChars(T, TokenPtr, tok::newline);
410 if (CommentState == LCS_InsideCComment)
411 skipLineStartingDecorations();
415 size_t End = StringRef(TokenPtr, CommentEnd - TokenPtr).
416 find_first_of("\n\r\\@&<");
417 if (End != StringRef::npos)
420 TokenPtr = CommentEnd;
421 formTextToken(T, TokenPtr);
428 void Lexer::setupAndLexVerbatimBlock(Token &T,
429 const char *TextBegin,
430 char Marker, const CommandInfo *Info) {
431 assert(Info->IsVerbatimBlockCommand);
433 VerbatimBlockEndCommandName.clear();
434 VerbatimBlockEndCommandName.append(Marker == '\\' ? "\\" : "@");
435 VerbatimBlockEndCommandName.append(Info->EndCommandName);
437 formTokenWithChars(T, TextBegin, tok::verbatim_block_begin);
438 T.setVerbatimBlockID(Info->getID());
440 // If there is a newline following the verbatim opening command, skip the
441 // newline so that we don't create an tok::verbatim_block_line with empty
443 if (BufferPtr != CommentEnd &&
444 isVerticalWhitespace(*BufferPtr)) {
445 BufferPtr = skipNewline(BufferPtr, CommentEnd);
446 State = LS_VerbatimBlockBody;
450 State = LS_VerbatimBlockFirstLine;
453 void Lexer::lexVerbatimBlockFirstLine(Token &T) {
455 assert(BufferPtr < CommentEnd);
457 // FIXME: It would be better to scan the text once, finding either the block
458 // end command or newline.
460 // Extract current line.
461 const char *Newline = findNewline(BufferPtr, CommentEnd);
462 StringRef Line(BufferPtr, Newline - BufferPtr);
464 // Look for end command in current line.
465 size_t Pos = Line.find(VerbatimBlockEndCommandName);
467 const char *NextLine;
468 if (Pos == StringRef::npos) {
469 // Current line is completely verbatim.
471 NextLine = skipNewline(Newline, CommentEnd);
472 } else if (Pos == 0) {
473 // Current line contains just an end command.
474 const char *End = BufferPtr + VerbatimBlockEndCommandName.size();
475 StringRef Name(BufferPtr + 1, End - (BufferPtr + 1));
476 formTokenWithChars(T, End, tok::verbatim_block_end);
477 T.setVerbatimBlockID(Traits.getCommandInfo(Name)->getID());
481 // There is some text, followed by end command. Extract text first.
482 TextEnd = BufferPtr + Pos;
484 // If there is only whitespace before end command, skip whitespace.
485 if (isWhitespace(BufferPtr, TextEnd)) {
491 StringRef Text(BufferPtr, TextEnd - BufferPtr);
492 formTokenWithChars(T, NextLine, tok::verbatim_block_line);
493 T.setVerbatimBlockText(Text);
495 State = LS_VerbatimBlockBody;
498 void Lexer::lexVerbatimBlockBody(Token &T) {
499 assert(State == LS_VerbatimBlockBody);
501 if (CommentState == LCS_InsideCComment)
502 skipLineStartingDecorations();
504 lexVerbatimBlockFirstLine(T);
507 void Lexer::setupAndLexVerbatimLine(Token &T, const char *TextBegin,
508 const CommandInfo *Info) {
509 assert(Info->IsVerbatimLineCommand);
510 formTokenWithChars(T, TextBegin, tok::verbatim_line_name);
511 T.setVerbatimLineID(Info->getID());
513 State = LS_VerbatimLineText;
516 void Lexer::lexVerbatimLineText(Token &T) {
517 assert(State == LS_VerbatimLineText);
519 // Extract current line.
520 const char *Newline = findNewline(BufferPtr, CommentEnd);
521 const StringRef Text(BufferPtr, Newline - BufferPtr);
522 formTokenWithChars(T, Newline, tok::verbatim_line_text);
523 T.setVerbatimLineText(Text);
528 void Lexer::lexHTMLCharacterReference(Token &T) {
529 const char *TokenPtr = BufferPtr;
530 assert(*TokenPtr == '&');
532 if (TokenPtr == CommentEnd) {
533 formTextToken(T, TokenPtr);
537 bool isNamed = false;
538 bool isDecimal = false;
540 if (isHTMLNamedCharacterReferenceCharacter(C)) {
542 TokenPtr = skipNamedCharacterReference(TokenPtr, CommentEnd);
544 } else if (C == '#') {
546 if (TokenPtr == CommentEnd) {
547 formTextToken(T, TokenPtr);
551 if (isHTMLDecimalCharacterReferenceCharacter(C)) {
553 TokenPtr = skipDecimalCharacterReference(TokenPtr, CommentEnd);
555 } else if (C == 'x' || C == 'X') {
558 TokenPtr = skipHexCharacterReference(TokenPtr, CommentEnd);
560 formTextToken(T, TokenPtr);
564 formTextToken(T, TokenPtr);
567 if (NamePtr == TokenPtr || TokenPtr == CommentEnd ||
569 formTextToken(T, TokenPtr);
572 StringRef Name(NamePtr, TokenPtr - NamePtr);
573 TokenPtr++; // Skip semicolon.
576 Resolved = resolveHTMLNamedCharacterReference(Name);
578 Resolved = resolveHTMLDecimalCharacterReference(Name);
580 Resolved = resolveHTMLHexCharacterReference(Name);
582 if (Resolved.empty()) {
583 formTextToken(T, TokenPtr);
586 formTokenWithChars(T, TokenPtr, tok::text);
591 void Lexer::setupAndLexHTMLStartTag(Token &T) {
592 assert(BufferPtr[0] == '<' &&
593 isHTMLIdentifierStartingCharacter(BufferPtr[1]));
594 const char *TagNameEnd = skipHTMLIdentifier(BufferPtr + 2, CommentEnd);
595 StringRef Name(BufferPtr + 1, TagNameEnd - (BufferPtr + 1));
596 if (!isHTMLTagName(Name)) {
597 formTextToken(T, TagNameEnd);
601 formTokenWithChars(T, TagNameEnd, tok::html_start_tag);
602 T.setHTMLTagStartName(Name);
604 BufferPtr = skipWhitespace(BufferPtr, CommentEnd);
606 const char C = *BufferPtr;
607 if (BufferPtr != CommentEnd &&
608 (C == '>' || C == '/' || isHTMLIdentifierStartingCharacter(C)))
609 State = LS_HTMLStartTag;
612 void Lexer::lexHTMLStartTag(Token &T) {
613 assert(State == LS_HTMLStartTag);
615 const char *TokenPtr = BufferPtr;
617 if (isHTMLIdentifierCharacter(C)) {
618 TokenPtr = skipHTMLIdentifier(TokenPtr, CommentEnd);
619 StringRef Ident(BufferPtr, TokenPtr - BufferPtr);
620 formTokenWithChars(T, TokenPtr, tok::html_ident);
621 T.setHTMLIdent(Ident);
626 formTokenWithChars(T, TokenPtr, tok::html_equals);
630 const char *OpenQuote = TokenPtr;
631 TokenPtr = skipHTMLQuotedString(TokenPtr, CommentEnd);
632 const char *ClosingQuote = TokenPtr;
633 if (TokenPtr != CommentEnd) // Skip closing quote.
635 formTokenWithChars(T, TokenPtr, tok::html_quoted_string);
636 T.setHTMLQuotedString(StringRef(OpenQuote + 1,
637 ClosingQuote - (OpenQuote + 1)));
642 formTokenWithChars(T, TokenPtr, tok::html_greater);
647 if (TokenPtr != CommentEnd && *TokenPtr == '>') {
649 formTokenWithChars(T, TokenPtr, tok::html_slash_greater);
651 formTextToken(T, TokenPtr);
658 // Now look ahead and return to normal state if we don't see any HTML tokens
660 BufferPtr = skipWhitespace(BufferPtr, CommentEnd);
661 if (BufferPtr == CommentEnd) {
667 if (!isHTMLIdentifierStartingCharacter(C) &&
668 C != '=' && C != '\"' && C != '\'' && C != '>') {
674 void Lexer::setupAndLexHTMLEndTag(Token &T) {
675 assert(BufferPtr[0] == '<' && BufferPtr[1] == '/');
677 const char *TagNameBegin = skipWhitespace(BufferPtr + 2, CommentEnd);
678 const char *TagNameEnd = skipHTMLIdentifier(TagNameBegin, CommentEnd);
679 StringRef Name(TagNameBegin, TagNameEnd - TagNameBegin);
680 if (!isHTMLTagName(Name)) {
681 formTextToken(T, TagNameEnd);
685 const char *End = skipWhitespace(TagNameEnd, CommentEnd);
687 formTokenWithChars(T, End, tok::html_end_tag);
688 T.setHTMLTagEndName(Name);
690 if (BufferPtr != CommentEnd && *BufferPtr == '>')
691 State = LS_HTMLEndTag;
694 void Lexer::lexHTMLEndTag(Token &T) {
695 assert(BufferPtr != CommentEnd && *BufferPtr == '>');
697 formTokenWithChars(T, BufferPtr + 1, tok::html_greater);
701 Lexer::Lexer(llvm::BumpPtrAllocator &Allocator, DiagnosticsEngine &Diags,
702 const CommandTraits &Traits,
703 SourceLocation FileLoc,
704 const char *BufferStart, const char *BufferEnd):
705 Allocator(Allocator), Diags(Diags), Traits(Traits),
706 BufferStart(BufferStart), BufferEnd(BufferEnd),
707 FileLoc(FileLoc), BufferPtr(BufferStart),
708 CommentState(LCS_BeforeComment), State(LS_Normal) {
711 void Lexer::lex(Token &T) {
713 switch (CommentState) {
714 case LCS_BeforeComment:
715 if (BufferPtr == BufferEnd) {
716 formTokenWithChars(T, BufferPtr, tok::eof);
720 assert(*BufferPtr == '/');
721 BufferPtr++; // Skip first slash.
723 case '/': { // BCPL comment.
724 BufferPtr++; // Skip second slash.
726 if (BufferPtr != BufferEnd) {
727 // Skip Doxygen magic marker, if it is present.
728 // It might be missing because of a typo //< or /*<, or because we
729 // merged this non-Doxygen comment into a bunch of Doxygen comments
730 // around it: /** ... */ /* ... */ /** ... */
731 const char C = *BufferPtr;
732 if (C == '/' || C == '!')
736 // Skip less-than symbol that marks trailing comments.
737 // Skip it even if the comment is not a Doxygen one, because //< and /*<
738 // are frequent typos.
739 if (BufferPtr != BufferEnd && *BufferPtr == '<')
742 CommentState = LCS_InsideBCPLComment;
743 if (State != LS_VerbatimBlockBody && State != LS_VerbatimBlockFirstLine)
745 CommentEnd = findBCPLCommentEnd(BufferPtr, BufferEnd);
748 case '*': { // C comment.
749 BufferPtr++; // Skip star.
751 // Skip Doxygen magic marker.
752 const char C = *BufferPtr;
753 if ((C == '*' && *(BufferPtr + 1) != '/') || C == '!')
756 // Skip less-than symbol that marks trailing comments.
757 if (BufferPtr != BufferEnd && *BufferPtr == '<')
760 CommentState = LCS_InsideCComment;
762 CommentEnd = findCCommentEnd(BufferPtr, BufferEnd);
766 llvm_unreachable("second character of comment should be '/' or '*'");
769 case LCS_BetweenComments: {
770 // Consecutive comments are extracted only if there is only whitespace
771 // between them. So we can search for the start of the next comment.
772 const char *EndWhitespace = BufferPtr;
773 while(EndWhitespace != BufferEnd && *EndWhitespace != '/')
776 // Turn any whitespace between comments (and there is only whitespace
777 // between them -- guaranteed by comment extraction) into a newline. We
778 // have two newlines between C comments in total (first one was synthesized
780 formTokenWithChars(T, EndWhitespace, tok::newline);
782 CommentState = LCS_BeforeComment;
786 case LCS_InsideBCPLComment:
787 case LCS_InsideCComment:
788 if (BufferPtr != CommentEnd) {
792 // Skip C comment closing sequence.
793 if (CommentState == LCS_InsideCComment) {
794 assert(BufferPtr[0] == '*' && BufferPtr[1] == '/');
796 assert(BufferPtr <= BufferEnd);
798 // Synthenize newline just after the C comment, regardless if there is
799 // actually a newline.
800 formTokenWithChars(T, BufferPtr, tok::newline);
802 CommentState = LCS_BetweenComments;
805 // Don't synthesized a newline after BCPL comment.
806 CommentState = LCS_BetweenComments;
813 StringRef Lexer::getSpelling(const Token &Tok,
814 const SourceManager &SourceMgr,
815 bool *Invalid) const {
816 SourceLocation Loc = Tok.getLocation();
817 std::pair<FileID, unsigned> LocInfo = SourceMgr.getDecomposedLoc(Loc);
819 bool InvalidTemp = false;
820 StringRef File = SourceMgr.getBufferData(LocInfo.first, &InvalidTemp);
826 const char *Begin = File.data() + LocInfo.second;
827 return StringRef(Begin, Tok.getLength());
830 } // end namespace comments
831 } // end namespace clang