1 //===--- CommentLexer.h - Lexer for structured comments ---------*- C++ -*-===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file defines lexer for structured comments and supporting token class.
11 //===----------------------------------------------------------------------===//
13 #ifndef LLVM_CLANG_AST_COMMENTLEXER_H
14 #define LLVM_CLANG_AST_COMMENTLEXER_H
16 #include "clang/Basic/Diagnostic.h"
17 #include "clang/Basic/SourceManager.h"
18 #include "llvm/ADT/SmallString.h"
19 #include "llvm/ADT/StringRef.h"
20 #include "llvm/Support/Allocator.h"
21 #include "llvm/Support/raw_ostream.h"
27 class TextTokenRetokenizer;
36 unknown_command, // Command that does not have an ID.
37 backslash_command, // Command with an ID, that used backslash marker.
38 at_command, // Command with an ID, that used 'at' marker.
44 html_start_tag, // <tag
47 html_quoted_string, // "blah\"blah" or 'blah\'blah'
49 html_slash_greater, // />
52 } // end namespace tok
57 friend class TextTokenRetokenizer;
59 /// The location of the token.
62 /// The actual kind of the token.
65 /// Integer value associated with a token.
67 /// If the token is a known command, contains command ID and TextPtr is
68 /// unused (command spelling can be found with CommandTraits). Otherwise,
69 /// contains the length of the string that starts at TextPtr.
72 /// Length of the token spelling in comment. Can be 0 for synthenized
76 /// Contains text value associated with a token.
80 SourceLocation getLocation() const LLVM_READONLY { return Loc; }
81 void setLocation(SourceLocation SL) { Loc = SL; }
83 SourceLocation getEndLocation() const LLVM_READONLY {
84 if (Length == 0 || Length == 1)
86 return Loc.getLocWithOffset(Length - 1);
89 tok::TokenKind getKind() const LLVM_READONLY { return Kind; }
90 void setKind(tok::TokenKind K) { Kind = K; }
92 bool is(tok::TokenKind K) const LLVM_READONLY { return Kind == K; }
93 bool isNot(tok::TokenKind K) const LLVM_READONLY { return Kind != K; }
95 unsigned getLength() const LLVM_READONLY { return Length; }
96 void setLength(unsigned L) { Length = L; }
98 StringRef getText() const LLVM_READONLY {
99 assert(is(tok::text));
100 return StringRef(TextPtr, IntVal);
103 void setText(StringRef Text) {
104 assert(is(tok::text));
105 TextPtr = Text.data();
106 IntVal = Text.size();
109 StringRef getUnknownCommandName() const LLVM_READONLY {
110 assert(is(tok::unknown_command));
111 return StringRef(TextPtr, IntVal);
114 void setUnknownCommandName(StringRef Name) {
115 assert(is(tok::unknown_command));
116 TextPtr = Name.data();
117 IntVal = Name.size();
120 unsigned getCommandID() const LLVM_READONLY {
121 assert(is(tok::backslash_command) || is(tok::at_command));
125 void setCommandID(unsigned ID) {
126 assert(is(tok::backslash_command) || is(tok::at_command));
130 unsigned getVerbatimBlockID() const LLVM_READONLY {
131 assert(is(tok::verbatim_block_begin) || is(tok::verbatim_block_end));
135 void setVerbatimBlockID(unsigned ID) {
136 assert(is(tok::verbatim_block_begin) || is(tok::verbatim_block_end));
140 StringRef getVerbatimBlockText() const LLVM_READONLY {
141 assert(is(tok::verbatim_block_line));
142 return StringRef(TextPtr, IntVal);
145 void setVerbatimBlockText(StringRef Text) {
146 assert(is(tok::verbatim_block_line));
147 TextPtr = Text.data();
148 IntVal = Text.size();
151 unsigned getVerbatimLineID() const LLVM_READONLY {
152 assert(is(tok::verbatim_line_name));
156 void setVerbatimLineID(unsigned ID) {
157 assert(is(tok::verbatim_line_name));
161 StringRef getVerbatimLineText() const LLVM_READONLY {
162 assert(is(tok::verbatim_line_text));
163 return StringRef(TextPtr, IntVal);
166 void setVerbatimLineText(StringRef Text) {
167 assert(is(tok::verbatim_line_text));
168 TextPtr = Text.data();
169 IntVal = Text.size();
172 StringRef getHTMLTagStartName() const LLVM_READONLY {
173 assert(is(tok::html_start_tag));
174 return StringRef(TextPtr, IntVal);
177 void setHTMLTagStartName(StringRef Name) {
178 assert(is(tok::html_start_tag));
179 TextPtr = Name.data();
180 IntVal = Name.size();
183 StringRef getHTMLIdent() const LLVM_READONLY {
184 assert(is(tok::html_ident));
185 return StringRef(TextPtr, IntVal);
188 void setHTMLIdent(StringRef Name) {
189 assert(is(tok::html_ident));
190 TextPtr = Name.data();
191 IntVal = Name.size();
194 StringRef getHTMLQuotedString() const LLVM_READONLY {
195 assert(is(tok::html_quoted_string));
196 return StringRef(TextPtr, IntVal);
199 void setHTMLQuotedString(StringRef Str) {
200 assert(is(tok::html_quoted_string));
201 TextPtr = Str.data();
205 StringRef getHTMLTagEndName() const LLVM_READONLY {
206 assert(is(tok::html_end_tag));
207 return StringRef(TextPtr, IntVal);
210 void setHTMLTagEndName(StringRef Name) {
211 assert(is(tok::html_end_tag));
212 TextPtr = Name.data();
213 IntVal = Name.size();
216 void dump(const Lexer &L, const SourceManager &SM) const;
222 Lexer(const Lexer &) = delete;
223 void operator=(const Lexer &) = delete;
225 /// Allocator for strings that are semantic values of tokens and have to be
226 /// computed (for example, resolved decimal character references).
227 llvm::BumpPtrAllocator &Allocator;
229 DiagnosticsEngine &Diags;
231 const CommandTraits &Traits;
233 const char *const BufferStart;
234 const char *const BufferEnd;
236 const char *BufferPtr;
238 /// One past end pointer for the current comment. For BCPL comments points
239 /// to newline or BufferEnd, for C comments points to star in '*/'.
240 const char *CommentEnd;
242 SourceLocation FileLoc;
244 /// If true, the commands, html tags, etc will be parsed and reported as
245 /// separate tokens inside the comment body. If false, the comment text will
246 /// be parsed into text and newline tokens.
249 enum LexerCommentState : uint8_t {
251 LCS_InsideBCPLComment,
256 /// Low-level lexer state, track if we are inside or outside of comment.
257 LexerCommentState CommentState;
259 enum LexerState : uint8_t {
260 /// Lexing normal comment text
263 /// Finished lexing verbatim block beginning command, will lex first body
265 LS_VerbatimBlockFirstLine,
267 /// Lexing verbatim block body line-by-line, skipping line-starting
269 LS_VerbatimBlockBody,
271 /// Finished lexing verbatim line beginning command, will lex text (one
275 /// Finished lexing \verbatim <TAG \endverbatim part, lexing tag attributes.
278 /// Finished lexing \verbatim </TAG \endverbatim part, lexing '>'.
282 /// Current lexing mode.
285 /// If State is LS_VerbatimBlock, contains the name of verbatim end
286 /// command, including command marker.
287 SmallString<16> VerbatimBlockEndCommandName;
289 /// Given a character reference name (e.g., "lt"), return the character that
290 /// it stands for (e.g., "<").
291 StringRef resolveHTMLNamedCharacterReference(StringRef Name) const;
293 /// Given a Unicode codepoint as base-10 integer, return the character.
294 StringRef resolveHTMLDecimalCharacterReference(StringRef Name) const;
296 /// Given a Unicode codepoint as base-16 integer, return the character.
297 StringRef resolveHTMLHexCharacterReference(StringRef Name) const;
299 void formTokenWithChars(Token &Result, const char *TokEnd,
300 tok::TokenKind Kind);
302 void formTextToken(Token &Result, const char *TokEnd) {
303 StringRef Text(BufferPtr, TokEnd - BufferPtr);
304 formTokenWithChars(Result, TokEnd, tok::text);
305 Result.setText(Text);
308 SourceLocation getSourceLocation(const char *Loc) const {
309 assert(Loc >= BufferStart && Loc <= BufferEnd &&
310 "Location out of range for this buffer!");
312 const unsigned CharNo = Loc - BufferStart;
313 return FileLoc.getLocWithOffset(CharNo);
316 DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) {
317 return Diags.Report(Loc, DiagID);
320 /// Eat string matching regexp \code \s*\* \endcode.
321 void skipLineStartingDecorations();
323 /// Skip over pure text.
324 const char *skipTextToken();
326 /// Lex comment text, including commands if ParseCommands is set to true.
327 void lexCommentText(Token &T);
329 void setupAndLexVerbatimBlock(Token &T, const char *TextBegin, char Marker,
330 const CommandInfo *Info);
332 void lexVerbatimBlockFirstLine(Token &T);
334 void lexVerbatimBlockBody(Token &T);
336 void setupAndLexVerbatimLine(Token &T, const char *TextBegin,
337 const CommandInfo *Info);
339 void lexVerbatimLineText(Token &T);
341 void lexHTMLCharacterReference(Token &T);
343 void setupAndLexHTMLStartTag(Token &T);
345 void lexHTMLStartTag(Token &T);
347 void setupAndLexHTMLEndTag(Token &T);
349 void lexHTMLEndTag(Token &T);
352 Lexer(llvm::BumpPtrAllocator &Allocator, DiagnosticsEngine &Diags,
353 const CommandTraits &Traits, SourceLocation FileLoc,
354 const char *BufferStart, const char *BufferEnd,
355 bool ParseCommands = true);
359 StringRef getSpelling(const Token &Tok, const SourceManager &SourceMgr) const;
362 } // end namespace comments
363 } // end namespace clang