1 //===--- RawCommentList.cpp - Processing raw comments -----------*- C++ -*-===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 #include "clang/AST/RawCommentList.h"
11 #include "clang/AST/ASTContext.h"
12 #include "clang/AST/Comment.h"
13 #include "clang/AST/CommentBriefParser.h"
14 #include "clang/AST/CommentCommandTraits.h"
15 #include "clang/AST/CommentLexer.h"
16 #include "clang/AST/CommentParser.h"
17 #include "clang/AST/CommentSema.h"
18 #include "clang/Basic/CharInfo.h"
19 #include "llvm/ADT/STLExtras.h"
21 using namespace clang;
24 /// Get comment kind and bool describing if it is a trailing comment.
25 std::pair<RawComment::CommentKind, bool> getCommentKind(StringRef Comment,
26 bool ParseAllComments) {
27 const size_t MinCommentLength = ParseAllComments ? 2 : 3;
28 if ((Comment.size() < MinCommentLength) || Comment[0] != '/')
29 return std::make_pair(RawComment::RCK_Invalid, false);
31 RawComment::CommentKind K;
32 if (Comment[1] == '/') {
33 if (Comment.size() < 3)
34 return std::make_pair(RawComment::RCK_OrdinaryBCPL, false);
36 if (Comment[2] == '/')
37 K = RawComment::RCK_BCPLSlash;
38 else if (Comment[2] == '!')
39 K = RawComment::RCK_BCPLExcl;
41 return std::make_pair(RawComment::RCK_OrdinaryBCPL, false);
43 assert(Comment.size() >= 4);
45 // Comment lexer does not understand escapes in comment markers, so pretend
46 // that this is not a comment.
47 if (Comment[1] != '*' ||
48 Comment[Comment.size() - 2] != '*' ||
49 Comment[Comment.size() - 1] != '/')
50 return std::make_pair(RawComment::RCK_Invalid, false);
52 if (Comment[2] == '*')
53 K = RawComment::RCK_JavaDoc;
54 else if (Comment[2] == '!')
55 K = RawComment::RCK_Qt;
57 return std::make_pair(RawComment::RCK_OrdinaryC, false);
59 const bool TrailingComment = (Comment.size() > 3) && (Comment[3] == '<');
60 return std::make_pair(K, TrailingComment);
63 bool mergedCommentIsTrailingComment(StringRef Comment) {
64 return (Comment.size() > 3) && (Comment[3] == '<');
67 /// Returns true if R1 and R2 both have valid locations that start on the same
69 bool commentsStartOnSameColumn(const SourceManager &SM, const RawComment &R1,
70 const RawComment &R2) {
71 SourceLocation L1 = R1.getBeginLoc();
72 SourceLocation L2 = R2.getBeginLoc();
74 unsigned C1 = SM.getPresumedColumnNumber(L1, &Invalid);
76 unsigned C2 = SM.getPresumedColumnNumber(L2, &Invalid);
77 return !Invalid && (C1 == C2);
81 } // unnamed namespace
83 /// Determines whether there is only whitespace in `Buffer` between `P`
84 /// and the previous line.
85 /// \param Buffer The buffer to search in.
86 /// \param P The offset from the beginning of `Buffer` to start from.
87 /// \return true if all of the characters in `Buffer` ranging from the closest
88 /// line-ending character before `P` (or the beginning of `Buffer`) to `P - 1`
90 static bool onlyWhitespaceOnLineBefore(const char *Buffer, unsigned P) {
91 // Search backwards until we see linefeed or carriage return.
92 for (unsigned I = P; I != 0; --I) {
93 char C = Buffer[I - 1];
94 if (isVerticalWhitespace(C))
96 if (!isHorizontalWhitespace(C))
99 // We hit the beginning of the buffer.
103 /// Returns whether `K` is an ordinary comment kind.
104 static bool isOrdinaryKind(RawComment::CommentKind K) {
105 return (K == RawComment::RCK_OrdinaryBCPL) ||
106 (K == RawComment::RCK_OrdinaryC);
109 RawComment::RawComment(const SourceManager &SourceMgr, SourceRange SR,
110 const CommentOptions &CommentOpts, bool Merged) :
111 Range(SR), RawTextValid(false), BriefTextValid(false),
112 IsAttached(false), IsTrailingComment(false),
113 IsAlmostTrailingComment(false) {
114 // Extract raw comment text, if possible.
115 if (SR.getBegin() == SR.getEnd() || getRawText(SourceMgr).empty()) {
120 // Guess comment kind.
121 std::pair<CommentKind, bool> K =
122 getCommentKind(RawText, CommentOpts.ParseAllComments);
124 // Guess whether an ordinary comment is trailing.
125 if (CommentOpts.ParseAllComments && isOrdinaryKind(K.first)) {
127 unsigned BeginOffset;
128 std::tie(BeginFileID, BeginOffset) =
129 SourceMgr.getDecomposedLoc(Range.getBegin());
130 if (BeginOffset != 0) {
131 bool Invalid = false;
133 SourceMgr.getBufferData(BeginFileID, &Invalid).data();
135 (!Invalid && !onlyWhitespaceOnLineBefore(Buffer, BeginOffset));
141 IsTrailingComment |= K.second;
143 IsAlmostTrailingComment = RawText.startswith("//<") ||
144 RawText.startswith("/*<");
148 IsTrailingComment || mergedCommentIsTrailingComment(RawText);
152 StringRef RawComment::getRawTextSlow(const SourceManager &SourceMgr) const {
155 unsigned BeginOffset;
158 std::tie(BeginFileID, BeginOffset) =
159 SourceMgr.getDecomposedLoc(Range.getBegin());
160 std::tie(EndFileID, EndOffset) = SourceMgr.getDecomposedLoc(Range.getEnd());
162 const unsigned Length = EndOffset - BeginOffset;
166 // The comment can't begin in one file and end in another.
167 assert(BeginFileID == EndFileID);
169 bool Invalid = false;
170 const char *BufferStart = SourceMgr.getBufferData(BeginFileID,
175 return StringRef(BufferStart + BeginOffset, Length);
178 const char *RawComment::extractBriefText(const ASTContext &Context) const {
179 // Lazily initialize RawText using the accessor before using it.
180 (void)getRawText(Context.getSourceManager());
182 // Since we will be copying the resulting text, all allocations made during
183 // parsing are garbage after resulting string is formed. Thus we can use
184 // a separate allocator for all temporary stuff.
185 llvm::BumpPtrAllocator Allocator;
187 comments::Lexer L(Allocator, Context.getDiagnostics(),
188 Context.getCommentCommandTraits(),
190 RawText.begin(), RawText.end());
191 comments::BriefParser P(L, Context.getCommentCommandTraits());
193 const std::string Result = P.Parse();
194 const unsigned BriefTextLength = Result.size();
195 char *BriefTextPtr = new (Context) char[BriefTextLength + 1];
196 memcpy(BriefTextPtr, Result.c_str(), BriefTextLength + 1);
197 BriefText = BriefTextPtr;
198 BriefTextValid = true;
203 comments::FullComment *RawComment::parse(const ASTContext &Context,
204 const Preprocessor *PP,
205 const Decl *D) const {
206 // Lazily initialize RawText using the accessor before using it.
207 (void)getRawText(Context.getSourceManager());
209 comments::Lexer L(Context.getAllocator(), Context.getDiagnostics(),
210 Context.getCommentCommandTraits(),
211 getSourceRange().getBegin(),
212 RawText.begin(), RawText.end());
213 comments::Sema S(Context.getAllocator(), Context.getSourceManager(),
214 Context.getDiagnostics(),
215 Context.getCommentCommandTraits(),
218 comments::Parser P(L, S, Context.getAllocator(), Context.getSourceManager(),
219 Context.getDiagnostics(),
220 Context.getCommentCommandTraits());
222 return P.parseFullComment();
225 static bool onlyWhitespaceBetween(SourceManager &SM,
226 SourceLocation Loc1, SourceLocation Loc2,
227 unsigned MaxNewlinesAllowed) {
228 std::pair<FileID, unsigned> Loc1Info = SM.getDecomposedLoc(Loc1);
229 std::pair<FileID, unsigned> Loc2Info = SM.getDecomposedLoc(Loc2);
231 // Question does not make sense if locations are in different files.
232 if (Loc1Info.first != Loc2Info.first)
235 bool Invalid = false;
236 const char *Buffer = SM.getBufferData(Loc1Info.first, &Invalid).data();
240 unsigned NumNewlines = 0;
241 assert(Loc1Info.second <= Loc2Info.second && "Loc1 after Loc2!");
242 // Look for non-whitespace characters and remember any newlines seen.
243 for (unsigned I = Loc1Info.second; I != Loc2Info.second; ++I) {
256 // Check if we have found more than the maximum allowed number of
258 if (NumNewlines > MaxNewlinesAllowed)
261 // Collapse \r\n and \n\r into a single newline.
262 if (I + 1 != Loc2Info.second &&
263 (Buffer[I + 1] == '\n' || Buffer[I + 1] == '\r') &&
264 Buffer[I] != Buffer[I + 1])
273 void RawCommentList::addComment(const RawComment &RC,
274 const CommentOptions &CommentOpts,
275 llvm::BumpPtrAllocator &Allocator) {
279 // Check if the comments are not in source order.
280 while (!Comments.empty() &&
281 !SourceMgr.isBeforeInTranslationUnit(Comments.back()->getBeginLoc(),
283 // If they are, just pop a few last comments that don't fit.
284 // This happens if an \#include directive contains comments.
288 // Ordinary comments are not interesting for us.
289 if (RC.isOrdinary() && !CommentOpts.ParseAllComments)
292 // If this is the first Doxygen comment, save it (because there isn't
293 // anything to merge it with).
294 if (Comments.empty()) {
295 Comments.push_back(new (Allocator) RawComment(RC));
299 const RawComment &C1 = *Comments.back();
300 const RawComment &C2 = RC;
302 // Merge comments only if there is only whitespace between them.
303 // Can't merge trailing and non-trailing comments unless the second is
304 // non-trailing ordinary in the same column, as in the case:
305 // int x; // documents x
308 // int x; // documents x
309 // int y; // documents y
311 // int x; // documents x
314 // Merge comments if they are on same or consecutive lines.
315 if ((C1.isTrailingComment() == C2.isTrailingComment() ||
316 (C1.isTrailingComment() && !C2.isTrailingComment() &&
317 isOrdinaryKind(C2.getKind()) &&
318 commentsStartOnSameColumn(SourceMgr, C1, C2))) &&
319 onlyWhitespaceBetween(SourceMgr, C1.getEndLoc(), C2.getBeginLoc(),
320 /*MaxNewlinesAllowed=*/1)) {
321 SourceRange MergedRange(C1.getBeginLoc(), C2.getEndLoc());
322 *Comments.back() = RawComment(SourceMgr, MergedRange, CommentOpts, true);
324 Comments.push_back(new (Allocator) RawComment(RC));
328 void RawCommentList::addDeserializedComments(ArrayRef<RawComment *> DeserializedComments) {
329 std::vector<RawComment *> MergedComments;
330 MergedComments.reserve(Comments.size() + DeserializedComments.size());
332 std::merge(Comments.begin(), Comments.end(),
333 DeserializedComments.begin(), DeserializedComments.end(),
334 std::back_inserter(MergedComments),
335 BeforeThanCompare<RawComment>(SourceMgr));
336 std::swap(Comments, MergedComments);
339 std::string RawComment::getFormattedText(const SourceManager &SourceMgr,
340 DiagnosticsEngine &Diags) const {
341 llvm::StringRef CommentText = getRawText(SourceMgr);
342 if (CommentText.empty())
345 llvm::BumpPtrAllocator Allocator;
346 // We do not parse any commands, so CommentOptions are ignored by
347 // comments::Lexer. Therefore, we just use default-constructed options.
348 CommentOptions DefOpts;
349 comments::CommandTraits EmptyTraits(Allocator, DefOpts);
350 comments::Lexer L(Allocator, Diags, EmptyTraits, getSourceRange().getBegin(),
351 CommentText.begin(), CommentText.end(),
352 /*ParseCommands=*/false);
355 // A column number of the first non-whitespace token in the comment text.
356 // We skip whitespace up to this column, but keep the whitespace after this
357 // column. IndentColumn is calculated when lexing the first line and reused
358 // for the rest of lines.
359 unsigned IndentColumn = 0;
361 // Processes one line of the comment and adds it to the result.
362 // Handles skipping the indent at the start of the line.
363 // Returns false when eof is reached and true otherwise.
364 auto LexLine = [&](bool IsFirstLine) -> bool {
366 // Lex the first token on the line. We handle it separately, because we to
367 // fix up its indentation.
369 if (Tok.is(comments::tok::eof))
371 if (Tok.is(comments::tok::newline)) {
375 llvm::StringRef TokText = L.getSpelling(Tok, SourceMgr);
376 bool LocInvalid = false;
378 SourceMgr.getSpellingColumnNumber(Tok.getLocation(), &LocInvalid);
379 assert(!LocInvalid && "getFormattedText for invalid location");
381 // Amount of leading whitespace in TokText.
382 size_t WhitespaceLen = TokText.find_first_not_of(" \t");
383 if (WhitespaceLen == StringRef::npos)
384 WhitespaceLen = TokText.size();
385 // Remember the amount of whitespace we skipped in the first line to remove
386 // indent up to that column in the following lines.
388 IndentColumn = TokColumn + WhitespaceLen;
390 // Amount of leading whitespace we actually want to skip.
391 // For the first line we skip all the whitespace.
392 // For the rest of the lines, we skip whitespace up to IndentColumn.
398 std::max<int>(static_cast<int>(IndentColumn) - TokColumn, 0));
399 llvm::StringRef Trimmed = TokText.drop_front(SkipLen);
401 // Lex all tokens in the rest of the line.
402 for (L.lex(Tok); Tok.isNot(comments::tok::eof); L.lex(Tok)) {
403 if (Tok.is(comments::tok::newline)) {
407 Result += L.getSpelling(Tok, SourceMgr);
409 // We've reached the end of file token.
413 auto DropTrailingNewLines = [](std::string &Str) {
414 while (Str.back() == '\n')
418 // Process first line separately to remember indent for the following lines.
419 if (!LexLine(/*IsFirstLine=*/true)) {
420 DropTrailingNewLines(Result);
423 // Process the rest of the lines.
424 while (LexLine(/*IsFirstLine=*/false))
426 DropTrailingNewLines(Result);