1 //===--- UnwrappedLineParser.cpp - Format C++ code ------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
11 /// \brief This file contains the implementation of the UnwrappedLineParser,
12 /// which turns a stream of tokens into UnwrappedLines.
14 //===----------------------------------------------------------------------===//
16 #define DEBUG_TYPE "format-parser"
18 #include "UnwrappedLineParser.h"
19 #include "clang/Basic/Diagnostic.h"
20 #include "llvm/Support/Debug.h"
25 class ScopedDeclarationState {
27 ScopedDeclarationState(UnwrappedLine &Line, std::vector<bool> &Stack,
28 bool MustBeDeclaration)
29 : Line(Line), Stack(Stack) {
30 Line.MustBeDeclaration = MustBeDeclaration;
31 Stack.push_back(MustBeDeclaration);
33 ~ScopedDeclarationState() {
36 Line.MustBeDeclaration = Stack.back();
38 Line.MustBeDeclaration = true;
42 std::vector<bool> &Stack;
45 class ScopedMacroState : public FormatTokenSource {
47 ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource,
48 FormatToken &ResetToken, bool &StructuralError)
49 : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken),
50 PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource),
51 StructuralError(StructuralError),
52 PreviousStructuralError(StructuralError) {
55 Line.InPPDirective = true;
59 TokenSource = PreviousTokenSource;
61 Line.InPPDirective = false;
62 Line.Level = PreviousLineLevel;
63 StructuralError = PreviousStructuralError;
66 virtual FormatToken getNextToken() {
67 // The \c UnwrappedLineParser guards against this by never calling
68 // \c getNextToken() after it has encountered the first eof token.
70 Token = PreviousTokenSource->getNextToken();
77 bool eof() { return Token.HasUnescapedNewline; }
79 FormatToken createEOF() {
80 FormatToken FormatTok;
81 FormatTok.Tok.startToken();
82 FormatTok.Tok.setKind(tok::eof);
87 FormatTokenSource *&TokenSource;
88 FormatToken &ResetToken;
89 unsigned PreviousLineLevel;
90 FormatTokenSource *PreviousTokenSource;
91 bool &StructuralError;
92 bool PreviousStructuralError;
97 class ScopedLineState {
99 ScopedLineState(UnwrappedLineParser &Parser,
100 bool SwitchToPreprocessorLines = false)
101 : Parser(Parser), SwitchToPreprocessorLines(SwitchToPreprocessorLines) {
102 if (SwitchToPreprocessorLines)
103 Parser.CurrentLines = &Parser.PreprocessorDirectives;
104 PreBlockLine = Parser.Line.take();
105 Parser.Line.reset(new UnwrappedLine());
106 Parser.Line->Level = PreBlockLine->Level;
107 Parser.Line->InPPDirective = PreBlockLine->InPPDirective;
111 if (!Parser.Line->Tokens.empty()) {
112 Parser.addUnwrappedLine();
114 assert(Parser.Line->Tokens.empty());
115 Parser.Line.reset(PreBlockLine);
116 Parser.MustBreakBeforeNextToken = true;
117 if (SwitchToPreprocessorLines)
118 Parser.CurrentLines = &Parser.Lines;
122 UnwrappedLineParser &Parser;
123 const bool SwitchToPreprocessorLines;
125 UnwrappedLine *PreBlockLine;
128 UnwrappedLineParser::UnwrappedLineParser(
129 clang::DiagnosticsEngine &Diag, const FormatStyle &Style,
130 FormatTokenSource &Tokens, UnwrappedLineConsumer &Callback)
131 : Line(new UnwrappedLine), MustBreakBeforeNextToken(false),
132 CurrentLines(&Lines), StructuralError(false), Diag(Diag), Style(Style),
133 Tokens(&Tokens), Callback(Callback) {}
135 bool UnwrappedLineParser::parse() {
136 DEBUG(llvm::dbgs() << "----\n");
139 for (std::vector<UnwrappedLine>::iterator I = Lines.begin(), E = Lines.end();
141 Callback.consumeUnwrappedLine(*I);
144 // Create line with eof token.
145 pushToken(FormatTok);
146 Callback.consumeUnwrappedLine(*Line);
147 return StructuralError;
150 void UnwrappedLineParser::parseFile() {
151 ScopedDeclarationState DeclarationState(
152 *Line, DeclarationScopeStack,
153 /*MustBeDeclaration=*/ !Line->InPPDirective);
154 parseLevel(/*HasOpeningBrace=*/ false);
155 // Make sure to format the remaining tokens.
160 void UnwrappedLineParser::parseLevel(bool HasOpeningBrace) {
162 switch (FormatTok.Tok.getKind()) {
168 // FIXME: Add parameter whether this can happen - if this happens, we must
169 // be in a non-declaration context.
170 parseBlock(/*MustBeDeclaration=*/ false);
176 Diag.Report(FormatTok.Tok.getLocation(),
177 Diag.getCustomDiagID(clang::DiagnosticsEngine::Error,
179 StructuralError = true;
184 parseStructuralElement();
190 void UnwrappedLineParser::parseBlock(bool MustBeDeclaration,
191 unsigned AddLevels) {
192 assert(FormatTok.Tok.is(tok::l_brace) && "'{' expected");
197 ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
199 Line->Level += AddLevels;
200 parseLevel(/*HasOpeningBrace=*/ true);
202 if (!FormatTok.Tok.is(tok::r_brace)) {
203 Line->Level -= AddLevels;
204 StructuralError = true;
208 nextToken(); // Munch the closing brace.
209 Line->Level -= AddLevels;
212 void UnwrappedLineParser::parsePPDirective() {
213 assert(FormatTok.Tok.is(tok::hash) && "'#' expected");
214 ScopedMacroState MacroState(*Line, Tokens, FormatTok, StructuralError);
217 if (FormatTok.Tok.getIdentifierInfo() == NULL) {
222 switch (FormatTok.Tok.getIdentifierInfo()->getPPKeywordID()) {
232 void UnwrappedLineParser::parsePPDefine() {
235 if (FormatTok.Tok.getKind() != tok::identifier) {
240 if (FormatTok.Tok.getKind() == tok::l_paren &&
241 FormatTok.WhiteSpaceLength == 0) {
247 // Errors during a preprocessor directive can only affect the layout of the
248 // preprocessor directive, and thus we ignore them. An alternative approach
249 // would be to use the same approach we use on the file level (no
250 // re-indentation if there was a structural error) within the macro
255 void UnwrappedLineParser::parsePPUnknown() {
262 // Here we blacklist certain tokens that are not usually the first token in an
263 // unwrapped line. This is used in attempt to distinguish macro calls without
264 // trailing semicolons from other constructs split to several lines.
265 bool tokenCanStartNewLine(clang::Token Tok) {
266 // Semicolon can be a null-statement, l_square can be a start of a macro or
267 // a C++11 attribute, but this doesn't seem to be common.
268 return Tok.isNot(tok::semi) && Tok.isNot(tok::l_brace) &&
269 Tok.isNot(tok::l_square) &&
270 // Tokens that can only be used as binary operators and a part of
271 // overloaded operator names.
272 Tok.isNot(tok::period) && Tok.isNot(tok::periodstar) &&
273 Tok.isNot(tok::arrow) && Tok.isNot(tok::arrowstar) &&
274 Tok.isNot(tok::less) && Tok.isNot(tok::greater) &&
275 Tok.isNot(tok::slash) && Tok.isNot(tok::percent) &&
276 Tok.isNot(tok::lessless) && Tok.isNot(tok::greatergreater) &&
277 Tok.isNot(tok::equal) && Tok.isNot(tok::plusequal) &&
278 Tok.isNot(tok::minusequal) && Tok.isNot(tok::starequal) &&
279 Tok.isNot(tok::slashequal) && Tok.isNot(tok::percentequal) &&
280 Tok.isNot(tok::ampequal) && Tok.isNot(tok::pipeequal) &&
281 Tok.isNot(tok::caretequal) && Tok.isNot(tok::greatergreaterequal) &&
282 Tok.isNot(tok::lesslessequal) &&
283 // Colon is used in labels, base class lists, initializer lists,
284 // range-based for loops, ternary operator, but should never be the
285 // first token in an unwrapped line.
286 Tok.isNot(tok::colon);
289 void UnwrappedLineParser::parseStructuralElement() {
290 assert(!FormatTok.Tok.is(tok::l_brace));
291 switch (FormatTok.Tok.getKind()) {
294 if (FormatTok.Tok.is(tok::l_brace)) {
298 switch (FormatTok.Tok.getObjCKeywordID()) {
299 case tok::objc_public:
300 case tok::objc_protected:
301 case tok::objc_package:
302 case tok::objc_private:
303 return parseAccessSpecifier();
304 case tok::objc_interface:
305 case tok::objc_implementation:
306 return parseObjCInterfaceOrImplementation();
307 case tok::objc_protocol:
308 return parseObjCProtocol();
310 return; // Handled by the caller.
311 case tok::objc_optional:
312 case tok::objc_required:
320 case tok::kw_namespace:
325 if (FormatTok.Tok.is(tok::kw_namespace)) {
331 case tok::kw_protected:
332 case tok::kw_private:
333 parseAccessSpecifier();
340 parseForOrWhileLoop();
348 case tok::kw_default:
360 if (FormatTok.Tok.is(tok::string_literal)) {
362 if (FormatTok.Tok.is(tok::l_brace)) {
363 parseBlock(/*MustBeDeclaration=*/ true, 0);
368 // In all other cases, parse the declaration.
374 switch (FormatTok.Tok.getKind()) {
377 if (FormatTok.Tok.is(tok::l_brace))
387 // A record declaration or definition is always the start of a structural
401 // A block outside of parentheses must be the last part of a
402 // structural element.
403 // FIXME: Figure out cases where this is not true, and add projections for
404 // them (the one we know is missing are lambdas).
405 parseBlock(/*MustBeDeclaration=*/ false);
408 case tok::identifier:
410 if (Line->Tokens.size() == 1) {
411 if (FormatTok.Tok.is(tok::colon)) {
415 // Recognize function-like macro usages without trailing semicolon.
416 if (FormatTok.Tok.is(tok::l_paren)) {
418 if (FormatTok.HasUnescapedNewline &&
419 tokenCanStartNewLine(FormatTok.Tok)) {
428 if (FormatTok.Tok.is(tok::l_brace)) {
439 void UnwrappedLineParser::parseBracedList() {
442 // FIXME: Once we have an expression parser in the UnwrappedLineParser,
443 // replace this by using parseAssigmentExpression() inside.
444 bool StartOfExpression = true;
446 // FIXME: When we start to support lambdas, we'll want to parse them away
447 // here, otherwise our bail-out scenarios below break. The better solution
448 // might be to just implement a more or less complete expression parser.
449 switch (FormatTok.Tok.getKind()) {
451 if (!StartOfExpression) {
452 // Probably a missing closing brace. Bail out.
457 StartOfExpression = false;
463 // Probably a missing closing brace. Bail out.
467 StartOfExpression = true;
471 StartOfExpression = false;
477 void UnwrappedLineParser::parseReturn() {
481 switch (FormatTok.Tok.getKind()) {
484 if (FormatTok.Tok.isNot(tok::semi)) {
485 // Assume missing ';'.
494 // Assume missing ';'.
508 void UnwrappedLineParser::parseParens() {
509 assert(FormatTok.Tok.is(tok::l_paren) && "'(' expected.");
512 switch (FormatTok.Tok.getKind()) {
521 ScopedLineState LineState(*this);
522 ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
523 /*MustBeDeclaration=*/ false);
525 parseLevel(/*HasOpeningBrace=*/ true);
531 if (FormatTok.Tok.is(tok::l_brace))
541 void UnwrappedLineParser::parseIfThenElse() {
542 assert(FormatTok.Tok.is(tok::kw_if) && "'if' expected");
544 if (FormatTok.Tok.is(tok::l_paren))
546 bool NeedsUnwrappedLine = false;
547 if (FormatTok.Tok.is(tok::l_brace)) {
548 parseBlock(/*MustBeDeclaration=*/ false);
549 NeedsUnwrappedLine = true;
553 parseStructuralElement();
556 if (FormatTok.Tok.is(tok::kw_else)) {
558 if (FormatTok.Tok.is(tok::l_brace)) {
559 parseBlock(/*MustBeDeclaration=*/ false);
561 } else if (FormatTok.Tok.is(tok::kw_if)) {
566 parseStructuralElement();
569 } else if (NeedsUnwrappedLine) {
574 void UnwrappedLineParser::parseNamespace() {
575 assert(FormatTok.Tok.is(tok::kw_namespace) && "'namespace' expected");
577 if (FormatTok.Tok.is(tok::identifier))
579 if (FormatTok.Tok.is(tok::l_brace)) {
580 parseBlock(/*MustBeDeclaration=*/ true, 0);
581 // Munch the semicolon after a namespace. This is more common than one would
582 // think. Puttin the semicolon into its own line is very ugly.
583 if (FormatTok.Tok.is(tok::semi))
587 // FIXME: Add error handling.
590 void UnwrappedLineParser::parseForOrWhileLoop() {
591 assert((FormatTok.Tok.is(tok::kw_for) || FormatTok.Tok.is(tok::kw_while)) &&
592 "'for' or 'while' expected");
594 if (FormatTok.Tok.is(tok::l_paren))
596 if (FormatTok.Tok.is(tok::l_brace)) {
597 parseBlock(/*MustBeDeclaration=*/ false);
602 parseStructuralElement();
607 void UnwrappedLineParser::parseDoWhile() {
608 assert(FormatTok.Tok.is(tok::kw_do) && "'do' expected");
610 if (FormatTok.Tok.is(tok::l_brace)) {
611 parseBlock(/*MustBeDeclaration=*/ false);
615 parseStructuralElement();
619 // FIXME: Add error handling.
620 if (!FormatTok.Tok.is(tok::kw_while)) {
626 parseStructuralElement();
629 void UnwrappedLineParser::parseLabel() {
630 if (FormatTok.Tok.isNot(tok::colon))
633 unsigned OldLineLevel = Line->Level;
634 if (Line->Level > 1 || (!Line->InPPDirective && Line->Level > 0))
636 if (CommentsBeforeNextToken.empty() && FormatTok.Tok.is(tok::l_brace)) {
637 parseBlock(/*MustBeDeclaration=*/ false);
638 if (FormatTok.Tok.is(tok::kw_break))
639 parseStructuralElement(); // "break;" after "}" goes on the same line.
642 Line->Level = OldLineLevel;
645 void UnwrappedLineParser::parseCaseLabel() {
646 assert(FormatTok.Tok.is(tok::kw_case) && "'case' expected");
647 // FIXME: fix handling of complex expressions here.
650 } while (!eof() && !FormatTok.Tok.is(tok::colon));
654 void UnwrappedLineParser::parseSwitch() {
655 assert(FormatTok.Tok.is(tok::kw_switch) && "'switch' expected");
657 if (FormatTok.Tok.is(tok::l_paren))
659 if (FormatTok.Tok.is(tok::l_brace)) {
660 parseBlock(/*MustBeDeclaration=*/ false, Style.IndentCaseLabels ? 2 : 1);
664 Line->Level += (Style.IndentCaseLabels ? 2 : 1);
665 parseStructuralElement();
666 Line->Level -= (Style.IndentCaseLabels ? 2 : 1);
670 void UnwrappedLineParser::parseAccessSpecifier() {
672 // Otherwise, we don't know what it is, and we'd better keep the next token.
673 if (FormatTok.Tok.is(tok::colon))
678 void UnwrappedLineParser::parseEnum() {
680 if (FormatTok.Tok.is(tok::identifier) ||
681 FormatTok.Tok.is(tok::kw___attribute) ||
682 FormatTok.Tok.is(tok::kw___declspec)) {
684 // We can have macros or attributes in between 'enum' and the enum name.
685 if (FormatTok.Tok.is(tok::l_paren)) {
688 if (FormatTok.Tok.is(tok::identifier))
691 if (FormatTok.Tok.is(tok::l_brace)) {
696 switch (FormatTok.Tok.getKind()) {
715 // We fall through to parsing a structural element afterwards, so that in
717 // "} n, m;" will end up in one unwrapped line.
720 void UnwrappedLineParser::parseRecord() {
722 if (FormatTok.Tok.is(tok::identifier) ||
723 FormatTok.Tok.is(tok::kw___attribute) ||
724 FormatTok.Tok.is(tok::kw___declspec)) {
726 // We can have macros or attributes in between 'class' and the class name.
727 if (FormatTok.Tok.is(tok::l_paren)) {
730 // The actual identifier can be a nested name specifier, and in macros
731 // it is often token-pasted.
732 while (FormatTok.Tok.is(tok::identifier) ||
733 FormatTok.Tok.is(tok::coloncolon) || FormatTok.Tok.is(tok::hashhash))
736 // Note that parsing away template declarations here leads to incorrectly
737 // accepting function declarations as record declarations.
738 // In general, we cannot solve this problem. Consider:
739 // class A<int> B() {}
740 // which can be a function definition or a class definition when B() is a
741 // macro. If we find enough real-world cases where this is a problem, we
742 // can parse for the 'template' keyword in the beginning of the statement,
743 // and thus rule out the record production in case there is no template
744 // (this would still leave us with an ambiguity between template function
745 // and class declarations).
746 if (FormatTok.Tok.is(tok::colon) || FormatTok.Tok.is(tok::less)) {
747 while (!eof() && FormatTok.Tok.isNot(tok::l_brace)) {
748 if (FormatTok.Tok.is(tok::semi))
754 if (FormatTok.Tok.is(tok::l_brace))
755 parseBlock(/*MustBeDeclaration=*/ true);
756 // We fall through to parsing a structural element afterwards, so
758 // will end up in one unwrapped line.
761 void UnwrappedLineParser::parseObjCProtocolList() {
762 assert(FormatTok.Tok.is(tok::less) && "'<' expected.");
765 while (!eof() && FormatTok.Tok.isNot(tok::greater));
766 nextToken(); // Skip '>'.
769 void UnwrappedLineParser::parseObjCUntilAtEnd() {
771 if (FormatTok.Tok.isObjCAtKeyword(tok::objc_end)) {
776 parseStructuralElement();
780 void UnwrappedLineParser::parseObjCInterfaceOrImplementation() {
782 nextToken(); // interface name
784 // @interface can be followed by either a base class, or a category.
785 if (FormatTok.Tok.is(tok::colon)) {
787 nextToken(); // base class name
788 } else if (FormatTok.Tok.is(tok::l_paren))
789 // Skip category, if present.
792 if (FormatTok.Tok.is(tok::less))
793 parseObjCProtocolList();
795 // If instance variables are present, keep the '{' on the first line too.
796 if (FormatTok.Tok.is(tok::l_brace))
797 parseBlock(/*MustBeDeclaration=*/ true);
799 // With instance variables, this puts '}' on its own line. Without instance
800 // variables, this ends the @interface line.
803 parseObjCUntilAtEnd();
806 void UnwrappedLineParser::parseObjCProtocol() {
808 nextToken(); // protocol name
810 if (FormatTok.Tok.is(tok::less))
811 parseObjCProtocolList();
813 // Check for protocol declaration.
814 if (FormatTok.Tok.is(tok::semi)) {
816 return addUnwrappedLine();
820 parseObjCUntilAtEnd();
823 void UnwrappedLineParser::addUnwrappedLine() {
824 if (Line->Tokens.empty())
827 llvm::dbgs() << "Line(" << Line->Level << ")"
828 << (Line->InPPDirective ? " MACRO" : "") << ": ";
829 for (std::list<FormatToken>::iterator I = Line->Tokens.begin(),
830 E = Line->Tokens.end();
832 llvm::dbgs() << I->Tok.getName() << " ";
835 llvm::dbgs() << "\n";
837 CurrentLines->push_back(*Line);
838 Line->Tokens.clear();
839 if (CurrentLines == &Lines && !PreprocessorDirectives.empty()) {
840 for (std::vector<UnwrappedLine>::iterator
841 I = PreprocessorDirectives.begin(),
842 E = PreprocessorDirectives.end();
844 CurrentLines->push_back(*I);
846 PreprocessorDirectives.clear();
850 bool UnwrappedLineParser::eof() const { return FormatTok.Tok.is(tok::eof); }
852 void UnwrappedLineParser::flushComments(bool NewlineBeforeNext) {
853 bool JustComments = Line->Tokens.empty();
854 for (SmallVectorImpl<FormatToken>::const_iterator
855 I = CommentsBeforeNextToken.begin(),
856 E = CommentsBeforeNextToken.end();
858 if (I->NewlinesBefore && JustComments) {
863 if (NewlineBeforeNext && JustComments) {
866 CommentsBeforeNextToken.clear();
869 void UnwrappedLineParser::nextToken() {
872 flushComments(FormatTok.NewlinesBefore > 0);
873 pushToken(FormatTok);
877 void UnwrappedLineParser::readToken() {
878 bool CommentsInCurrentLine = true;
880 FormatTok = Tokens->getNextToken();
881 while (!Line->InPPDirective && FormatTok.Tok.is(tok::hash) &&
882 (FormatTok.HasUnescapedNewline || FormatTok.IsFirst)) {
883 // If there is an unfinished unwrapped line, we flush the preprocessor
884 // directives only after that unwrapped line was finished later.
885 bool SwitchToPreprocessorLines =
886 !Line->Tokens.empty() && CurrentLines == &Lines;
887 ScopedLineState BlockState(*this, SwitchToPreprocessorLines);
888 // Comments stored before the preprocessor directive need to be output
889 // before the preprocessor directive, at the same level as the
890 // preprocessor directive, as we consider them to apply to the directive.
891 flushComments(FormatTok.NewlinesBefore > 0);
894 if (!FormatTok.Tok.is(tok::comment))
896 if (FormatTok.NewlinesBefore > 0 || FormatTok.IsFirst) {
897 CommentsInCurrentLine = false;
899 if (CommentsInCurrentLine) {
900 pushToken(FormatTok);
902 CommentsBeforeNextToken.push_back(FormatTok);
907 void UnwrappedLineParser::pushToken(const FormatToken &Tok) {
908 Line->Tokens.push_back(Tok);
909 if (MustBreakBeforeNextToken) {
910 Line->Tokens.back().MustBreakBefore = true;
911 MustBreakBeforeNextToken = false;
915 } // end namespace format
916 } // end namespace clang