1 //===--- TokenAnnotator.cpp - Format C++ code -----------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
11 /// \brief This file implements a token annotator, i.e. creates
12 /// \c AnnotatedTokens out of \c FormatTokens with required extra information.
14 //===----------------------------------------------------------------------===//
16 #include "TokenAnnotator.h"
17 #include "clang/Basic/SourceManager.h"
18 #include "llvm/ADT/SmallPtrSet.h"
19 #include "llvm/Support/Debug.h"
21 #define DEBUG_TYPE "format-token-annotator"
28 /// \brief A parser that gathers additional information about tokens.
30 /// The \c TokenAnnotator tries to match parenthesis and square brakets and
31 /// store a parenthesis levels. It also tries to resolve matching "<" and ">"
32 /// into template parameter lists.
33 class AnnotatingParser {
35 AnnotatingParser(const FormatStyle &Style, AnnotatedLine &Line,
36 const AdditionalKeywords &Keywords)
37 : Style(Style), Line(Line), CurrentToken(Line.First), AutoFound(false),
39 Contexts.push_back(Context(tok::unknown, 1, /*IsExpression=*/false));
40 resetTokenMetadata(CurrentToken);
47 FormatToken *Left = CurrentToken->Previous;
48 Left->ParentBracket = Contexts.back().ContextKind;
49 ScopedContextCreator ContextCreator(*this, tok::less, 10);
51 // If this angle is in the context of an expression, we need to be more
52 // hesitant to detect it as opening template parameters.
53 bool InExprContext = Contexts.back().IsExpression;
55 Contexts.back().IsExpression = false;
56 // If there's a template keyword before the opening angle bracket, this is a
57 // template parameter, not an argument.
58 Contexts.back().InTemplateArgument =
59 Left->Previous && Left->Previous->Tok.isNot(tok::kw_template);
61 if (Style.Language == FormatStyle::LK_Java &&
62 CurrentToken->is(tok::question))
65 while (CurrentToken) {
66 if (CurrentToken->is(tok::greater)) {
67 Left->MatchingParen = CurrentToken;
68 CurrentToken->MatchingParen = Left;
69 CurrentToken->Type = TT_TemplateCloser;
73 if (CurrentToken->is(tok::question) &&
74 Style.Language == FormatStyle::LK_Java) {
78 if (CurrentToken->isOneOf(tok::r_paren, tok::r_square, tok::r_brace) ||
79 (CurrentToken->isOneOf(tok::colon, tok::question) && InExprContext))
81 // If a && or || is found and interpreted as a binary operator, this set
82 // of angles is likely part of something like "a < b && c > d". If the
83 // angles are inside an expression, the ||/&& might also be a binary
84 // operator that was misinterpreted because we are parsing template
86 // FIXME: This is getting out of hand, write a decent parser.
87 if (CurrentToken->Previous->isOneOf(tok::pipepipe, tok::ampamp) &&
88 CurrentToken->Previous->is(TT_BinaryOperator) &&
89 Contexts[Contexts.size() - 2].IsExpression &&
90 Line.First->isNot(tok::kw_template))
92 updateParameterCount(Left, CurrentToken);
99 bool parseParens(bool LookForDecls = false) {
102 FormatToken *Left = CurrentToken->Previous;
103 Left->ParentBracket = Contexts.back().ContextKind;
104 ScopedContextCreator ContextCreator(*this, tok::l_paren, 1);
106 // FIXME: This is a bit of a hack. Do better.
107 Contexts.back().ColonIsForRangeExpr =
108 Contexts.size() == 2 && Contexts[0].ColonIsForRangeExpr;
110 bool StartsObjCMethodExpr = false;
111 if (CurrentToken->is(tok::caret)) {
112 // (^ can start a block type.
113 Left->Type = TT_ObjCBlockLParen;
114 } else if (FormatToken *MaybeSel = Left->Previous) {
115 // @selector( starts a selector.
116 if (MaybeSel->isObjCAtKeyword(tok::objc_selector) && MaybeSel->Previous &&
117 MaybeSel->Previous->is(tok::at)) {
118 StartsObjCMethodExpr = true;
122 if (Left->Previous &&
123 (Left->Previous->isOneOf(tok::kw_static_assert, tok::kw_if,
124 tok::kw_while, tok::l_paren, tok::comma) ||
125 Left->Previous->is(TT_BinaryOperator))) {
126 // static_assert, if and while usually contain expressions.
127 Contexts.back().IsExpression = true;
128 } else if (Left->Previous && Left->Previous->is(tok::r_square) &&
129 Left->Previous->MatchingParen &&
130 Left->Previous->MatchingParen->is(TT_LambdaLSquare)) {
131 // This is a parameter list of a lambda expression.
132 Contexts.back().IsExpression = false;
133 } else if (Line.InPPDirective &&
135 !Left->Previous->isOneOf(tok::identifier,
136 TT_OverloadedOperator))) {
137 Contexts.back().IsExpression = true;
138 } else if (Contexts[Contexts.size() - 2].CaretFound) {
139 // This is the parameter list of an ObjC block.
140 Contexts.back().IsExpression = false;
141 } else if (Left->Previous && Left->Previous->is(tok::kw___attribute)) {
142 Left->Type = TT_AttributeParen;
143 } else if (Left->Previous && Left->Previous->is(TT_ForEachMacro)) {
144 // The first argument to a foreach macro is a declaration.
145 Contexts.back().IsForEachMacro = true;
146 Contexts.back().IsExpression = false;
147 } else if (Left->Previous && Left->Previous->MatchingParen &&
148 Left->Previous->MatchingParen->is(TT_ObjCBlockLParen)) {
149 Contexts.back().IsExpression = false;
152 if (StartsObjCMethodExpr) {
153 Contexts.back().ColonIsObjCMethodExpr = true;
154 Left->Type = TT_ObjCMethodExpr;
157 bool MightBeFunctionType = CurrentToken->is(tok::star);
158 bool HasMultipleLines = false;
159 bool HasMultipleParametersOnALine = false;
160 bool MightBeObjCForRangeLoop =
161 Left->Previous && Left->Previous->is(tok::kw_for);
162 while (CurrentToken) {
163 // LookForDecls is set when "if (" has been seen. Check for
164 // 'identifier' '*' 'identifier' followed by not '=' -- this
165 // '*' has to be a binary operator but determineStarAmpUsage() will
166 // categorize it as an unary operator, so set the right type here.
167 if (LookForDecls && CurrentToken->Next) {
168 FormatToken *Prev = CurrentToken->getPreviousNonComment();
170 FormatToken *PrevPrev = Prev->getPreviousNonComment();
171 FormatToken *Next = CurrentToken->Next;
172 if (PrevPrev && PrevPrev->is(tok::identifier) &&
173 Prev->isOneOf(tok::star, tok::amp, tok::ampamp) &&
174 CurrentToken->is(tok::identifier) && Next->isNot(tok::equal)) {
175 Prev->Type = TT_BinaryOperator;
176 LookForDecls = false;
181 if (CurrentToken->Previous->is(TT_PointerOrReference) &&
182 CurrentToken->Previous->Previous->isOneOf(tok::l_paren,
184 MightBeFunctionType = true;
185 if (CurrentToken->Previous->is(TT_BinaryOperator))
186 Contexts.back().IsExpression = true;
187 if (CurrentToken->is(tok::r_paren)) {
188 if (MightBeFunctionType && CurrentToken->Next &&
189 (CurrentToken->Next->is(tok::l_paren) ||
190 (CurrentToken->Next->is(tok::l_square) &&
191 !Contexts.back().IsExpression)))
192 Left->Type = TT_FunctionTypeLParen;
193 Left->MatchingParen = CurrentToken;
194 CurrentToken->MatchingParen = Left;
196 if (StartsObjCMethodExpr) {
197 CurrentToken->Type = TT_ObjCMethodExpr;
198 if (Contexts.back().FirstObjCSelectorName) {
199 Contexts.back().FirstObjCSelectorName->LongestObjCSelectorName =
200 Contexts.back().LongestObjCSelectorName;
204 if (Left->is(TT_AttributeParen))
205 CurrentToken->Type = TT_AttributeParen;
206 if (Left->Previous && Left->Previous->is(TT_JavaAnnotation))
207 CurrentToken->Type = TT_JavaAnnotation;
208 if (Left->Previous && Left->Previous->is(TT_LeadingJavaAnnotation))
209 CurrentToken->Type = TT_LeadingJavaAnnotation;
211 if (!HasMultipleLines)
212 Left->PackingKind = PPK_Inconclusive;
213 else if (HasMultipleParametersOnALine)
214 Left->PackingKind = PPK_BinPacked;
216 Left->PackingKind = PPK_OnePerLine;
221 if (CurrentToken->isOneOf(tok::r_square, tok::r_brace))
224 if (CurrentToken->is(tok::l_brace))
225 Left->Type = TT_Unknown; // Not TT_ObjCBlockLParen
226 if (CurrentToken->is(tok::comma) && CurrentToken->Next &&
227 !CurrentToken->Next->HasUnescapedNewline &&
228 !CurrentToken->Next->isTrailingComment())
229 HasMultipleParametersOnALine = true;
230 if (CurrentToken->isOneOf(tok::kw_const, tok::kw_auto) ||
231 CurrentToken->isSimpleTypeSpecifier())
232 Contexts.back().IsExpression = false;
233 if (CurrentToken->isOneOf(tok::semi, tok::colon))
234 MightBeObjCForRangeLoop = false;
235 if (MightBeObjCForRangeLoop && CurrentToken->is(Keywords.kw_in))
236 CurrentToken->Type = TT_ObjCForIn;
237 // When we discover a 'new', we set CanBeExpression to 'false' in order to
238 // parse the type correctly. Reset that after a comma.
239 if (CurrentToken->is(tok::comma))
240 Contexts.back().CanBeExpression = true;
242 FormatToken *Tok = CurrentToken;
245 updateParameterCount(Left, Tok);
246 if (CurrentToken && CurrentToken->HasUnescapedNewline)
247 HasMultipleLines = true;
256 // A '[' could be an index subscript (after an identifier or after
257 // ')' or ']'), it could be the start of an Objective-C method
258 // expression, or it could the the start of an Objective-C array literal.
259 FormatToken *Left = CurrentToken->Previous;
260 Left->ParentBracket = Contexts.back().ContextKind;
261 FormatToken *Parent = Left->getPreviousNonComment();
262 bool StartsObjCMethodExpr =
263 Contexts.back().CanBeExpression && Left->isNot(TT_LambdaLSquare) &&
264 CurrentToken->isNot(tok::l_brace) &&
266 Parent->isOneOf(tok::colon, tok::l_square, tok::l_paren,
267 tok::kw_return, tok::kw_throw) ||
268 Parent->isUnaryOperator() ||
269 Parent->isOneOf(TT_ObjCForIn, TT_CastRParen) ||
270 getBinOpPrecedence(Parent->Tok.getKind(), true, true) > prec::Unknown);
271 ScopedContextCreator ContextCreator(*this, tok::l_square, 10);
272 Contexts.back().IsExpression = true;
273 bool ColonFound = false;
275 if (StartsObjCMethodExpr) {
276 Contexts.back().ColonIsObjCMethodExpr = true;
277 Left->Type = TT_ObjCMethodExpr;
278 } else if (Parent && Parent->is(tok::at)) {
279 Left->Type = TT_ArrayInitializerLSquare;
280 } else if (Left->is(TT_Unknown)) {
281 Left->Type = TT_ArraySubscriptLSquare;
284 while (CurrentToken) {
285 if (CurrentToken->is(tok::r_square)) {
286 if (CurrentToken->Next && CurrentToken->Next->is(tok::l_paren) &&
287 Left->is(TT_ObjCMethodExpr)) {
288 // An ObjC method call is rarely followed by an open parenthesis.
289 // FIXME: Do we incorrectly label ":" with this?
290 StartsObjCMethodExpr = false;
291 Left->Type = TT_Unknown;
293 if (StartsObjCMethodExpr && CurrentToken->Previous != Left) {
294 CurrentToken->Type = TT_ObjCMethodExpr;
295 // determineStarAmpUsage() thinks that '*' '[' is allocating an
296 // array of pointers, but if '[' starts a selector then '*' is a
298 if (Parent && Parent->is(TT_PointerOrReference))
299 Parent->Type = TT_BinaryOperator;
301 Left->MatchingParen = CurrentToken;
302 CurrentToken->MatchingParen = Left;
303 if (Contexts.back().FirstObjCSelectorName) {
304 Contexts.back().FirstObjCSelectorName->LongestObjCSelectorName =
305 Contexts.back().LongestObjCSelectorName;
306 if (Left->BlockParameterCount > 1)
307 Contexts.back().FirstObjCSelectorName->LongestObjCSelectorName = 0;
312 if (CurrentToken->isOneOf(tok::r_paren, tok::r_brace))
314 if (CurrentToken->is(tok::colon)) {
315 if (Left->is(TT_ArraySubscriptLSquare)) {
316 Left->Type = TT_ObjCMethodExpr;
317 StartsObjCMethodExpr = true;
318 Contexts.back().ColonIsObjCMethodExpr = true;
319 if (Parent && Parent->is(tok::r_paren))
320 Parent->Type = TT_CastRParen;
324 if (CurrentToken->is(tok::comma) &&
325 Style.Language != FormatStyle::LK_Proto &&
326 (Left->is(TT_ArraySubscriptLSquare) ||
327 (Left->is(TT_ObjCMethodExpr) && !ColonFound)))
328 Left->Type = TT_ArrayInitializerLSquare;
329 FormatToken *Tok = CurrentToken;
332 updateParameterCount(Left, Tok);
339 FormatToken *Left = CurrentToken->Previous;
340 Left->ParentBracket = Contexts.back().ContextKind;
342 if (Contexts.back().CaretFound)
343 Left->Type = TT_ObjCBlockLBrace;
344 Contexts.back().CaretFound = false;
346 ScopedContextCreator ContextCreator(*this, tok::l_brace, 1);
347 Contexts.back().ColonIsDictLiteral = true;
348 if (Left->BlockKind == BK_BracedInit)
349 Contexts.back().IsExpression = true;
351 while (CurrentToken) {
352 if (CurrentToken->is(tok::r_brace)) {
353 Left->MatchingParen = CurrentToken;
354 CurrentToken->MatchingParen = Left;
358 if (CurrentToken->isOneOf(tok::r_paren, tok::r_square))
360 updateParameterCount(Left, CurrentToken);
361 if (CurrentToken->isOneOf(tok::colon, tok::l_brace)) {
362 FormatToken *Previous = CurrentToken->getPreviousNonComment();
363 if ((CurrentToken->is(tok::colon) ||
364 Style.Language == FormatStyle::LK_Proto) &&
365 Previous->is(tok::identifier))
366 Previous->Type = TT_SelectorName;
367 if (CurrentToken->is(tok::colon) ||
368 Style.Language == FormatStyle::LK_JavaScript)
369 Left->Type = TT_DictLiteral;
378 void updateParameterCount(FormatToken *Left, FormatToken *Current) {
379 if (Current->is(TT_LambdaLSquare) ||
380 (Current->is(tok::caret) && Current->is(TT_UnaryOperator)) ||
381 (Style.Language == FormatStyle::LK_JavaScript &&
382 Current->is(Keywords.kw_function))) {
383 ++Left->BlockParameterCount;
385 if (Current->is(tok::comma)) {
386 ++Left->ParameterCount;
388 Left->Role.reset(new CommaSeparatedList(Style));
389 Left->Role->CommaFound(Current);
390 } else if (Left->ParameterCount == 0 && Current->isNot(tok::comment)) {
391 Left->ParameterCount = 1;
395 bool parseConditional() {
396 while (CurrentToken) {
397 if (CurrentToken->is(tok::colon)) {
398 CurrentToken->Type = TT_ConditionalExpr;
408 bool parseTemplateDeclaration() {
409 if (CurrentToken && CurrentToken->is(tok::less)) {
410 CurrentToken->Type = TT_TemplateOpener;
415 CurrentToken->Previous->ClosesTemplateDeclaration = true;
421 bool consumeToken() {
422 FormatToken *Tok = CurrentToken;
424 switch (Tok->Tok.getKind()) {
427 if (!Tok->Previous && Line.MustBeDeclaration)
428 Tok->Type = TT_ObjCMethodSpecifier;
433 // Colons from ?: are handled in parseConditional().
434 if (Style.Language == FormatStyle::LK_JavaScript) {
435 if (Contexts.back().ColonIsForRangeExpr || // colon in for loop
436 (Contexts.size() == 1 && // switch/case labels
437 !Line.First->isOneOf(tok::kw_enum, tok::kw_case)) ||
438 Contexts.back().ContextKind == tok::l_paren || // function params
439 Contexts.back().ContextKind == tok::l_square || // array type
440 Line.MustBeDeclaration) { // method/property declaration
441 Tok->Type = TT_JsTypeColon;
445 if (Contexts.back().ColonIsDictLiteral) {
446 Tok->Type = TT_DictLiteral;
447 } else if (Contexts.back().ColonIsObjCMethodExpr ||
448 Line.First->is(TT_ObjCMethodSpecifier)) {
449 Tok->Type = TT_ObjCMethodExpr;
450 Tok->Previous->Type = TT_SelectorName;
451 if (Tok->Previous->ColumnWidth >
452 Contexts.back().LongestObjCSelectorName) {
453 Contexts.back().LongestObjCSelectorName = Tok->Previous->ColumnWidth;
455 if (!Contexts.back().FirstObjCSelectorName)
456 Contexts.back().FirstObjCSelectorName = Tok->Previous;
457 } else if (Contexts.back().ColonIsForRangeExpr) {
458 Tok->Type = TT_RangeBasedForLoopColon;
459 } else if (CurrentToken && CurrentToken->is(tok::numeric_constant)) {
460 Tok->Type = TT_BitFieldColon;
461 } else if (Contexts.size() == 1 &&
462 !Line.First->isOneOf(tok::kw_enum, tok::kw_case)) {
463 if (Tok->Previous->is(tok::r_paren))
464 Tok->Type = TT_CtorInitializerColon;
466 Tok->Type = TT_InheritanceColon;
467 } else if (Tok->Previous->is(tok::identifier) && Tok->Next &&
468 Tok->Next->isOneOf(tok::r_paren, tok::comma)) {
469 // This handles a special macro in ObjC code where selectors including
470 // the colon are passed as macro arguments.
471 Tok->Type = TT_ObjCMethodExpr;
472 } else if (Contexts.back().ContextKind == tok::l_paren) {
473 Tok->Type = TT_InlineASMColon;
478 if (CurrentToken && CurrentToken->is(tok::l_paren)) {
480 if (!parseParens(/*LookForDecls=*/true))
485 Contexts.back().ColonIsForRangeExpr = true;
493 if (Line.MustBeDeclaration && Contexts.size() == 1 &&
494 !Contexts.back().IsExpression && Line.First->isNot(TT_ObjCProperty) &&
496 !Tok->Previous->isOneOf(tok::kw_decltype, TT_LeadingJavaAnnotation)))
497 Line.MightBeFunctionDecl = true;
508 if (!NonTemplateLess.count(Tok) &&
510 (!Tok->Previous->Tok.isLiteral() &&
511 !(Tok->Previous->is(tok::r_paren) && Contexts.size() > 1))) &&
513 Tok->Type = TT_TemplateOpener;
515 Tok->Type = TT_BinaryOperator;
516 NonTemplateLess.insert(Tok);
525 // Lines can start with '}'.
530 Tok->Type = TT_BinaryOperator;
532 case tok::kw_operator:
533 while (CurrentToken &&
534 !CurrentToken->isOneOf(tok::l_paren, tok::semi, tok::r_paren)) {
535 if (CurrentToken->isOneOf(tok::star, tok::amp))
536 CurrentToken->Type = TT_PointerOrReference;
538 if (CurrentToken && CurrentToken->Previous->is(TT_BinaryOperator))
539 CurrentToken->Previous->Type = TT_OverloadedOperator;
542 CurrentToken->Type = TT_OverloadedOperatorLParen;
543 if (CurrentToken->Previous->is(TT_BinaryOperator))
544 CurrentToken->Previous->Type = TT_OverloadedOperator;
548 if (Style.Language == FormatStyle::LK_JavaScript && Tok->Next &&
549 Tok->Next->isOneOf(tok::semi, tok::colon, tok::r_paren,
551 // Question marks before semicolons, colons, etc. indicate optional
552 // types (fields, parameters), e.g.
553 // function(x?: string, y?) {...}
555 Tok->Type = TT_JsTypeOptionalQuestion;
558 // Declarations cannot be conditional expressions, this can only be part
559 // of a type declaration.
560 if (Line.MustBeDeclaration &&
561 Style.Language == FormatStyle::LK_JavaScript)
565 case tok::kw_template:
566 parseTemplateDeclaration();
569 if (Contexts.back().InCtorInitializer)
570 Tok->Type = TT_CtorInitializerComma;
571 else if (Contexts.back().FirstStartOfName &&
572 (Contexts.size() == 1 || Line.First->is(tok::kw_for))) {
573 Contexts.back().FirstStartOfName->PartOfMultiVariableDeclStmt = true;
574 Line.IsMultiVariableDeclStmt = true;
576 if (Contexts.back().IsForEachMacro)
577 Contexts.back().IsExpression = true;
585 void parseIncludeDirective() {
586 if (CurrentToken && CurrentToken->is(tok::less)) {
588 while (CurrentToken) {
589 if (CurrentToken->isNot(tok::comment) || CurrentToken->Next)
590 CurrentToken->Type = TT_ImplicitStringLiteral;
596 void parseWarningOrError() {
598 // We still want to format the whitespace left of the first token of the
601 while (CurrentToken) {
602 CurrentToken->Type = TT_ImplicitStringLiteral;
608 next(); // Consume "pragma".
610 CurrentToken->isOneOf(Keywords.kw_mark, Keywords.kw_option)) {
611 bool IsMark = CurrentToken->is(Keywords.kw_mark);
612 next(); // Consume "mark".
613 next(); // Consume first token (so we fix leading whitespace).
614 while (CurrentToken) {
615 if (IsMark || CurrentToken->Previous->is(TT_BinaryOperator))
616 CurrentToken->Type = TT_ImplicitStringLiteral;
622 LineType parsePreprocessorDirective() {
623 LineType Type = LT_PreprocessorDirective;
627 if (CurrentToken->Tok.is(tok::numeric_constant)) {
628 CurrentToken->SpacesRequiredBefore = 1;
631 // Hashes in the middle of a line can lead to any strange token
633 if (!CurrentToken->Tok.getIdentifierInfo())
635 switch (CurrentToken->Tok.getIdentifierInfo()->getPPKeywordID()) {
636 case tok::pp_include:
637 case tok::pp_include_next:
640 parseIncludeDirective();
641 Type = LT_ImportStatement;
644 case tok::pp_warning:
645 parseWarningOrError();
652 Contexts.back().IsExpression = true;
664 LineType parseLine() {
665 NonTemplateLess.clear();
666 if (CurrentToken->is(tok::hash))
667 return parsePreprocessorDirective();
669 // Directly allow to 'import <string-literal>' to support protocol buffer
670 // definitions (code.google.com/p/protobuf) or missing "#" (either way we
671 // should not break the line).
672 IdentifierInfo *Info = CurrentToken->Tok.getIdentifierInfo();
673 if ((Style.Language == FormatStyle::LK_Java &&
674 CurrentToken->is(Keywords.kw_package)) ||
675 (Info && Info->getPPKeywordID() == tok::pp_import &&
676 CurrentToken->Next &&
677 CurrentToken->Next->isOneOf(tok::string_literal, tok::identifier,
680 parseIncludeDirective();
681 return LT_ImportStatement;
684 // If this line starts and ends in '<' and '>', respectively, it is likely
685 // part of "#define <a/b.h>".
686 if (CurrentToken->is(tok::less) && Line.Last->is(tok::greater)) {
687 parseIncludeDirective();
688 return LT_ImportStatement;
691 // In .proto files, top-level options are very similar to import statements
692 // and should not be line-wrapped.
693 if (Style.Language == FormatStyle::LK_Proto && Line.Level == 0 &&
694 CurrentToken->is(Keywords.kw_option)) {
696 if (CurrentToken && CurrentToken->is(tok::identifier))
697 return LT_ImportStatement;
700 bool KeywordVirtualFound = false;
701 bool ImportStatement = false;
702 while (CurrentToken) {
703 if (CurrentToken->is(tok::kw_virtual))
704 KeywordVirtualFound = true;
705 if (IsImportStatement(*CurrentToken))
706 ImportStatement = true;
710 if (KeywordVirtualFound)
711 return LT_VirtualFunctionDecl;
713 return LT_ImportStatement;
715 if (Line.First->is(TT_ObjCMethodSpecifier)) {
716 if (Contexts.back().FirstObjCSelectorName)
717 Contexts.back().FirstObjCSelectorName->LongestObjCSelectorName =
718 Contexts.back().LongestObjCSelectorName;
719 return LT_ObjCMethodDecl;
726 bool IsImportStatement(const FormatToken &Tok) {
727 // FIXME: Closure-library specific stuff should not be hard-coded but be
729 return Style.Language == FormatStyle::LK_JavaScript &&
730 Tok.TokenText == "goog" && Tok.Next && Tok.Next->is(tok::period) &&
731 Tok.Next->Next && (Tok.Next->Next->TokenText == "module" ||
732 Tok.Next->Next->TokenText == "require" ||
733 Tok.Next->Next->TokenText == "provide") &&
734 Tok.Next->Next->Next && Tok.Next->Next->Next->is(tok::l_paren);
737 void resetTokenMetadata(FormatToken *Token) {
741 // Reset token type in case we have already looked at it and then
742 // recovered from an error (e.g. failure to find the matching >).
743 if (!CurrentToken->isOneOf(TT_LambdaLSquare, TT_ForEachMacro,
744 TT_FunctionLBrace, TT_ImplicitStringLiteral,
745 TT_InlineASMBrace, TT_RegexLiteral,
746 TT_TrailingReturnArrow))
747 CurrentToken->Type = TT_Unknown;
748 CurrentToken->Role.reset();
749 CurrentToken->MatchingParen = nullptr;
750 CurrentToken->FakeLParens.clear();
751 CurrentToken->FakeRParens = 0;
756 CurrentToken->NestingLevel = Contexts.size() - 1;
757 CurrentToken->BindingStrength = Contexts.back().BindingStrength;
758 modifyContext(*CurrentToken);
759 determineTokenType(*CurrentToken);
760 CurrentToken = CurrentToken->Next;
763 resetTokenMetadata(CurrentToken);
766 /// \brief A struct to hold information valid in a specific context, e.g.
767 /// a pair of parenthesis.
769 Context(tok::TokenKind ContextKind, unsigned BindingStrength,
771 : ContextKind(ContextKind), BindingStrength(BindingStrength),
772 IsExpression(IsExpression) {}
774 tok::TokenKind ContextKind;
775 unsigned BindingStrength;
777 unsigned LongestObjCSelectorName = 0;
778 bool ColonIsForRangeExpr = false;
779 bool ColonIsDictLiteral = false;
780 bool ColonIsObjCMethodExpr = false;
781 FormatToken *FirstObjCSelectorName = nullptr;
782 FormatToken *FirstStartOfName = nullptr;
783 bool CanBeExpression = true;
784 bool InTemplateArgument = false;
785 bool InCtorInitializer = false;
786 bool CaretFound = false;
787 bool IsForEachMacro = false;
790 /// \brief Puts a new \c Context onto the stack \c Contexts for the lifetime
791 /// of each instance.
792 struct ScopedContextCreator {
795 ScopedContextCreator(AnnotatingParser &P, tok::TokenKind ContextKind,
798 P.Contexts.push_back(Context(ContextKind,
799 P.Contexts.back().BindingStrength + Increase,
800 P.Contexts.back().IsExpression));
803 ~ScopedContextCreator() { P.Contexts.pop_back(); }
806 void modifyContext(const FormatToken &Current) {
807 if (Current.getPrecedence() == prec::Assignment &&
808 !Line.First->isOneOf(tok::kw_template, tok::kw_using) &&
809 (!Current.Previous || Current.Previous->isNot(tok::kw_operator))) {
810 Contexts.back().IsExpression = true;
811 if (!Line.First->is(TT_UnaryOperator)) {
812 for (FormatToken *Previous = Current.Previous;
813 Previous && !Previous->isOneOf(tok::comma, tok::semi);
814 Previous = Previous->Previous) {
815 if (Previous->isOneOf(tok::r_square, tok::r_paren)) {
816 Previous = Previous->MatchingParen;
820 if (Previous->opensScope())
822 if (Previous->isOneOf(TT_BinaryOperator, TT_UnaryOperator) &&
823 Previous->isOneOf(tok::star, tok::amp, tok::ampamp) &&
824 Previous->Previous && Previous->Previous->isNot(tok::equal))
825 Previous->Type = TT_PointerOrReference;
828 } else if (Current.is(tok::lessless) &&
829 (!Current.Previous || !Current.Previous->is(tok::kw_operator))) {
830 Contexts.back().IsExpression = true;
831 } else if (Current.isOneOf(tok::kw_return, tok::kw_throw)) {
832 Contexts.back().IsExpression = true;
833 } else if (Current.is(TT_TrailingReturnArrow)) {
834 Contexts.back().IsExpression = false;
835 } else if (Current.is(tok::l_paren) && !Line.MustBeDeclaration &&
836 !Line.InPPDirective &&
837 (!Current.Previous ||
838 Current.Previous->isNot(tok::kw_decltype))) {
839 bool ParametersOfFunctionType =
840 Current.Previous && Current.Previous->is(tok::r_paren) &&
841 Current.Previous->MatchingParen &&
842 Current.Previous->MatchingParen->is(TT_FunctionTypeLParen);
843 bool IsForOrCatch = Current.Previous &&
844 Current.Previous->isOneOf(tok::kw_for, tok::kw_catch);
845 Contexts.back().IsExpression = !ParametersOfFunctionType && !IsForOrCatch;
846 } else if (Current.isOneOf(tok::r_paren, tok::greater, tok::comma)) {
847 for (FormatToken *Previous = Current.Previous;
848 Previous && Previous->isOneOf(tok::star, tok::amp);
849 Previous = Previous->Previous)
850 Previous->Type = TT_PointerOrReference;
851 if (Line.MustBeDeclaration)
852 Contexts.back().IsExpression = Contexts.front().InCtorInitializer;
853 } else if (Current.Previous &&
854 Current.Previous->is(TT_CtorInitializerColon)) {
855 Contexts.back().IsExpression = true;
856 Contexts.back().InCtorInitializer = true;
857 } else if (Current.is(tok::kw_new)) {
858 Contexts.back().CanBeExpression = false;
859 } else if (Current.is(tok::semi) || Current.is(tok::exclaim)) {
860 // This should be the condition or increment in a for-loop.
861 Contexts.back().IsExpression = true;
865 void determineTokenType(FormatToken &Current) {
866 if (!Current.is(TT_Unknown))
867 // The token type is already known.
870 // Line.MightBeFunctionDecl can only be true after the parentheses of a
871 // function declaration have been found. In this case, 'Current' is a
872 // trailing token of this declaration and thus cannot be a name.
873 if (Current.is(Keywords.kw_instanceof)) {
874 Current.Type = TT_BinaryOperator;
875 } else if (isStartOfName(Current) &&
876 (!Line.MightBeFunctionDecl || Current.NestingLevel != 0)) {
877 Contexts.back().FirstStartOfName = &Current;
878 Current.Type = TT_StartOfName;
879 } else if (Current.is(tok::kw_auto)) {
881 } else if (Current.is(tok::arrow) &&
882 Style.Language == FormatStyle::LK_Java) {
883 Current.Type = TT_LambdaArrow;
884 } else if (Current.is(tok::arrow) && AutoFound && Line.MustBeDeclaration &&
885 Current.NestingLevel == 0) {
886 Current.Type = TT_TrailingReturnArrow;
887 } else if (Current.isOneOf(tok::star, tok::amp, tok::ampamp)) {
889 determineStarAmpUsage(Current, Contexts.back().CanBeExpression &&
890 Contexts.back().IsExpression,
891 Contexts.back().InTemplateArgument);
892 } else if (Current.isOneOf(tok::minus, tok::plus, tok::caret)) {
893 Current.Type = determinePlusMinusCaretUsage(Current);
894 if (Current.is(TT_UnaryOperator) && Current.is(tok::caret))
895 Contexts.back().CaretFound = true;
896 } else if (Current.isOneOf(tok::minusminus, tok::plusplus)) {
897 Current.Type = determineIncrementUsage(Current);
898 } else if (Current.isOneOf(tok::exclaim, tok::tilde)) {
899 Current.Type = TT_UnaryOperator;
900 } else if (Current.is(tok::question)) {
901 if (Style.Language == FormatStyle::LK_JavaScript &&
902 Line.MustBeDeclaration) {
903 // In JavaScript, `interface X { foo?(): bar; }` is an optional method
904 // on the interface, not a ternary expression.
905 Current.Type = TT_JsTypeOptionalQuestion;
907 Current.Type = TT_ConditionalExpr;
909 } else if (Current.isBinaryOperator() &&
910 (!Current.Previous || Current.Previous->isNot(tok::l_square))) {
911 Current.Type = TT_BinaryOperator;
912 } else if (Current.is(tok::comment)) {
913 if (Current.TokenText.startswith("/*")) {
914 if (Current.TokenText.endswith("*/"))
915 Current.Type = TT_BlockComment;
917 // The lexer has for some reason determined a comment here. But we
918 // cannot really handle it, if it isn't properly terminated.
919 Current.Tok.setKind(tok::unknown);
921 Current.Type = TT_LineComment;
923 } else if (Current.is(tok::r_paren)) {
924 if (rParenEndsCast(Current))
925 Current.Type = TT_CastRParen;
926 if (Current.MatchingParen && Current.Next &&
927 !Current.Next->isBinaryOperator() &&
928 !Current.Next->isOneOf(tok::semi, tok::colon, tok::l_brace))
929 if (FormatToken *BeforeParen = Current.MatchingParen->Previous)
930 if (BeforeParen->is(tok::identifier) &&
931 BeforeParen->TokenText == BeforeParen->TokenText.upper() &&
932 (!BeforeParen->Previous ||
933 BeforeParen->Previous->ClosesTemplateDeclaration))
934 Current.Type = TT_FunctionAnnotationRParen;
935 } else if (Current.is(tok::at) && Current.Next) {
936 if (Current.Next->isStringLiteral()) {
937 Current.Type = TT_ObjCStringLiteral;
939 switch (Current.Next->Tok.getObjCKeywordID()) {
940 case tok::objc_interface:
941 case tok::objc_implementation:
942 case tok::objc_protocol:
943 Current.Type = TT_ObjCDecl;
945 case tok::objc_property:
946 Current.Type = TT_ObjCProperty;
952 } else if (Current.is(tok::period)) {
953 FormatToken *PreviousNoComment = Current.getPreviousNonComment();
954 if (PreviousNoComment &&
955 PreviousNoComment->isOneOf(tok::comma, tok::l_brace))
956 Current.Type = TT_DesignatedInitializerPeriod;
957 else if (Style.Language == FormatStyle::LK_Java && Current.Previous &&
958 Current.Previous->isOneOf(TT_JavaAnnotation,
959 TT_LeadingJavaAnnotation)) {
960 Current.Type = Current.Previous->Type;
962 } else if (Current.isOneOf(tok::identifier, tok::kw_const) &&
964 !Current.Previous->isOneOf(tok::equal, tok::at) &&
965 Line.MightBeFunctionDecl && Contexts.size() == 1) {
966 // Line.MightBeFunctionDecl can only be true after the parentheses of a
967 // function declaration have been found.
968 Current.Type = TT_TrailingAnnotation;
969 } else if ((Style.Language == FormatStyle::LK_Java ||
970 Style.Language == FormatStyle::LK_JavaScript) &&
972 if (Current.Previous->is(tok::at) &&
973 Current.isNot(Keywords.kw_interface)) {
974 const FormatToken &AtToken = *Current.Previous;
975 const FormatToken *Previous = AtToken.getPreviousNonComment();
976 if (!Previous || Previous->is(TT_LeadingJavaAnnotation))
977 Current.Type = TT_LeadingJavaAnnotation;
979 Current.Type = TT_JavaAnnotation;
980 } else if (Current.Previous->is(tok::period) &&
981 Current.Previous->isOneOf(TT_JavaAnnotation,
982 TT_LeadingJavaAnnotation)) {
983 Current.Type = Current.Previous->Type;
988 /// \brief Take a guess at whether \p Tok starts a name of a function or
989 /// variable declaration.
991 /// This is a heuristic based on whether \p Tok is an identifier following
992 /// something that is likely a type.
993 bool isStartOfName(const FormatToken &Tok) {
994 if (Tok.isNot(tok::identifier) || !Tok.Previous)
997 if (Tok.Previous->is(TT_LeadingJavaAnnotation))
1000 // Skip "const" as it does not have an influence on whether this is a name.
1001 FormatToken *PreviousNotConst = Tok.Previous;
1002 while (PreviousNotConst && PreviousNotConst->is(tok::kw_const))
1003 PreviousNotConst = PreviousNotConst->Previous;
1005 if (!PreviousNotConst)
1008 bool IsPPKeyword = PreviousNotConst->is(tok::identifier) &&
1009 PreviousNotConst->Previous &&
1010 PreviousNotConst->Previous->is(tok::hash);
1012 if (PreviousNotConst->is(TT_TemplateCloser))
1013 return PreviousNotConst && PreviousNotConst->MatchingParen &&
1014 PreviousNotConst->MatchingParen->Previous &&
1015 PreviousNotConst->MatchingParen->Previous->isNot(tok::period) &&
1016 PreviousNotConst->MatchingParen->Previous->isNot(tok::kw_template);
1018 if (PreviousNotConst->is(tok::r_paren) && PreviousNotConst->MatchingParen &&
1019 PreviousNotConst->MatchingParen->Previous &&
1020 PreviousNotConst->MatchingParen->Previous->is(tok::kw_decltype))
1023 return (!IsPPKeyword && PreviousNotConst->is(tok::identifier)) ||
1024 PreviousNotConst->is(TT_PointerOrReference) ||
1025 PreviousNotConst->isSimpleTypeSpecifier();
1028 /// \brief Determine whether ')' is ending a cast.
1029 bool rParenEndsCast(const FormatToken &Tok) {
1030 FormatToken *LeftOfParens = nullptr;
1031 if (Tok.MatchingParen)
1032 LeftOfParens = Tok.MatchingParen->getPreviousNonComment();
1033 if (LeftOfParens && LeftOfParens->is(tok::r_paren) &&
1034 LeftOfParens->MatchingParen)
1035 LeftOfParens = LeftOfParens->MatchingParen->Previous;
1036 if (LeftOfParens && LeftOfParens->is(tok::r_square) &&
1037 LeftOfParens->MatchingParen &&
1038 LeftOfParens->MatchingParen->is(TT_LambdaLSquare))
1041 if (Tok.Next->is(tok::question))
1043 if (Style.Language == FormatStyle::LK_JavaScript &&
1044 Tok.Next->is(Keywords.kw_in))
1046 if (Style.Language == FormatStyle::LK_Java && Tok.Next->is(tok::l_paren))
1049 bool IsCast = false;
1050 bool ParensAreEmpty = Tok.Previous == Tok.MatchingParen;
1051 bool ParensAreType =
1053 Tok.Previous->isOneOf(TT_PointerOrReference, TT_TemplateCloser) ||
1054 Tok.Previous->isSimpleTypeSpecifier();
1055 bool ParensCouldEndDecl =
1056 Tok.Next && Tok.Next->isOneOf(tok::equal, tok::semi, tok::l_brace);
1057 bool IsSizeOfOrAlignOf =
1058 LeftOfParens && LeftOfParens->isOneOf(tok::kw_sizeof, tok::kw_alignof);
1059 if (ParensAreType && !ParensCouldEndDecl && !IsSizeOfOrAlignOf &&
1060 (Contexts.size() > 1 && Contexts[Contexts.size() - 2].IsExpression))
1062 else if (Tok.Next && Tok.Next->isNot(tok::string_literal) &&
1063 (Tok.Next->Tok.isLiteral() ||
1064 Tok.Next->isOneOf(tok::kw_sizeof, tok::kw_alignof)))
1066 // If there is an identifier after the (), it is likely a cast, unless
1067 // there is also an identifier before the ().
1068 else if (LeftOfParens && Tok.Next &&
1069 (LeftOfParens->Tok.getIdentifierInfo() == nullptr ||
1070 LeftOfParens->is(tok::kw_return)) &&
1071 !LeftOfParens->isOneOf(TT_OverloadedOperator, tok::at,
1072 TT_TemplateCloser)) {
1073 if (Tok.Next->isOneOf(tok::identifier, tok::numeric_constant)) {
1076 // Use heuristics to recognize c style casting.
1077 FormatToken *Prev = Tok.Previous;
1078 if (Prev && Prev->isOneOf(tok::amp, tok::star))
1079 Prev = Prev->Previous;
1081 if (Prev && Tok.Next && Tok.Next->Next) {
1082 bool NextIsUnary = Tok.Next->isUnaryOperator() ||
1083 Tok.Next->isOneOf(tok::amp, tok::star);
1085 NextIsUnary && !Tok.Next->is(tok::plus) &&
1086 Tok.Next->Next->isOneOf(tok::identifier, tok::numeric_constant);
1089 for (; Prev != Tok.MatchingParen; Prev = Prev->Previous) {
1091 !Prev->isOneOf(tok::kw_const, tok::identifier, tok::coloncolon)) {
1098 return IsCast && !ParensAreEmpty;
1101 /// \brief Return the type of the given token assuming it is * or &.
1102 TokenType determineStarAmpUsage(const FormatToken &Tok, bool IsExpression,
1103 bool InTemplateArgument) {
1104 if (Style.Language == FormatStyle::LK_JavaScript)
1105 return TT_BinaryOperator;
1107 const FormatToken *PrevToken = Tok.getPreviousNonComment();
1109 return TT_UnaryOperator;
1111 const FormatToken *NextToken = Tok.getNextNonComment();
1113 (NextToken->is(tok::l_brace) && !NextToken->getNextNonComment()))
1116 if (PrevToken->is(tok::coloncolon))
1117 return TT_PointerOrReference;
1119 if (PrevToken->isOneOf(tok::l_paren, tok::l_square, tok::l_brace,
1120 tok::comma, tok::semi, tok::kw_return, tok::colon,
1121 tok::equal, tok::kw_delete, tok::kw_sizeof) ||
1122 PrevToken->isOneOf(TT_BinaryOperator, TT_ConditionalExpr,
1123 TT_UnaryOperator, TT_CastRParen))
1124 return TT_UnaryOperator;
1126 if (NextToken->is(tok::l_square) && NextToken->isNot(TT_LambdaLSquare))
1127 return TT_PointerOrReference;
1128 if (NextToken->isOneOf(tok::kw_operator, tok::comma, tok::semi))
1129 return TT_PointerOrReference;
1131 if (PrevToken->is(tok::r_paren) && PrevToken->MatchingParen &&
1132 PrevToken->MatchingParen->Previous &&
1133 PrevToken->MatchingParen->Previous->isOneOf(tok::kw_typeof,
1135 return TT_PointerOrReference;
1137 if (PrevToken->Tok.isLiteral() ||
1138 PrevToken->isOneOf(tok::r_paren, tok::r_square, tok::kw_true,
1139 tok::kw_false, tok::r_brace) ||
1140 NextToken->Tok.isLiteral() ||
1141 NextToken->isOneOf(tok::kw_true, tok::kw_false) ||
1142 NextToken->isUnaryOperator() ||
1143 // If we know we're in a template argument, there are no named
1144 // declarations. Thus, having an identifier on the right-hand side
1145 // indicates a binary operator.
1146 (InTemplateArgument && NextToken->Tok.isAnyIdentifier()))
1147 return TT_BinaryOperator;
1149 // "&&(" is quite unlikely to be two successive unary "&".
1150 if (Tok.is(tok::ampamp) && NextToken && NextToken->is(tok::l_paren))
1151 return TT_BinaryOperator;
1153 // This catches some cases where evaluation order is used as control flow:
1155 const FormatToken *NextNextToken = NextToken->getNextNonComment();
1156 if (NextNextToken && NextNextToken->is(tok::arrow))
1157 return TT_BinaryOperator;
1159 // It is very unlikely that we are going to find a pointer or reference type
1160 // definition on the RHS of an assignment.
1161 if (IsExpression && !Contexts.back().CaretFound)
1162 return TT_BinaryOperator;
1164 return TT_PointerOrReference;
1167 TokenType determinePlusMinusCaretUsage(const FormatToken &Tok) {
1168 const FormatToken *PrevToken = Tok.getPreviousNonComment();
1169 if (!PrevToken || PrevToken->is(TT_CastRParen))
1170 return TT_UnaryOperator;
1172 // Use heuristics to recognize unary operators.
1173 if (PrevToken->isOneOf(tok::equal, tok::l_paren, tok::comma, tok::l_square,
1174 tok::question, tok::colon, tok::kw_return,
1175 tok::kw_case, tok::at, tok::l_brace))
1176 return TT_UnaryOperator;
1178 // There can't be two consecutive binary operators.
1179 if (PrevToken->is(TT_BinaryOperator))
1180 return TT_UnaryOperator;
1182 // Fall back to marking the token as binary operator.
1183 return TT_BinaryOperator;
1186 /// \brief Determine whether ++/-- are pre- or post-increments/-decrements.
1187 TokenType determineIncrementUsage(const FormatToken &Tok) {
1188 const FormatToken *PrevToken = Tok.getPreviousNonComment();
1189 if (!PrevToken || PrevToken->is(TT_CastRParen))
1190 return TT_UnaryOperator;
1191 if (PrevToken->isOneOf(tok::r_paren, tok::r_square, tok::identifier))
1192 return TT_TrailingUnaryOperator;
1194 return TT_UnaryOperator;
1197 SmallVector<Context, 8> Contexts;
1199 const FormatStyle &Style;
1200 AnnotatedLine &Line;
1201 FormatToken *CurrentToken;
1203 const AdditionalKeywords &Keywords;
1205 // Set of "<" tokens that do not open a template parameter list. If parseAngle
1206 // determines that a specific token can't be a template opener, it will make
1207 // same decision irrespective of the decisions for tokens leading up to it.
1208 // Store this information to prevent this from causing exponential runtime.
1209 llvm::SmallPtrSet<FormatToken *, 16> NonTemplateLess;
1212 static const int PrecedenceUnaryOperator = prec::PointerToMember + 1;
1213 static const int PrecedenceArrowAndPeriod = prec::PointerToMember + 2;
1215 /// \brief Parses binary expressions by inserting fake parenthesis based on
1216 /// operator precedence.
1217 class ExpressionParser {
1219 ExpressionParser(const FormatStyle &Style, const AdditionalKeywords &Keywords,
1220 AnnotatedLine &Line)
1221 : Style(Style), Keywords(Keywords), Current(Line.First) {}
1223 /// \brief Parse expressions with the given operatore precedence.
1224 void parse(int Precedence = 0) {
1225 // Skip 'return' and ObjC selector colons as they are not part of a binary
1227 while (Current && (Current->is(tok::kw_return) ||
1228 (Current->is(tok::colon) &&
1229 Current->isOneOf(TT_ObjCMethodExpr, TT_DictLiteral))))
1232 if (!Current || Precedence > PrecedenceArrowAndPeriod)
1235 // Conditional expressions need to be parsed separately for proper nesting.
1236 if (Precedence == prec::Conditional) {
1237 parseConditionalExpr();
1241 // Parse unary operators, which all have a higher precedence than binary
1243 if (Precedence == PrecedenceUnaryOperator) {
1244 parseUnaryOperator();
1248 FormatToken *Start = Current;
1249 FormatToken *LatestOperator = nullptr;
1250 unsigned OperatorIndex = 0;
1253 // Consume operators with higher precedence.
1254 parse(Precedence + 1);
1256 int CurrentPrecedence = getCurrentPrecedence();
1258 if (Current && Current->is(TT_SelectorName) &&
1259 Precedence == CurrentPrecedence) {
1261 addFakeParenthesis(Start, prec::Level(Precedence));
1265 // At the end of the line or when an operator with higher precedence is
1266 // found, insert fake parenthesis and return.
1267 if (!Current || (Current->closesScope() && Current->MatchingParen) ||
1268 (CurrentPrecedence != -1 && CurrentPrecedence < Precedence) ||
1269 (CurrentPrecedence == prec::Conditional &&
1270 Precedence == prec::Assignment && Current->is(tok::colon))) {
1274 // Consume scopes: (), [], <> and {}
1275 if (Current->opensScope()) {
1276 while (Current && !Current->closesScope()) {
1283 if (CurrentPrecedence == Precedence) {
1284 LatestOperator = Current;
1285 Current->OperatorIndex = OperatorIndex;
1288 next(/*SkipPastLeadingComments=*/Precedence > 0);
1292 if (LatestOperator && (Current || Precedence > 0)) {
1293 LatestOperator->LastOperator = true;
1294 if (Precedence == PrecedenceArrowAndPeriod) {
1295 // Call expressions don't have a binary operator precedence.
1296 addFakeParenthesis(Start, prec::Unknown);
1298 addFakeParenthesis(Start, prec::Level(Precedence));
1304 /// \brief Gets the precedence (+1) of the given token for binary operators
1305 /// and other tokens that we treat like binary operators.
1306 int getCurrentPrecedence() {
1308 const FormatToken *NextNonComment = Current->getNextNonComment();
1309 if (Current->is(TT_ConditionalExpr))
1310 return prec::Conditional;
1311 else if (NextNonComment && NextNonComment->is(tok::colon) &&
1312 NextNonComment->is(TT_DictLiteral))
1314 else if (Current->is(TT_LambdaArrow))
1316 else if (Current->isOneOf(tok::semi, TT_InlineASMColon,
1318 (Current->is(tok::comment) && NextNonComment &&
1319 NextNonComment->is(TT_SelectorName)))
1321 else if (Current->is(TT_RangeBasedForLoopColon))
1323 else if (Current->is(TT_BinaryOperator) || Current->is(tok::comma))
1324 return Current->getPrecedence();
1325 else if (Current->isOneOf(tok::period, tok::arrow))
1326 return PrecedenceArrowAndPeriod;
1327 else if (Style.Language == FormatStyle::LK_Java &&
1328 Current->isOneOf(Keywords.kw_extends, Keywords.kw_implements,
1329 Keywords.kw_throws))
1335 void addFakeParenthesis(FormatToken *Start, prec::Level Precedence) {
1336 Start->FakeLParens.push_back(Precedence);
1337 if (Precedence > prec::Unknown)
1338 Start->StartsBinaryExpression = true;
1340 FormatToken *Previous = Current->Previous;
1341 while (Previous->is(tok::comment) && Previous->Previous)
1342 Previous = Previous->Previous;
1343 ++Previous->FakeRParens;
1344 if (Precedence > prec::Unknown)
1345 Previous->EndsBinaryExpression = true;
1349 /// \brief Parse unary operator expressions and surround them with fake
1350 /// parentheses if appropriate.
1351 void parseUnaryOperator() {
1352 if (!Current || Current->isNot(TT_UnaryOperator)) {
1353 parse(PrecedenceArrowAndPeriod);
1357 FormatToken *Start = Current;
1359 parseUnaryOperator();
1361 // The actual precedence doesn't matter.
1362 addFakeParenthesis(Start, prec::Unknown);
1365 void parseConditionalExpr() {
1366 while (Current && Current->isTrailingComment()) {
1369 FormatToken *Start = Current;
1370 parse(prec::LogicalOr);
1371 if (!Current || !Current->is(tok::question))
1374 parse(prec::Assignment);
1375 if (!Current || Current->isNot(TT_ConditionalExpr))
1378 parse(prec::Assignment);
1379 addFakeParenthesis(Start, prec::Conditional);
1382 void next(bool SkipPastLeadingComments = true) {
1384 Current = Current->Next;
1386 (Current->NewlinesBefore == 0 || SkipPastLeadingComments) &&
1387 Current->isTrailingComment())
1388 Current = Current->Next;
1391 const FormatStyle &Style;
1392 const AdditionalKeywords &Keywords;
1393 FormatToken *Current;
1396 } // end anonymous namespace
1398 void TokenAnnotator::setCommentLineLevels(
1399 SmallVectorImpl<AnnotatedLine *> &Lines) {
1400 const AnnotatedLine *NextNonCommentLine = nullptr;
1401 for (SmallVectorImpl<AnnotatedLine *>::reverse_iterator I = Lines.rbegin(),
1404 if (NextNonCommentLine && (*I)->First->is(tok::comment) &&
1405 (*I)->First->Next == nullptr)
1406 (*I)->Level = NextNonCommentLine->Level;
1408 NextNonCommentLine = (*I)->First->isNot(tok::r_brace) ? (*I) : nullptr;
1410 setCommentLineLevels((*I)->Children);
1414 void TokenAnnotator::annotate(AnnotatedLine &Line) {
1415 for (SmallVectorImpl<AnnotatedLine *>::iterator I = Line.Children.begin(),
1416 E = Line.Children.end();
1420 AnnotatingParser Parser(Style, Line, Keywords);
1421 Line.Type = Parser.parseLine();
1422 if (Line.Type == LT_Invalid)
1425 ExpressionParser ExprParser(Style, Keywords, Line);
1428 if (Line.First->is(TT_ObjCMethodSpecifier))
1429 Line.Type = LT_ObjCMethodDecl;
1430 else if (Line.First->is(TT_ObjCDecl))
1431 Line.Type = LT_ObjCDecl;
1432 else if (Line.First->is(TT_ObjCProperty))
1433 Line.Type = LT_ObjCProperty;
1435 Line.First->SpacesRequiredBefore = 1;
1436 Line.First->CanBreakBefore = Line.First->MustBreakBefore;
1439 // This function heuristically determines whether 'Current' starts the name of a
1440 // function declaration.
1441 static bool isFunctionDeclarationName(const FormatToken &Current) {
1442 if (!Current.is(TT_StartOfName) || Current.NestingLevel != 0)
1444 const FormatToken *Next = Current.Next;
1445 for (; Next; Next = Next->Next) {
1446 if (Next->is(TT_TemplateOpener)) {
1447 Next = Next->MatchingParen;
1448 } else if (Next->is(tok::coloncolon)) {
1450 if (!Next || !Next->is(tok::identifier))
1452 } else if (Next->is(tok::l_paren)) {
1460 assert(Next->is(tok::l_paren));
1461 if (Next->Next == Next->MatchingParen)
1463 for (const FormatToken *Tok = Next->Next; Tok && Tok != Next->MatchingParen;
1465 if (Tok->is(tok::kw_const) || Tok->isSimpleTypeSpecifier() ||
1466 Tok->isOneOf(TT_PointerOrReference, TT_StartOfName))
1468 if (Tok->isOneOf(tok::l_brace, tok::string_literal, TT_ObjCMethodExpr) ||
1469 Tok->Tok.isLiteral())
1475 void TokenAnnotator::calculateFormattingInformation(AnnotatedLine &Line) {
1476 for (SmallVectorImpl<AnnotatedLine *>::iterator I = Line.Children.begin(),
1477 E = Line.Children.end();
1479 calculateFormattingInformation(**I);
1482 Line.First->TotalLength =
1483 Line.First->IsMultiline ? Style.ColumnLimit : Line.First->ColumnWidth;
1484 if (!Line.First->Next)
1486 FormatToken *Current = Line.First->Next;
1487 bool InFunctionDecl = Line.MightBeFunctionDecl;
1489 if (isFunctionDeclarationName(*Current))
1490 Current->Type = TT_FunctionDeclarationName;
1491 if (Current->is(TT_LineComment)) {
1492 if (Current->Previous->BlockKind == BK_BracedInit &&
1493 Current->Previous->opensScope())
1494 Current->SpacesRequiredBefore = Style.Cpp11BracedListStyle ? 0 : 1;
1496 Current->SpacesRequiredBefore = Style.SpacesBeforeTrailingComments;
1498 // If we find a trailing comment, iterate backwards to determine whether
1499 // it seems to relate to a specific parameter. If so, break before that
1500 // parameter to avoid changing the comment's meaning. E.g. don't move 'b'
1501 // to the previous line in:
1505 if (!Current->HasUnescapedNewline) {
1506 for (FormatToken *Parameter = Current->Previous; Parameter;
1507 Parameter = Parameter->Previous) {
1508 if (Parameter->isOneOf(tok::comment, tok::r_brace))
1510 if (Parameter->Previous && Parameter->Previous->is(tok::comma)) {
1511 if (!Parameter->Previous->is(TT_CtorInitializerComma) &&
1512 Parameter->HasUnescapedNewline)
1513 Parameter->MustBreakBefore = true;
1518 } else if (Current->SpacesRequiredBefore == 0 &&
1519 spaceRequiredBefore(Line, *Current)) {
1520 Current->SpacesRequiredBefore = 1;
1523 Current->MustBreakBefore =
1524 Current->MustBreakBefore || mustBreakBefore(Line, *Current);
1526 if (Style.AlwaysBreakAfterDefinitionReturnType && InFunctionDecl &&
1527 Current->is(TT_FunctionDeclarationName) &&
1528 !Line.Last->isOneOf(tok::semi, tok::comment)) // Only for definitions.
1529 // FIXME: Line.Last points to other characters than tok::semi
1531 Current->MustBreakBefore = true;
1533 Current->CanBreakBefore =
1534 Current->MustBreakBefore || canBreakBefore(Line, *Current);
1535 unsigned ChildSize = 0;
1536 if (Current->Previous->Children.size() == 1) {
1537 FormatToken &LastOfChild = *Current->Previous->Children[0]->Last;
1538 ChildSize = LastOfChild.isTrailingComment() ? Style.ColumnLimit
1539 : LastOfChild.TotalLength + 1;
1541 const FormatToken *Prev = Current->Previous;
1542 if (Current->MustBreakBefore || Prev->Children.size() > 1 ||
1543 (Prev->Children.size() == 1 &&
1544 Prev->Children[0]->First->MustBreakBefore) ||
1545 Current->IsMultiline)
1546 Current->TotalLength = Prev->TotalLength + Style.ColumnLimit;
1548 Current->TotalLength = Prev->TotalLength + Current->ColumnWidth +
1549 ChildSize + Current->SpacesRequiredBefore;
1551 if (Current->is(TT_CtorInitializerColon))
1552 InFunctionDecl = false;
1554 // FIXME: Only calculate this if CanBreakBefore is true once static
1555 // initializers etc. are sorted out.
1556 // FIXME: Move magic numbers to a better place.
1557 Current->SplitPenalty = 20 * Current->BindingStrength +
1558 splitPenalty(Line, *Current, InFunctionDecl);
1560 Current = Current->Next;
1563 calculateUnbreakableTailLengths(Line);
1564 for (Current = Line.First; Current != nullptr; Current = Current->Next) {
1566 Current->Role->precomputeFormattingInfos(Current);
1569 DEBUG({ printDebugInfo(Line); });
1572 void TokenAnnotator::calculateUnbreakableTailLengths(AnnotatedLine &Line) {
1573 unsigned UnbreakableTailLength = 0;
1574 FormatToken *Current = Line.Last;
1576 Current->UnbreakableTailLength = UnbreakableTailLength;
1577 if (Current->CanBreakBefore ||
1578 Current->isOneOf(tok::comment, tok::string_literal)) {
1579 UnbreakableTailLength = 0;
1581 UnbreakableTailLength +=
1582 Current->ColumnWidth + Current->SpacesRequiredBefore;
1584 Current = Current->Previous;
1588 unsigned TokenAnnotator::splitPenalty(const AnnotatedLine &Line,
1589 const FormatToken &Tok,
1590 bool InFunctionDecl) {
1591 const FormatToken &Left = *Tok.Previous;
1592 const FormatToken &Right = Tok;
1594 if (Left.is(tok::semi))
1597 if (Style.Language == FormatStyle::LK_Java) {
1598 if (Right.isOneOf(Keywords.kw_extends, Keywords.kw_throws))
1600 if (Right.is(Keywords.kw_implements))
1602 if (Left.is(tok::comma) && Left.NestingLevel == 0)
1604 } else if (Style.Language == FormatStyle::LK_JavaScript) {
1605 if (Right.is(Keywords.kw_function) && Left.isNot(tok::comma))
1609 if (Left.is(tok::comma) || (Right.is(tok::identifier) && Right.Next &&
1610 Right.Next->is(TT_DictLiteral)))
1612 if (Right.is(tok::l_square)) {
1613 if (Style.Language == FormatStyle::LK_Proto)
1615 // Slightly prefer formatting local lambda definitions like functions.
1616 if (Right.is(TT_LambdaLSquare) && Left.is(tok::equal))
1618 if (!Right.isOneOf(TT_ObjCMethodExpr, TT_LambdaLSquare))
1622 if (Right.isOneOf(TT_StartOfName, TT_FunctionDeclarationName) ||
1623 Right.is(tok::kw_operator)) {
1624 if (Line.First->is(tok::kw_for) && Right.PartOfMultiVariableDeclStmt)
1626 if (Left.is(TT_StartOfName))
1628 if (InFunctionDecl && Right.NestingLevel == 0)
1629 return Style.PenaltyReturnTypeOnItsOwnLine;
1632 if (Right.is(TT_PointerOrReference))
1634 if (Right.is(TT_TrailingReturnArrow))
1636 if (Left.is(tok::equal) && Right.is(tok::l_brace))
1638 if (Left.is(TT_CastRParen))
1640 if (Left.is(tok::coloncolon) ||
1641 (Right.is(tok::period) && Style.Language == FormatStyle::LK_Proto))
1643 if (Left.isOneOf(tok::kw_class, tok::kw_struct))
1646 if (Left.isOneOf(TT_RangeBasedForLoopColon, TT_InheritanceColon))
1649 if (Right.isMemberAccess()) {
1650 if (Left.is(tok::r_paren) && Left.MatchingParen &&
1651 Left.MatchingParen->ParameterCount > 0)
1652 return 20; // Should be smaller than breaking at a nested comma.
1656 if (Right.is(TT_TrailingAnnotation) &&
1657 (!Right.Next || Right.Next->isNot(tok::l_paren))) {
1658 // Moving trailing annotations to the next line is fine for ObjC method
1660 if (Line.First->is(TT_ObjCMethodSpecifier))
1663 // Generally, breaking before a trailing annotation is bad unless it is
1664 // function-like. It seems to be especially preferable to keep standard
1665 // annotations (i.e. "const", "final" and "override") on the same line.
1666 // Use a slightly higher penalty after ")" so that annotations like
1667 // "const override" are kept together.
1668 bool is_short_annotation = Right.TokenText.size() < 10;
1669 return (Left.is(tok::r_paren) ? 100 : 120) + (is_short_annotation ? 50 : 0);
1672 // In for-loops, prefer breaking at ',' and ';'.
1673 if (Line.First->is(tok::kw_for) && Left.is(tok::equal))
1676 // In Objective-C method expressions, prefer breaking before "param:" over
1677 // breaking after it.
1678 if (Right.is(TT_SelectorName))
1680 if (Left.is(tok::colon) && Left.is(TT_ObjCMethodExpr))
1681 return Line.MightBeFunctionDecl ? 50 : 500;
1683 if (Left.is(tok::l_paren) && InFunctionDecl && Style.AlignAfterOpenBracket)
1685 if (Left.is(tok::l_paren) && Left.Previous && Left.Previous->is(tok::kw_if))
1687 if (Left.is(tok::equal) && InFunctionDecl)
1689 if (Right.is(tok::r_brace))
1691 if (Left.is(TT_TemplateOpener))
1693 if (Left.opensScope()) {
1694 if (!Style.AlignAfterOpenBracket)
1696 return Left.ParameterCount > 1 ? Style.PenaltyBreakBeforeFirstCallParameter
1699 if (Left.is(TT_JavaAnnotation))
1702 if (Right.is(tok::lessless)) {
1703 if (Left.is(tok::string_literal) &&
1704 (!Right.LastOperator || Right.OperatorIndex != 1)) {
1705 StringRef Content = Left.TokenText;
1706 if (Content.startswith("\""))
1707 Content = Content.drop_front(1);
1708 if (Content.endswith("\""))
1709 Content = Content.drop_back(1);
1710 Content = Content.trim();
1711 if (Content.size() > 1 &&
1712 (Content.back() == ':' || Content.back() == '='))
1715 return 1; // Breaking at a << is really cheap.
1717 if (Left.is(TT_ConditionalExpr))
1718 return prec::Conditional;
1719 prec::Level Level = Left.getPrecedence();
1720 if (Level != prec::Unknown)
1722 Level = Right.getPrecedence();
1723 if (Level != prec::Unknown)
1729 bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
1730 const FormatToken &Left,
1731 const FormatToken &Right) {
1732 if (Left.is(tok::kw_return) && Right.isNot(tok::semi))
1734 if (Style.ObjCSpaceAfterProperty && Line.Type == LT_ObjCProperty &&
1735 Left.Tok.getObjCKeywordID() == tok::objc_property)
1737 if (Right.is(tok::hashhash))
1738 return Left.is(tok::hash);
1739 if (Left.isOneOf(tok::hashhash, tok::hash))
1740 return Right.is(tok::hash);
1741 if (Left.is(tok::l_paren) && Right.is(tok::r_paren))
1742 return Style.SpaceInEmptyParentheses;
1743 if (Left.is(tok::l_paren) || Right.is(tok::r_paren))
1744 return (Right.is(TT_CastRParen) ||
1745 (Left.MatchingParen && Left.MatchingParen->is(TT_CastRParen)))
1746 ? Style.SpacesInCStyleCastParentheses
1747 : Style.SpacesInParentheses;
1748 if (Right.isOneOf(tok::semi, tok::comma))
1750 if (Right.is(tok::less) &&
1751 (Left.is(tok::kw_template) ||
1752 (Line.Type == LT_ObjCDecl && Style.ObjCSpaceBeforeProtocolList)))
1754 if (Left.isOneOf(tok::exclaim, tok::tilde))
1756 if (Left.is(tok::at) &&
1757 Right.isOneOf(tok::identifier, tok::string_literal, tok::char_constant,
1758 tok::numeric_constant, tok::l_paren, tok::l_brace,
1759 tok::kw_true, tok::kw_false))
1761 if (Left.is(tok::coloncolon))
1763 if (Left.is(tok::less) || Right.isOneOf(tok::greater, tok::less))
1765 if (Right.is(tok::ellipsis))
1766 return Left.Tok.isLiteral();
1767 if (Left.is(tok::l_square) && Right.is(tok::amp))
1769 if (Right.is(TT_PointerOrReference))
1770 return !(Left.is(tok::r_paren) && Left.MatchingParen &&
1771 (Left.MatchingParen->is(TT_OverloadedOperatorLParen) ||
1772 (Left.MatchingParen->Previous &&
1773 Left.MatchingParen->Previous->is(
1774 TT_FunctionDeclarationName)))) &&
1775 (Left.Tok.isLiteral() ||
1776 (!Left.isOneOf(TT_PointerOrReference, tok::l_paren) &&
1777 (Style.PointerAlignment != FormatStyle::PAS_Left ||
1778 Line.IsMultiVariableDeclStmt)));
1779 if (Right.is(TT_FunctionTypeLParen) && Left.isNot(tok::l_paren) &&
1780 (!Left.is(TT_PointerOrReference) ||
1781 (Style.PointerAlignment != FormatStyle::PAS_Right &&
1782 !Line.IsMultiVariableDeclStmt)))
1784 if (Left.is(TT_PointerOrReference))
1785 return Right.Tok.isLiteral() || Right.is(TT_BlockComment) ||
1786 (!Right.isOneOf(TT_PointerOrReference, TT_ArraySubscriptLSquare,
1788 (Style.PointerAlignment != FormatStyle::PAS_Right &&
1789 !Line.IsMultiVariableDeclStmt) &&
1791 !Left.Previous->isOneOf(tok::l_paren, tok::coloncolon));
1792 if (Right.is(tok::star) && Left.is(tok::l_paren))
1794 if (Left.is(tok::l_square))
1795 return (Left.is(TT_ArrayInitializerLSquare) &&
1796 Style.SpacesInContainerLiterals && Right.isNot(tok::r_square)) ||
1797 (Left.is(TT_ArraySubscriptLSquare) && Style.SpacesInSquareBrackets &&
1798 Right.isNot(tok::r_square));
1799 if (Right.is(tok::r_square))
1800 return Right.MatchingParen &&
1801 ((Style.SpacesInContainerLiterals &&
1802 Right.MatchingParen->is(TT_ArrayInitializerLSquare)) ||
1803 (Style.SpacesInSquareBrackets &&
1804 Right.MatchingParen->is(TT_ArraySubscriptLSquare)));
1805 if (Right.is(tok::l_square) &&
1806 !Right.isOneOf(TT_ObjCMethodExpr, TT_LambdaLSquare) &&
1807 !Left.isOneOf(tok::numeric_constant, TT_DictLiteral))
1809 if (Left.is(tok::colon))
1810 return !Left.is(TT_ObjCMethodExpr);
1811 if (Left.is(tok::l_brace) && Right.is(tok::r_brace))
1812 return !Left.Children.empty(); // No spaces in "{}".
1813 if ((Left.is(tok::l_brace) && Left.BlockKind != BK_Block) ||
1814 (Right.is(tok::r_brace) && Right.MatchingParen &&
1815 Right.MatchingParen->BlockKind != BK_Block))
1816 return !Style.Cpp11BracedListStyle;
1817 if (Left.is(TT_BlockComment))
1818 return !Left.TokenText.endswith("=*/");
1819 if (Right.is(tok::l_paren)) {
1820 if (Left.is(tok::r_paren) && Left.is(TT_AttributeParen))
1822 return Line.Type == LT_ObjCDecl || Left.is(tok::semi) ||
1823 (Style.SpaceBeforeParens != FormatStyle::SBPO_Never &&
1824 (Left.isOneOf(tok::kw_if, tok::kw_for, tok::kw_while,
1825 tok::kw_switch, tok::kw_case, TT_ForEachMacro) ||
1826 (Left.isOneOf(tok::kw_try, Keywords.kw___except, tok::kw_catch,
1827 tok::kw_new, tok::kw_delete) &&
1828 (!Left.Previous || Left.Previous->isNot(tok::period))))) ||
1829 (Style.SpaceBeforeParens == FormatStyle::SBPO_Always &&
1830 (Left.is(tok::identifier) || Left.isFunctionLikeKeyword() || Left.is(tok::r_paren)) &&
1831 Line.Type != LT_PreprocessorDirective);
1833 if (Left.is(tok::at) && Right.Tok.getObjCKeywordID() != tok::objc_not_keyword)
1835 if (Right.is(TT_UnaryOperator))
1836 return !Left.isOneOf(tok::l_paren, tok::l_square, tok::at) &&
1837 (Left.isNot(tok::colon) || Left.isNot(TT_ObjCMethodExpr));
1838 if ((Left.isOneOf(tok::identifier, tok::greater, tok::r_square,
1840 Left.isSimpleTypeSpecifier()) &&
1841 Right.is(tok::l_brace) && Right.getNextNonComment() &&
1842 Right.BlockKind != BK_Block)
1844 if (Left.is(tok::period) || Right.is(tok::period))
1846 if (Right.is(tok::hash) && Left.is(tok::identifier) && Left.TokenText == "L")
1848 if (Left.is(TT_TemplateCloser) && Left.MatchingParen &&
1849 Left.MatchingParen->Previous &&
1850 Left.MatchingParen->Previous->is(tok::period))
1851 // A.<B>DoSomething();
1853 if (Left.is(TT_TemplateCloser) && Right.is(tok::l_square))
1858 bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line,
1859 const FormatToken &Right) {
1860 const FormatToken &Left = *Right.Previous;
1861 if (Style.Language == FormatStyle::LK_Proto) {
1862 if (Right.is(tok::period) &&
1863 Left.isOneOf(Keywords.kw_optional, Keywords.kw_required,
1864 Keywords.kw_repeated))
1866 if (Right.is(tok::l_paren) &&
1867 Left.isOneOf(Keywords.kw_returns, Keywords.kw_option))
1869 } else if (Style.Language == FormatStyle::LK_JavaScript) {
1870 if (Left.is(Keywords.kw_var))
1872 if (Right.isOneOf(TT_JsTypeColon, TT_JsTypeOptionalQuestion))
1874 if ((Left.is(tok::l_brace) || Right.is(tok::r_brace)) &&
1875 Line.First->isOneOf(Keywords.kw_import, tok::kw_export))
1877 if (Left.is(tok::ellipsis))
1879 if (Left.is(TT_TemplateCloser) &&
1880 !Right.isOneOf(tok::equal, tok::l_brace, tok::comma, tok::l_square,
1881 Keywords.kw_implements, Keywords.kw_extends))
1882 // Type assertions ('<type>expr') are not followed by whitespace. Other
1883 // locations that should have whitespace following are identified by the
1884 // above set of follower tokens.
1886 } else if (Style.Language == FormatStyle::LK_Java) {
1887 if (Left.is(tok::r_square) && Right.is(tok::l_brace))
1889 if (Left.is(TT_LambdaArrow) || Right.is(TT_LambdaArrow))
1891 if (Left.is(Keywords.kw_synchronized) && Right.is(tok::l_paren))
1892 return Style.SpaceBeforeParens != FormatStyle::SBPO_Never;
1893 if ((Left.isOneOf(tok::kw_static, tok::kw_public, tok::kw_private,
1894 tok::kw_protected) ||
1895 Left.isOneOf(Keywords.kw_final, Keywords.kw_abstract,
1896 Keywords.kw_native)) &&
1897 Right.is(TT_TemplateOpener))
1900 if (Right.Tok.getIdentifierInfo() && Left.Tok.getIdentifierInfo())
1901 return true; // Never ever merge two identifiers.
1902 if (Left.is(TT_ImplicitStringLiteral))
1903 return Right.WhitespaceRange.getBegin() != Right.WhitespaceRange.getEnd();
1904 if (Line.Type == LT_ObjCMethodDecl) {
1905 if (Left.is(TT_ObjCMethodSpecifier))
1907 if (Left.is(tok::r_paren) && Right.is(tok::identifier))
1908 // Don't space between ')' and <id>
1911 if (Line.Type == LT_ObjCProperty &&
1912 (Right.is(tok::equal) || Left.is(tok::equal)))
1915 if (Right.is(TT_TrailingReturnArrow) || Left.is(TT_TrailingReturnArrow))
1917 if (Left.is(tok::comma))
1919 if (Right.is(tok::comma))
1921 if (Right.isOneOf(TT_CtorInitializerColon, TT_ObjCBlockLParen))
1923 if (Left.is(tok::kw_operator))
1924 return Right.is(tok::coloncolon);
1925 if (Right.is(TT_OverloadedOperatorLParen))
1927 if (Right.is(tok::colon)) {
1928 if (Line.First->isOneOf(tok::kw_case, tok::kw_default) ||
1929 !Right.getNextNonComment() || Right.getNextNonComment()->is(tok::semi))
1931 if (Right.is(TT_ObjCMethodExpr))
1933 if (Left.is(tok::question))
1935 if (Right.is(TT_InlineASMColon) && Left.is(tok::coloncolon))
1937 if (Right.is(TT_DictLiteral))
1938 return Style.SpacesInContainerLiterals;
1941 if (Left.is(TT_UnaryOperator))
1942 return Right.is(TT_BinaryOperator);
1944 // If the next token is a binary operator or a selector name, we have
1945 // incorrectly classified the parenthesis as a cast. FIXME: Detect correctly.
1946 if (Left.is(TT_CastRParen))
1947 return Style.SpaceAfterCStyleCast ||
1948 Right.isOneOf(TT_BinaryOperator, TT_SelectorName);
1950 if (Left.is(tok::greater) && Right.is(tok::greater)) {
1951 return Right.is(TT_TemplateCloser) && Left.is(TT_TemplateCloser) &&
1952 (Style.Standard != FormatStyle::LS_Cpp11 || Style.SpacesInAngles);
1954 if (Right.isOneOf(tok::arrow, tok::period, tok::arrowstar, tok::periodstar) ||
1955 Left.isOneOf(tok::arrow, tok::period, tok::arrowstar, tok::periodstar))
1957 if (!Style.SpaceBeforeAssignmentOperators &&
1958 Right.getPrecedence() == prec::Assignment)
1960 if (Right.is(tok::coloncolon) && Left.isNot(tok::l_brace))
1961 return (Left.is(TT_TemplateOpener) &&
1962 Style.Standard == FormatStyle::LS_Cpp03) ||
1963 !(Left.isOneOf(tok::identifier, tok::l_paren, tok::r_paren) ||
1964 Left.isOneOf(TT_TemplateCloser, TT_TemplateOpener));
1965 if ((Left.is(TT_TemplateOpener)) != (Right.is(TT_TemplateCloser)))
1966 return Style.SpacesInAngles;
1967 if ((Right.is(TT_BinaryOperator) && !Left.is(tok::l_paren)) ||
1968 Left.isOneOf(TT_BinaryOperator, TT_ConditionalExpr))
1970 if (Left.is(TT_TemplateCloser) && Right.is(tok::l_paren) &&
1971 Right.isNot(TT_FunctionTypeLParen))
1972 return Style.SpaceBeforeParens == FormatStyle::SBPO_Always;
1973 if (Right.is(TT_TemplateOpener) && Left.is(tok::r_paren) &&
1974 Left.MatchingParen && Left.MatchingParen->is(TT_OverloadedOperatorLParen))
1976 if (Right.is(tok::less) && Left.isNot(tok::l_paren) &&
1977 Line.First->is(tok::hash))
1979 if (Right.is(TT_TrailingUnaryOperator))
1981 if (Left.is(TT_RegexLiteral))
1983 return spaceRequiredBetween(Line, Left, Right);
1986 // Returns 'true' if 'Tok' is a brace we'd want to break before in Allman style.
1987 static bool isAllmanBrace(const FormatToken &Tok) {
1988 return Tok.is(tok::l_brace) && Tok.BlockKind == BK_Block &&
1989 !Tok.isOneOf(TT_ObjCBlockLBrace, TT_DictLiteral);
1992 bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line,
1993 const FormatToken &Right) {
1994 const FormatToken &Left = *Right.Previous;
1995 if (Right.NewlinesBefore > 1)
1998 // If the last token before a '}' is a comma or a trailing comment, the
1999 // intention is to insert a line break after it in order to make shuffling
2000 // around entries easier.
2001 const FormatToken *BeforeClosingBrace = nullptr;
2002 if (Left.isOneOf(tok::l_brace, TT_ArrayInitializerLSquare) &&
2003 Left.BlockKind != BK_Block && Left.MatchingParen)
2004 BeforeClosingBrace = Left.MatchingParen->Previous;
2005 else if (Right.MatchingParen &&
2006 Right.MatchingParen->isOneOf(tok::l_brace,
2007 TT_ArrayInitializerLSquare))
2008 BeforeClosingBrace = &Left;
2009 if (BeforeClosingBrace && (BeforeClosingBrace->is(tok::comma) ||
2010 BeforeClosingBrace->isTrailingComment()))
2013 if (Right.is(tok::comment))
2014 return Left.BlockKind != BK_BracedInit &&
2015 Left.isNot(TT_CtorInitializerColon) &&
2016 (Right.NewlinesBefore > 0 && Right.HasUnescapedNewline);
2017 if (Left.isTrailingComment())
2019 if (Left.isStringLiteral() &&
2020 (Right.isStringLiteral() || Right.is(TT_ObjCStringLiteral)))
2022 if (Right.Previous->IsUnterminatedLiteral)
2024 if (Right.is(tok::lessless) && Right.Next &&
2025 Right.Previous->is(tok::string_literal) &&
2026 Right.Next->is(tok::string_literal))
2028 if (Right.Previous->ClosesTemplateDeclaration &&
2029 Right.Previous->MatchingParen &&
2030 Right.Previous->MatchingParen->NestingLevel == 0 &&
2031 Style.AlwaysBreakTemplateDeclarations)
2033 if ((Right.isOneOf(TT_CtorInitializerComma, TT_CtorInitializerColon)) &&
2034 Style.BreakConstructorInitializersBeforeComma &&
2035 !Style.ConstructorInitializerAllOnOneLineOrOnePerLine)
2037 if (Right.is(tok::string_literal) && Right.TokenText.startswith("R\""))
2038 // Raw string literals are special wrt. line breaks. The author has made a
2039 // deliberate choice and might have aligned the contents of the string
2040 // literal accordingly. Thus, we try keep existing line breaks.
2041 return Right.NewlinesBefore > 0;
2042 if (Right.Previous->is(tok::l_brace) && Right.NestingLevel == 1 &&
2043 Style.Language == FormatStyle::LK_Proto)
2044 // Don't put enums onto single lines in protocol buffers.
2046 if (Right.is(TT_InlineASMBrace))
2047 return Right.HasUnescapedNewline;
2048 if (Style.Language == FormatStyle::LK_JavaScript && Right.is(tok::r_brace) &&
2049 Left.is(tok::l_brace) && !Left.Children.empty())
2050 // Support AllowShortFunctionsOnASingleLine for JavaScript.
2051 return Style.AllowShortFunctionsOnASingleLine == FormatStyle::SFS_None ||
2052 (Left.NestingLevel == 0 && Line.Level == 0 &&
2053 Style.AllowShortFunctionsOnASingleLine == FormatStyle::SFS_Inline);
2054 if (isAllmanBrace(Left) || isAllmanBrace(Right))
2055 return Style.BreakBeforeBraces == FormatStyle::BS_Allman ||
2056 Style.BreakBeforeBraces == FormatStyle::BS_GNU;
2057 if (Style.Language == FormatStyle::LK_Proto && Left.isNot(tok::l_brace) &&
2058 Right.is(TT_SelectorName))
2060 if (Left.is(TT_ObjCBlockLBrace) && !Style.AllowShortBlocksOnASingleLine)
2063 if ((Style.Language == FormatStyle::LK_Java ||
2064 Style.Language == FormatStyle::LK_JavaScript) &&
2065 Left.is(TT_LeadingJavaAnnotation) &&
2066 Right.isNot(TT_LeadingJavaAnnotation) && Right.isNot(tok::l_paren) &&
2067 Line.Last->is(tok::l_brace))
2070 if (Style.Language == FormatStyle::LK_JavaScript) {
2071 // FIXME: This might apply to other languages and token kinds.
2072 if (Right.is(tok::char_constant) && Left.is(tok::plus) && Left.Previous &&
2073 Left.Previous->is(tok::char_constant))
2075 if (Left.is(TT_DictLiteral) && Left.is(tok::l_brace) &&
2076 Left.NestingLevel == 0 && Left.Previous &&
2077 Left.Previous->is(tok::equal) &&
2078 Line.First->isOneOf(tok::identifier, Keywords.kw_import,
2080 // kw_var is a pseudo-token that's a tok::identifier, so matches above.
2081 !Line.First->is(Keywords.kw_var))
2082 // Enum style object literal.
2084 } else if (Style.Language == FormatStyle::LK_Java) {
2085 if (Right.is(tok::plus) && Left.is(tok::string_literal) && Right.Next &&
2086 Right.Next->is(tok::string_literal))
2093 bool TokenAnnotator::canBreakBefore(const AnnotatedLine &Line,
2094 const FormatToken &Right) {
2095 const FormatToken &Left = *Right.Previous;
2097 if (Style.Language == FormatStyle::LK_Java) {
2098 if (Left.isOneOf(Keywords.kw_throws, Keywords.kw_extends,
2099 Keywords.kw_implements))
2101 if (Right.isOneOf(Keywords.kw_throws, Keywords.kw_extends,
2102 Keywords.kw_implements))
2106 if (Left.is(tok::at))
2108 if (Left.Tok.getObjCKeywordID() == tok::objc_interface)
2110 if (Left.isOneOf(TT_JavaAnnotation, TT_LeadingJavaAnnotation))
2111 return !Right.is(tok::l_paren);
2112 if (Right.is(TT_PointerOrReference))
2113 return Line.IsMultiVariableDeclStmt ||
2114 (Style.PointerAlignment == FormatStyle::PAS_Right &&
2115 (!Right.Next || Right.Next->isNot(TT_FunctionDeclarationName)));
2116 if (Right.isOneOf(TT_StartOfName, TT_FunctionDeclarationName) ||
2117 Right.is(tok::kw_operator))
2119 if (Left.is(TT_PointerOrReference))
2121 if (Right.isTrailingComment())
2122 // We rely on MustBreakBefore being set correctly here as we should not
2123 // change the "binding" behavior of a comment.
2124 // The first comment in a braced lists is always interpreted as belonging to
2125 // the first list element. Otherwise, it should be placed outside of the
2127 return Left.BlockKind == BK_BracedInit;
2128 if (Left.is(tok::question) && Right.is(tok::colon))
2130 if (Right.is(TT_ConditionalExpr) || Right.is(tok::question))
2131 return Style.BreakBeforeTernaryOperators;
2132 if (Left.is(TT_ConditionalExpr) || Left.is(tok::question))
2133 return !Style.BreakBeforeTernaryOperators;
2134 if (Right.is(TT_InheritanceColon))
2136 if (Right.is(tok::colon) &&
2137 !Right.isOneOf(TT_CtorInitializerColon, TT_InlineASMColon))
2139 if (Left.is(tok::colon) && (Left.isOneOf(TT_DictLiteral, TT_ObjCMethodExpr)))
2141 if (Right.is(TT_SelectorName) || (Right.is(tok::identifier) && Right.Next &&
2142 Right.Next->is(TT_ObjCMethodExpr)))
2143 return Left.isNot(tok::period); // FIXME: Properly parse ObjC calls.
2144 if (Left.is(tok::r_paren) && Line.Type == LT_ObjCProperty)
2146 if (Left.ClosesTemplateDeclaration)
2148 if (Right.isOneOf(TT_RangeBasedForLoopColon, TT_OverloadedOperatorLParen,
2149 TT_OverloadedOperator))
2151 if (Left.is(TT_RangeBasedForLoopColon))
2153 if (Right.is(TT_RangeBasedForLoopColon))
2155 if (Left.isOneOf(TT_TemplateCloser, TT_UnaryOperator) ||
2156 Left.is(tok::kw_operator))
2158 if (Left.is(tok::equal) && !Right.isOneOf(tok::kw_default, tok::kw_delete) &&
2159 Line.Type == LT_VirtualFunctionDecl)
2161 if (Left.is(tok::l_paren) && Left.is(TT_AttributeParen))
2163 if (Left.is(tok::l_paren) && Left.Previous &&
2164 (Left.Previous->isOneOf(TT_BinaryOperator, TT_CastRParen)))
2166 if (Right.is(TT_ImplicitStringLiteral))
2169 if (Right.is(tok::r_paren) || Right.is(TT_TemplateCloser))
2172 // We only break before r_brace if there was a corresponding break before
2173 // the l_brace, which is tracked by BreakBeforeClosingBrace.
2174 if (Right.is(tok::r_brace))
2175 return Right.MatchingParen && Right.MatchingParen->BlockKind == BK_Block;
2177 // Allow breaking after a trailing annotation, e.g. after a method
2179 if (Left.is(TT_TrailingAnnotation))
2180 return !Right.isOneOf(tok::l_brace, tok::semi, tok::equal, tok::l_paren,
2181 tok::less, tok::coloncolon);
2183 if (Right.is(tok::kw___attribute))
2186 if (Left.is(tok::identifier) && Right.is(tok::string_literal))
2189 if (Right.is(tok::identifier) && Right.Next && Right.Next->is(TT_DictLiteral))
2192 if (Left.is(TT_CtorInitializerComma) &&
2193 Style.BreakConstructorInitializersBeforeComma)
2195 if (Right.is(TT_CtorInitializerComma) &&
2196 Style.BreakConstructorInitializersBeforeComma)
2198 if ((Left.is(tok::greater) && Right.is(tok::greater)) ||
2199 (Left.is(tok::less) && Right.is(tok::less)))
2201 if (Right.is(TT_BinaryOperator) &&
2202 Style.BreakBeforeBinaryOperators != FormatStyle::BOS_None &&
2203 (Style.BreakBeforeBinaryOperators == FormatStyle::BOS_All ||
2204 Right.getPrecedence() != prec::Assignment))
2206 if (Left.is(TT_ArrayInitializerLSquare))
2208 if (Right.is(tok::kw_typename) && Left.isNot(tok::kw_const))
2210 if (Left.isBinaryOperator() && !Left.isOneOf(tok::arrowstar, tok::lessless) &&
2211 Style.BreakBeforeBinaryOperators != FormatStyle::BOS_All &&
2212 (Style.BreakBeforeBinaryOperators == FormatStyle::BOS_None ||
2213 Left.getPrecedence() == prec::Assignment))
2215 return Left.isOneOf(tok::comma, tok::coloncolon, tok::semi, tok::l_brace,
2216 tok::kw_class, tok::kw_struct) ||
2217 Right.isMemberAccess() ||
2218 Right.isOneOf(TT_TrailingReturnArrow, TT_LambdaArrow, tok::lessless,
2219 tok::colon, tok::l_square, tok::at) ||
2220 (Left.is(tok::r_paren) &&
2221 Right.isOneOf(tok::identifier, tok::kw_const)) ||
2222 (Left.is(tok::l_paren) && !Right.is(tok::r_paren));
2225 void TokenAnnotator::printDebugInfo(const AnnotatedLine &Line) {
2226 llvm::errs() << "AnnotatedTokens:\n";
2227 const FormatToken *Tok = Line.First;
2229 llvm::errs() << " M=" << Tok->MustBreakBefore
2230 << " C=" << Tok->CanBreakBefore << " T=" << Tok->Type
2231 << " S=" << Tok->SpacesRequiredBefore
2232 << " B=" << Tok->BlockParameterCount
2233 << " P=" << Tok->SplitPenalty << " Name=" << Tok->Tok.getName()
2234 << " L=" << Tok->TotalLength << " PPK=" << Tok->PackingKind
2236 for (unsigned i = 0, e = Tok->FakeLParens.size(); i != e; ++i)
2237 llvm::errs() << Tok->FakeLParens[i] << "/";
2238 llvm::errs() << " FakeRParens=" << Tok->FakeRParens << "\n";
2240 assert(Tok == Line.Last);
2243 llvm::errs() << "----\n";
2246 } // namespace format
2247 } // namespace clang