unittests/Lex/LexerTest.cpp

   1 //===- unittests/Lex/LexerTest.cpp ------ Lexer tests ---------------------===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9
  10 #include "clang/Lex/Lexer.h"
  11 #include "clang/Basic/Diagnostic.h"
  12 #include "clang/Basic/DiagnosticOptions.h"
  13 #include "clang/Basic/FileManager.h"
  14 #include "clang/Basic/LangOptions.h"
  15 #include "clang/Basic/MemoryBufferCache.h"
  16 #include "clang/Basic/SourceManager.h"
  17 #include "clang/Basic/TargetInfo.h"
  18 #include "clang/Basic/TargetOptions.h"
  19 #include "clang/Lex/HeaderSearch.h"
  20 #include "clang/Lex/HeaderSearchOptions.h"
  21 #include "clang/Lex/ModuleLoader.h"
  22 #include "clang/Lex/Preprocessor.h"
  23 #include "clang/Lex/PreprocessorOptions.h"
  24 #include "gtest/gtest.h"
  25
  26 using namespace clang;
  27
  28 namespace {
  29
  30 // The test fixture.
  31 class LexerTest : public ::testing::Test {
  32 protected:
  33   LexerTest()
  34     : FileMgr(FileMgrOpts),
  35       DiagID(new DiagnosticIDs()),
  36       Diags(DiagID, new DiagnosticOptions, new IgnoringDiagConsumer()),
  37       SourceMgr(Diags, FileMgr),
  38       TargetOpts(new TargetOptions)
  39   {
  40     TargetOpts->Triple = "x86_64-apple-darwin11.1.0";
  41     Target = TargetInfo::CreateTargetInfo(Diags, TargetOpts);
  42   }
  43
  44   std::vector<Token> Lex(StringRef Source) {
  45     std::unique_ptr<llvm::MemoryBuffer> Buf =
  46         llvm::MemoryBuffer::getMemBuffer(Source);
  47     SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(Buf)));
  48
  49     TrivialModuleLoader ModLoader;
  50     MemoryBufferCache PCMCache;
  51     HeaderSearch HeaderInfo(std::make_shared<HeaderSearchOptions>(), SourceMgr,
  52                             Diags, LangOpts, Target.get());
  53     Preprocessor PP(std::make_shared<PreprocessorOptions>(), Diags, LangOpts,
  54                     SourceMgr, PCMCache, HeaderInfo, ModLoader,
  55                     /*IILookup =*/nullptr,
  56                     /*OwnsHeaderSearch =*/false);
  57     PP.Initialize(*Target);
  58     PP.EnterMainSourceFile();
  59
  60     std::vector<Token> toks;
  61     while (1) {
  62       Token tok;
  63       PP.Lex(tok);
  64       if (tok.is(tok::eof))
  65         break;
  66       toks.push_back(tok);
  67     }
  68
  69     return toks;
  70   }
  71
  72   std::vector<Token> CheckLex(StringRef Source,
  73                               ArrayRef<tok::TokenKind> ExpectedTokens) {
  74     auto toks = Lex(Source);
  75     EXPECT_EQ(ExpectedTokens.size(), toks.size());
  76     for (unsigned i = 0, e = ExpectedTokens.size(); i != e; ++i) {
  77       EXPECT_EQ(ExpectedTokens[i], toks[i].getKind());
  78     }
  79
  80     return toks;
  81   }
  82
  83   std::string getSourceText(Token Begin, Token End) {
  84     bool Invalid;
  85     StringRef Str =
  86         Lexer::getSourceText(CharSourceRange::getTokenRange(SourceRange(
  87                                     Begin.getLocation(), End.getLocation())),
  88                              SourceMgr, LangOpts, &Invalid);
  89     if (Invalid)
  90       return "<INVALID>";
  91     return Str;
  92   }
  93
  94   FileSystemOptions FileMgrOpts;
  95   FileManager FileMgr;
  96   IntrusiveRefCntPtr<DiagnosticIDs> DiagID;
  97   DiagnosticsEngine Diags;
  98   SourceManager SourceMgr;
  99   LangOptions LangOpts;
 100   std::shared_ptr<TargetOptions> TargetOpts;
 101   IntrusiveRefCntPtr<TargetInfo> Target;
 102 };
 103
 104 TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgument) {
 105   std::vector<tok::TokenKind> ExpectedTokens;
 106   ExpectedTokens.push_back(tok::identifier);
 107   ExpectedTokens.push_back(tok::l_paren);
 108   ExpectedTokens.push_back(tok::identifier);
 109   ExpectedTokens.push_back(tok::r_paren);
 110
 111   std::vector<Token> toks = CheckLex("#define M(x) x\n"
 112                                      "M(f(M(i)))",
 113                                      ExpectedTokens);
 114
 115   EXPECT_EQ("M(i)", getSourceText(toks[2], toks[2]));
 116 }
 117
 118 TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgumentForEndOfMacro) {
 119   std::vector<tok::TokenKind> ExpectedTokens;
 120   ExpectedTokens.push_back(tok::identifier);
 121   ExpectedTokens.push_back(tok::identifier);
 122
 123   std::vector<Token> toks = CheckLex("#define M(x) x\n"
 124                                      "M(M(i) c)",
 125                                      ExpectedTokens);
 126
 127   EXPECT_EQ("M(i)", getSourceText(toks[0], toks[0]));
 128 }
 129
 130 TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForBeginOfMacro) {
 131   std::vector<tok::TokenKind> ExpectedTokens;
 132   ExpectedTokens.push_back(tok::identifier);
 133   ExpectedTokens.push_back(tok::identifier);
 134   ExpectedTokens.push_back(tok::identifier);
 135
 136   std::vector<Token> toks = CheckLex("#define M(x) x\n"
 137                                      "M(c c M(i))",
 138                                      ExpectedTokens);
 139
 140   EXPECT_EQ("c M(i)", getSourceText(toks[1], toks[2]));
 141 }
 142
 143 TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForEndOfMacro) {
 144   std::vector<tok::TokenKind> ExpectedTokens;
 145   ExpectedTokens.push_back(tok::identifier);
 146   ExpectedTokens.push_back(tok::identifier);
 147   ExpectedTokens.push_back(tok::identifier);
 148
 149   std::vector<Token> toks = CheckLex("#define M(x) x\n"
 150                                      "M(M(i) c c)",
 151                                      ExpectedTokens);
 152
 153   EXPECT_EQ("M(i) c", getSourceText(toks[0], toks[1]));
 154 }
 155
 156 TEST_F(LexerTest, GetSourceTextInSeparateFnMacros) {
 157   std::vector<tok::TokenKind> ExpectedTokens;
 158   ExpectedTokens.push_back(tok::identifier);
 159   ExpectedTokens.push_back(tok::identifier);
 160   ExpectedTokens.push_back(tok::identifier);
 161   ExpectedTokens.push_back(tok::identifier);
 162
 163   std::vector<Token> toks = CheckLex("#define M(x) x\n"
 164                                      "M(c M(i)) M(M(i) c)",
 165                                      ExpectedTokens);
 166
 167   EXPECT_EQ("<INVALID>", getSourceText(toks[1], toks[2]));
 168 }
 169
 170 TEST_F(LexerTest, GetSourceTextWorksAcrossTokenPastes) {
 171   std::vector<tok::TokenKind> ExpectedTokens;
 172   ExpectedTokens.push_back(tok::identifier);
 173   ExpectedTokens.push_back(tok::l_paren);
 174   ExpectedTokens.push_back(tok::identifier);
 175   ExpectedTokens.push_back(tok::r_paren);
 176
 177   std::vector<Token> toks = CheckLex("#define M(x) x\n"
 178                                      "#define C(x) M(x##c)\n"
 179                                      "M(f(C(i)))",
 180                                      ExpectedTokens);
 181
 182   EXPECT_EQ("C(i)", getSourceText(toks[2], toks[2]));
 183 }
 184
 185 TEST_F(LexerTest, GetSourceTextExpandsAcrossMultipleMacroCalls) {
 186   std::vector<tok::TokenKind> ExpectedTokens;
 187   ExpectedTokens.push_back(tok::identifier);
 188   ExpectedTokens.push_back(tok::l_paren);
 189   ExpectedTokens.push_back(tok::identifier);
 190   ExpectedTokens.push_back(tok::r_paren);
 191
 192   std::vector<Token> toks = CheckLex("#define M(x) x\n"
 193                                      "f(M(M(i)))",
 194                                      ExpectedTokens);
 195   EXPECT_EQ("M(M(i))", getSourceText(toks[2], toks[2]));
 196 }
 197
 198 TEST_F(LexerTest, GetSourceTextInMiddleOfMacroArgument) {
 199   std::vector<tok::TokenKind> ExpectedTokens;
 200   ExpectedTokens.push_back(tok::identifier);
 201   ExpectedTokens.push_back(tok::l_paren);
 202   ExpectedTokens.push_back(tok::identifier);
 203   ExpectedTokens.push_back(tok::r_paren);
 204
 205   std::vector<Token> toks = CheckLex("#define M(x) x\n"
 206                                      "M(f(i))",
 207                                      ExpectedTokens);
 208   EXPECT_EQ("i", getSourceText(toks[2], toks[2]));
 209 }
 210
 211 TEST_F(LexerTest, GetSourceTextExpandsAroundDifferentMacroCalls) {
 212   std::vector<tok::TokenKind> ExpectedTokens;
 213   ExpectedTokens.push_back(tok::identifier);
 214   ExpectedTokens.push_back(tok::l_paren);
 215   ExpectedTokens.push_back(tok::identifier);
 216   ExpectedTokens.push_back(tok::r_paren);
 217
 218   std::vector<Token> toks = CheckLex("#define M(x) x\n"
 219                                      "#define C(x) x\n"
 220                                      "f(C(M(i)))",
 221                                      ExpectedTokens);
 222   EXPECT_EQ("C(M(i))", getSourceText(toks[2], toks[2]));
 223 }
 224
 225 TEST_F(LexerTest, GetSourceTextOnlyExpandsIfFirstTokenInMacro) {
 226   std::vector<tok::TokenKind> ExpectedTokens;
 227   ExpectedTokens.push_back(tok::identifier);
 228   ExpectedTokens.push_back(tok::l_paren);
 229   ExpectedTokens.push_back(tok::identifier);
 230   ExpectedTokens.push_back(tok::identifier);
 231   ExpectedTokens.push_back(tok::r_paren);
 232
 233   std::vector<Token> toks = CheckLex("#define M(x) x\n"
 234                                      "#define C(x) c x\n"
 235                                      "f(C(M(i)))",
 236                                      ExpectedTokens);
 237   EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3]));
 238 }
 239
 240 TEST_F(LexerTest, GetSourceTextExpandsRecursively) {
 241   std::vector<tok::TokenKind> ExpectedTokens;
 242   ExpectedTokens.push_back(tok::identifier);
 243   ExpectedTokens.push_back(tok::identifier);
 244   ExpectedTokens.push_back(tok::l_paren);
 245   ExpectedTokens.push_back(tok::identifier);
 246   ExpectedTokens.push_back(tok::r_paren);
 247
 248   std::vector<Token> toks = CheckLex("#define M(x) x\n"
 249                                      "#define C(x) c M(x)\n"
 250                                      "C(f(M(i)))",
 251                                      ExpectedTokens);
 252   EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3]));
 253 }
 254
 255 TEST_F(LexerTest, LexAPI) {
 256   std::vector<tok::TokenKind> ExpectedTokens;
 257   ExpectedTokens.push_back(tok::l_square);
 258   ExpectedTokens.push_back(tok::identifier);
 259   ExpectedTokens.push_back(tok::r_square);
 260   ExpectedTokens.push_back(tok::l_square);
 261   ExpectedTokens.push_back(tok::identifier);
 262   ExpectedTokens.push_back(tok::r_square);
 263   ExpectedTokens.push_back(tok::identifier);
 264   ExpectedTokens.push_back(tok::identifier);
 265   ExpectedTokens.push_back(tok::identifier);
 266   ExpectedTokens.push_back(tok::identifier);
 267
 268   std::vector<Token> toks = CheckLex("#define M(x) [x]\n"
 269                                      "#define N(x) x\n"
 270                                      "#define INN(x) x\n"
 271                                      "#define NOF1 INN(val)\n"
 272                                      "#define NOF2 val\n"
 273                                      "M(foo) N([bar])\n"
 274                                      "N(INN(val)) N(NOF1) N(NOF2) N(val)",
 275                                      ExpectedTokens);
 276
 277   SourceLocation lsqrLoc = toks[0].getLocation();
 278   SourceLocation idLoc = toks[1].getLocation();
 279   SourceLocation rsqrLoc = toks[2].getLocation();
 280   std::pair<SourceLocation,SourceLocation>
 281     macroPair = SourceMgr.getExpansionRange(lsqrLoc);
 282   SourceRange macroRange = SourceRange(macroPair.first, macroPair.second);
 283
 284   SourceLocation Loc;
 285   EXPECT_TRUE(Lexer::isAtStartOfMacroExpansion(lsqrLoc, SourceMgr, LangOpts, &Loc));
 286   EXPECT_EQ(Loc, macroRange.getBegin());
 287   EXPECT_FALSE(Lexer::isAtStartOfMacroExpansion(idLoc, SourceMgr, LangOpts));
 288   EXPECT_FALSE(Lexer::isAtEndOfMacroExpansion(idLoc, SourceMgr, LangOpts));
 289   EXPECT_TRUE(Lexer::isAtEndOfMacroExpansion(rsqrLoc, SourceMgr, LangOpts, &Loc));
 290   EXPECT_EQ(Loc, macroRange.getEnd());
 291
 292   CharSourceRange range = Lexer::makeFileCharRange(
 293            CharSourceRange::getTokenRange(lsqrLoc, idLoc), SourceMgr, LangOpts);
 294   EXPECT_TRUE(range.isInvalid());
 295   range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(idLoc, rsqrLoc),
 296                                    SourceMgr, LangOpts);
 297   EXPECT_TRUE(range.isInvalid());
 298   range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc),
 299                                    SourceMgr, LangOpts);
 300   EXPECT_TRUE(!range.isTokenRange());
 301   EXPECT_EQ(range.getAsRange(),
 302             SourceRange(macroRange.getBegin(),
 303                         macroRange.getEnd().getLocWithOffset(1)));
 304
 305   StringRef text = Lexer::getSourceText(
 306                                CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc),
 307                                SourceMgr, LangOpts);
 308   EXPECT_EQ(text, "M(foo)");
 309
 310   SourceLocation macroLsqrLoc = toks[3].getLocation();
 311   SourceLocation macroIdLoc = toks[4].getLocation();
 312   SourceLocation macroRsqrLoc = toks[5].getLocation();
 313   SourceLocation fileLsqrLoc = SourceMgr.getSpellingLoc(macroLsqrLoc);
 314   SourceLocation fileIdLoc = SourceMgr.getSpellingLoc(macroIdLoc);
 315   SourceLocation fileRsqrLoc = SourceMgr.getSpellingLoc(macroRsqrLoc);
 316
 317   range = Lexer::makeFileCharRange(
 318       CharSourceRange::getTokenRange(macroLsqrLoc, macroIdLoc),
 319       SourceMgr, LangOpts);
 320   EXPECT_EQ(SourceRange(fileLsqrLoc, fileIdLoc.getLocWithOffset(3)),
 321             range.getAsRange());
 322
 323   range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(macroIdLoc, macroRsqrLoc),
 324                                    SourceMgr, LangOpts);
 325   EXPECT_EQ(SourceRange(fileIdLoc, fileRsqrLoc.getLocWithOffset(1)),
 326             range.getAsRange());
 327
 328   macroPair = SourceMgr.getExpansionRange(macroLsqrLoc);
 329   range = Lexer::makeFileCharRange(
 330                      CharSourceRange::getTokenRange(macroLsqrLoc, macroRsqrLoc),
 331                      SourceMgr, LangOpts);
 332   EXPECT_EQ(SourceRange(macroPair.first, macroPair.second.getLocWithOffset(1)),
 333             range.getAsRange());
 334
 335   text = Lexer::getSourceText(
 336           CharSourceRange::getTokenRange(SourceRange(macroLsqrLoc, macroIdLoc)),
 337           SourceMgr, LangOpts);
 338   EXPECT_EQ(text, "[bar");
 339
 340
 341   SourceLocation idLoc1 = toks[6].getLocation();
 342   SourceLocation idLoc2 = toks[7].getLocation();
 343   SourceLocation idLoc3 = toks[8].getLocation();
 344   SourceLocation idLoc4 = toks[9].getLocation();
 345   EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc1, SourceMgr, LangOpts));
 346   EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc2, SourceMgr, LangOpts));
 347   EXPECT_EQ("NOF2", Lexer::getImmediateMacroName(idLoc3, SourceMgr, LangOpts));
 348   EXPECT_EQ("N", Lexer::getImmediateMacroName(idLoc4, SourceMgr, LangOpts));
 349 }
 350
 351 TEST_F(LexerTest, DontMergeMacroArgsFromDifferentMacroFiles) {
 352   std::vector<Token> toks =
 353       Lex("#define helper1 0\n"
 354           "void helper2(const char *, ...);\n"
 355           "#define M1(a, ...) helper2(a, ##__VA_ARGS__)\n"
 356           "#define M2(a, ...) M1(a, helper1, ##__VA_ARGS__)\n"
 357           "void f1() { M2(\"a\", \"b\"); }");
 358
 359   // Check the file corresponding to the "helper1" macro arg in M2.
 360   //
 361   // The lexer used to report its size as 31, meaning that the end of the
 362   // expansion would be on the *next line* (just past `M2("a", "b")`). Make
 363   // sure that we get the correct end location (the comma after "helper1").
 364   SourceLocation helper1ArgLoc = toks[20].getLocation();
 365   EXPECT_EQ(SourceMgr.getFileIDSize(SourceMgr.getFileID(helper1ArgLoc)), 8U);
 366 }
 367
 368 } // anonymous namespace