unittests/Lex/LexerTest.cpp

   1 //===- unittests/Lex/LexerTest.cpp ------ Lexer tests ---------------------===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9
  10 #include "clang/Lex/Lexer.h"
  11 #include "clang/Basic/Diagnostic.h"
  12 #include "clang/Basic/DiagnosticOptions.h"
  13 #include "clang/Basic/FileManager.h"
  14 #include "clang/Basic/LangOptions.h"
  15 #include "clang/Basic/SourceManager.h"
  16 #include "clang/Basic/TargetInfo.h"
  17 #include "clang/Basic/TargetOptions.h"
  18 #include "clang/Lex/HeaderSearch.h"
  19 #include "clang/Lex/HeaderSearchOptions.h"
  20 #include "clang/Lex/ModuleLoader.h"
  21 #include "clang/Lex/Preprocessor.h"
  22 #include "clang/Lex/PreprocessorOptions.h"
  23 #include "gtest/gtest.h"
  24
  25 using namespace llvm;
  26 using namespace clang;
  27
  28 namespace {
  29
  30 class VoidModuleLoader : public ModuleLoader {
  31   ModuleLoadResult loadModule(SourceLocation ImportLoc,
  32                               ModuleIdPath Path,
  33                               Module::NameVisibilityKind Visibility,
  34                               bool IsInclusionDirective) override {
  35     return ModuleLoadResult();
  36   }
  37
  38   void makeModuleVisible(Module *Mod,
  39                          Module::NameVisibilityKind Visibility,
  40                          SourceLocation ImportLoc,
  41                          bool Complain) override { }
  42
  43   GlobalModuleIndex *loadGlobalModuleIndex(SourceLocation TriggerLoc) override
  44     { return nullptr; }
  45   bool lookupMissingImports(StringRef Name, SourceLocation TriggerLoc) override
  46     { return 0; };
  47 };
  48
  49 // The test fixture.
  50 class LexerTest : public ::testing::Test {
  51 protected:
  52   LexerTest()
  53     : FileMgr(FileMgrOpts),
  54       DiagID(new DiagnosticIDs()),
  55       Diags(DiagID, new DiagnosticOptions, new IgnoringDiagConsumer()),
  56       SourceMgr(Diags, FileMgr),
  57       TargetOpts(new TargetOptions)
  58   {
  59     TargetOpts->Triple = "x86_64-apple-darwin11.1.0";
  60     Target = TargetInfo::CreateTargetInfo(Diags, TargetOpts);
  61   }
  62
  63   std::vector<Token> CheckLex(StringRef Source,
  64                               ArrayRef<tok::TokenKind> ExpectedTokens) {
  65     std::unique_ptr<MemoryBuffer> Buf = MemoryBuffer::getMemBuffer(Source);
  66     SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(Buf)));
  67
  68     VoidModuleLoader ModLoader;
  69     HeaderSearch HeaderInfo(new HeaderSearchOptions, SourceMgr, Diags, LangOpts,
  70                             Target.get());
  71     Preprocessor PP(new PreprocessorOptions(), Diags, LangOpts, SourceMgr,
  72                     HeaderInfo, ModLoader, /*IILookup =*/nullptr,
  73                     /*OwnsHeaderSearch =*/false);
  74     PP.Initialize(*Target);
  75     PP.EnterMainSourceFile();
  76
  77     std::vector<Token> toks;
  78     while (1) {
  79       Token tok;
  80       PP.Lex(tok);
  81       if (tok.is(tok::eof))
  82         break;
  83       toks.push_back(tok);
  84     }
  85
  86     EXPECT_EQ(ExpectedTokens.size(), toks.size());
  87     for (unsigned i = 0, e = ExpectedTokens.size(); i != e; ++i) {
  88       EXPECT_EQ(ExpectedTokens[i], toks[i].getKind());
  89     }
  90
  91     return toks;
  92   }
  93
  94   std::string getSourceText(Token Begin, Token End) {
  95     bool Invalid;
  96     StringRef Str =
  97         Lexer::getSourceText(CharSourceRange::getTokenRange(SourceRange(
  98                                     Begin.getLocation(), End.getLocation())),
  99                              SourceMgr, LangOpts, &Invalid);
 100     if (Invalid)
 101       return "<INVALID>";
 102     return Str;
 103   }
 104
 105   FileSystemOptions FileMgrOpts;
 106   FileManager FileMgr;
 107   IntrusiveRefCntPtr<DiagnosticIDs> DiagID;
 108   DiagnosticsEngine Diags;
 109   SourceManager SourceMgr;
 110   LangOptions LangOpts;
 111   std::shared_ptr<TargetOptions> TargetOpts;
 112   IntrusiveRefCntPtr<TargetInfo> Target;
 113 };
 114
 115 TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgument) {
 116   std::vector<tok::TokenKind> ExpectedTokens;
 117   ExpectedTokens.push_back(tok::identifier);
 118   ExpectedTokens.push_back(tok::l_paren);
 119   ExpectedTokens.push_back(tok::identifier);
 120   ExpectedTokens.push_back(tok::r_paren);
 121
 122   std::vector<Token> toks = CheckLex("#define M(x) x\n"
 123                                      "M(f(M(i)))",
 124                                      ExpectedTokens);
 125
 126   EXPECT_EQ("M(i)", getSourceText(toks[2], toks[2]));
 127 }
 128
 129 TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgumentForEndOfMacro) {
 130   std::vector<tok::TokenKind> ExpectedTokens;
 131   ExpectedTokens.push_back(tok::identifier);
 132   ExpectedTokens.push_back(tok::identifier);
 133
 134   std::vector<Token> toks = CheckLex("#define M(x) x\n"
 135                                      "M(M(i) c)",
 136                                      ExpectedTokens);
 137
 138   EXPECT_EQ("M(i)", getSourceText(toks[0], toks[0]));
 139 }
 140
 141 TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForBeginOfMacro) {
 142   std::vector<tok::TokenKind> ExpectedTokens;
 143   ExpectedTokens.push_back(tok::identifier);
 144   ExpectedTokens.push_back(tok::identifier);
 145   ExpectedTokens.push_back(tok::identifier);
 146
 147   std::vector<Token> toks = CheckLex("#define M(x) x\n"
 148                                      "M(c c M(i))",
 149                                      ExpectedTokens);
 150
 151   EXPECT_EQ("c M(i)", getSourceText(toks[1], toks[2]));
 152 }
 153
 154 TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForEndOfMacro) {
 155   std::vector<tok::TokenKind> ExpectedTokens;
 156   ExpectedTokens.push_back(tok::identifier);
 157   ExpectedTokens.push_back(tok::identifier);
 158   ExpectedTokens.push_back(tok::identifier);
 159
 160   std::vector<Token> toks = CheckLex("#define M(x) x\n"
 161                                      "M(M(i) c c)",
 162                                      ExpectedTokens);
 163
 164   EXPECT_EQ("M(i) c", getSourceText(toks[0], toks[1]));
 165 }
 166
 167 TEST_F(LexerTest, GetSourceTextInSeparateFnMacros) {
 168   std::vector<tok::TokenKind> ExpectedTokens;
 169   ExpectedTokens.push_back(tok::identifier);
 170   ExpectedTokens.push_back(tok::identifier);
 171   ExpectedTokens.push_back(tok::identifier);
 172   ExpectedTokens.push_back(tok::identifier);
 173
 174   std::vector<Token> toks = CheckLex("#define M(x) x\n"
 175                                      "M(c M(i)) M(M(i) c)",
 176                                      ExpectedTokens);
 177
 178   EXPECT_EQ("<INVALID>", getSourceText(toks[1], toks[2]));
 179 }
 180
 181 TEST_F(LexerTest, GetSourceTextWorksAcrossTokenPastes) {
 182   std::vector<tok::TokenKind> ExpectedTokens;
 183   ExpectedTokens.push_back(tok::identifier);
 184   ExpectedTokens.push_back(tok::l_paren);
 185   ExpectedTokens.push_back(tok::identifier);
 186   ExpectedTokens.push_back(tok::r_paren);
 187
 188   std::vector<Token> toks = CheckLex("#define M(x) x\n"
 189                                      "#define C(x) M(x##c)\n"
 190                                      "M(f(C(i)))",
 191                                      ExpectedTokens);
 192
 193   EXPECT_EQ("C(i)", getSourceText(toks[2], toks[2]));
 194 }
 195
 196 TEST_F(LexerTest, GetSourceTextExpandsAcrossMultipleMacroCalls) {
 197   std::vector<tok::TokenKind> ExpectedTokens;
 198   ExpectedTokens.push_back(tok::identifier);
 199   ExpectedTokens.push_back(tok::l_paren);
 200   ExpectedTokens.push_back(tok::identifier);
 201   ExpectedTokens.push_back(tok::r_paren);
 202
 203   std::vector<Token> toks = CheckLex("#define M(x) x\n"
 204                                      "f(M(M(i)))",
 205                                      ExpectedTokens);
 206   EXPECT_EQ("M(M(i))", getSourceText(toks[2], toks[2]));
 207 }
 208
 209 TEST_F(LexerTest, GetSourceTextInMiddleOfMacroArgument) {
 210   std::vector<tok::TokenKind> ExpectedTokens;
 211   ExpectedTokens.push_back(tok::identifier);
 212   ExpectedTokens.push_back(tok::l_paren);
 213   ExpectedTokens.push_back(tok::identifier);
 214   ExpectedTokens.push_back(tok::r_paren);
 215
 216   std::vector<Token> toks = CheckLex("#define M(x) x\n"
 217                                      "M(f(i))",
 218                                      ExpectedTokens);
 219   EXPECT_EQ("i", getSourceText(toks[2], toks[2]));
 220 }
 221
 222 TEST_F(LexerTest, GetSourceTextExpandsAroundDifferentMacroCalls) {
 223   std::vector<tok::TokenKind> ExpectedTokens;
 224   ExpectedTokens.push_back(tok::identifier);
 225   ExpectedTokens.push_back(tok::l_paren);
 226   ExpectedTokens.push_back(tok::identifier);
 227   ExpectedTokens.push_back(tok::r_paren);
 228
 229   std::vector<Token> toks = CheckLex("#define M(x) x\n"
 230                                      "#define C(x) x\n"
 231                                      "f(C(M(i)))",
 232                                      ExpectedTokens);
 233   EXPECT_EQ("C(M(i))", getSourceText(toks[2], toks[2]));
 234 }
 235
 236 TEST_F(LexerTest, GetSourceTextOnlyExpandsIfFirstTokenInMacro) {
 237   std::vector<tok::TokenKind> ExpectedTokens;
 238   ExpectedTokens.push_back(tok::identifier);
 239   ExpectedTokens.push_back(tok::l_paren);
 240   ExpectedTokens.push_back(tok::identifier);
 241   ExpectedTokens.push_back(tok::identifier);
 242   ExpectedTokens.push_back(tok::r_paren);
 243
 244   std::vector<Token> toks = CheckLex("#define M(x) x\n"
 245                                      "#define C(x) c x\n"
 246                                      "f(C(M(i)))",
 247                                      ExpectedTokens);
 248   EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3]));
 249 }
 250
 251 TEST_F(LexerTest, GetSourceTextExpandsRecursively) {
 252   std::vector<tok::TokenKind> ExpectedTokens;
 253   ExpectedTokens.push_back(tok::identifier);
 254   ExpectedTokens.push_back(tok::identifier);
 255   ExpectedTokens.push_back(tok::l_paren);
 256   ExpectedTokens.push_back(tok::identifier);
 257   ExpectedTokens.push_back(tok::r_paren);
 258
 259   std::vector<Token> toks = CheckLex("#define M(x) x\n"
 260                                      "#define C(x) c M(x)\n"
 261                                      "C(f(M(i)))",
 262                                      ExpectedTokens);
 263   EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3]));
 264 }
 265
 266 TEST_F(LexerTest, LexAPI) {
 267   std::vector<tok::TokenKind> ExpectedTokens;
 268   ExpectedTokens.push_back(tok::l_square);
 269   ExpectedTokens.push_back(tok::identifier);
 270   ExpectedTokens.push_back(tok::r_square);
 271   ExpectedTokens.push_back(tok::l_square);
 272   ExpectedTokens.push_back(tok::identifier);
 273   ExpectedTokens.push_back(tok::r_square);
 274   ExpectedTokens.push_back(tok::identifier);
 275   ExpectedTokens.push_back(tok::identifier);
 276   ExpectedTokens.push_back(tok::identifier);
 277   ExpectedTokens.push_back(tok::identifier);
 278
 279   std::vector<Token> toks = CheckLex("#define M(x) [x]\n"
 280                                      "#define N(x) x\n"
 281                                      "#define INN(x) x\n"
 282                                      "#define NOF1 INN(val)\n"
 283                                      "#define NOF2 val\n"
 284                                      "M(foo) N([bar])\n"
 285                                      "N(INN(val)) N(NOF1) N(NOF2) N(val)",
 286                                      ExpectedTokens);
 287
 288   SourceLocation lsqrLoc = toks[0].getLocation();
 289   SourceLocation idLoc = toks[1].getLocation();
 290   SourceLocation rsqrLoc = toks[2].getLocation();
 291   std::pair<SourceLocation,SourceLocation>
 292     macroPair = SourceMgr.getExpansionRange(lsqrLoc);
 293   SourceRange macroRange = SourceRange(macroPair.first, macroPair.second);
 294
 295   SourceLocation Loc;
 296   EXPECT_TRUE(Lexer::isAtStartOfMacroExpansion(lsqrLoc, SourceMgr, LangOpts, &Loc));
 297   EXPECT_EQ(Loc, macroRange.getBegin());
 298   EXPECT_FALSE(Lexer::isAtStartOfMacroExpansion(idLoc, SourceMgr, LangOpts));
 299   EXPECT_FALSE(Lexer::isAtEndOfMacroExpansion(idLoc, SourceMgr, LangOpts));
 300   EXPECT_TRUE(Lexer::isAtEndOfMacroExpansion(rsqrLoc, SourceMgr, LangOpts, &Loc));
 301   EXPECT_EQ(Loc, macroRange.getEnd());
 302
 303   CharSourceRange range = Lexer::makeFileCharRange(
 304            CharSourceRange::getTokenRange(lsqrLoc, idLoc), SourceMgr, LangOpts);
 305   EXPECT_TRUE(range.isInvalid());
 306   range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(idLoc, rsqrLoc),
 307                                    SourceMgr, LangOpts);
 308   EXPECT_TRUE(range.isInvalid());
 309   range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc),
 310                                    SourceMgr, LangOpts);
 311   EXPECT_TRUE(!range.isTokenRange());
 312   EXPECT_EQ(range.getAsRange(),
 313             SourceRange(macroRange.getBegin(),
 314                         macroRange.getEnd().getLocWithOffset(1)));
 315
 316   StringRef text = Lexer::getSourceText(
 317                                CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc),
 318                                SourceMgr, LangOpts);
 319   EXPECT_EQ(text, "M(foo)");
 320
 321   SourceLocation macroLsqrLoc = toks[3].getLocation();
 322   SourceLocation macroIdLoc = toks[4].getLocation();
 323   SourceLocation macroRsqrLoc = toks[5].getLocation();
 324   SourceLocation fileLsqrLoc = SourceMgr.getSpellingLoc(macroLsqrLoc);
 325   SourceLocation fileIdLoc = SourceMgr.getSpellingLoc(macroIdLoc);
 326   SourceLocation fileRsqrLoc = SourceMgr.getSpellingLoc(macroRsqrLoc);
 327
 328   range = Lexer::makeFileCharRange(
 329       CharSourceRange::getTokenRange(macroLsqrLoc, macroIdLoc),
 330       SourceMgr, LangOpts);
 331   EXPECT_EQ(SourceRange(fileLsqrLoc, fileIdLoc.getLocWithOffset(3)),
 332             range.getAsRange());
 333
 334   range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(macroIdLoc, macroRsqrLoc),
 335                                    SourceMgr, LangOpts);
 336   EXPECT_EQ(SourceRange(fileIdLoc, fileRsqrLoc.getLocWithOffset(1)),
 337             range.getAsRange());
 338
 339   macroPair = SourceMgr.getExpansionRange(macroLsqrLoc);
 340   range = Lexer::makeFileCharRange(
 341                      CharSourceRange::getTokenRange(macroLsqrLoc, macroRsqrLoc),
 342                      SourceMgr, LangOpts);
 343   EXPECT_EQ(SourceRange(macroPair.first, macroPair.second.getLocWithOffset(1)),
 344             range.getAsRange());
 345
 346   text = Lexer::getSourceText(
 347           CharSourceRange::getTokenRange(SourceRange(macroLsqrLoc, macroIdLoc)),
 348           SourceMgr, LangOpts);
 349   EXPECT_EQ(text, "[bar");
 350
 351
 352   SourceLocation idLoc1 = toks[6].getLocation();
 353   SourceLocation idLoc2 = toks[7].getLocation();
 354   SourceLocation idLoc3 = toks[8].getLocation();
 355   SourceLocation idLoc4 = toks[9].getLocation();
 356   EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc1, SourceMgr, LangOpts));
 357   EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc2, SourceMgr, LangOpts));
 358   EXPECT_EQ("NOF2", Lexer::getImmediateMacroName(idLoc3, SourceMgr, LangOpts));
 359   EXPECT_EQ("N", Lexer::getImmediateMacroName(idLoc4, SourceMgr, LangOpts));
 360 }
 361
 362 } // anonymous namespace