1 //===--- PTHLexer.cpp - Lex from a token stream ---------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file implements the PTHLexer interface.
12 //===----------------------------------------------------------------------===//
14 #include "clang/Lex/PTHLexer.h"
15 #include "clang/Basic/FileManager.h"
16 #include "clang/Basic/FileSystemStatCache.h"
17 #include "clang/Basic/IdentifierTable.h"
18 #include "clang/Basic/OnDiskHashTable.h"
19 #include "clang/Basic/TokenKinds.h"
20 #include "clang/Lex/LexDiagnostic.h"
21 #include "clang/Lex/PTHManager.h"
22 #include "clang/Lex/Preprocessor.h"
23 #include "clang/Lex/Token.h"
24 #include "llvm/ADT/OwningPtr.h"
25 #include "llvm/ADT/StringExtras.h"
26 #include "llvm/ADT/StringMap.h"
27 #include "llvm/Support/MemoryBuffer.h"
28 #include "llvm/Support/system_error.h"
29 using namespace clang;
30 using namespace clang::io;
32 #define DISK_TOKEN_SIZE (1+1+2+4+4)
34 //===----------------------------------------------------------------------===//
36 //===----------------------------------------------------------------------===//
38 PTHLexer::PTHLexer(Preprocessor &PP, FileID FID, const unsigned char *D,
39 const unsigned char *ppcond, PTHManager &PM)
40 : PreprocessorLexer(&PP, FID), TokBuf(D), CurPtr(D), LastHashTokPtr(0),
41 PPCond(ppcond), CurPPCondPtr(ppcond), PTHMgr(PM) {
43 FileStartLoc = PP.getSourceManager().getLocForStartOfFile(FID);
46 bool PTHLexer::Lex(Token& Tok) {
47 //===--------------------------------------==//
48 // Read the raw token data.
49 //===--------------------------------------==//
51 // Shadow CurPtr into an automatic variable.
52 const unsigned char *CurPtrShadow = CurPtr;
54 // Read in the data for the token.
55 unsigned Word0 = ReadLE32(CurPtrShadow);
56 uint32_t IdentifierID = ReadLE32(CurPtrShadow);
57 uint32_t FileOffset = ReadLE32(CurPtrShadow);
59 tok::TokenKind TKind = (tok::TokenKind) (Word0 & 0xFF);
60 Token::TokenFlags TFlags = (Token::TokenFlags) ((Word0 >> 8) & 0xFF);
61 uint32_t Len = Word0 >> 16;
63 CurPtr = CurPtrShadow;
65 //===--------------------------------------==//
66 // Construct the token itself.
67 //===--------------------------------------==//
72 assert(!LexingRawMode);
73 Tok.setLocation(FileStartLoc.getLocWithOffset(FileOffset));
76 // Handle identifiers.
77 if (Tok.isLiteral()) {
78 Tok.setLiteralData((const char*) (PTHMgr.SpellingBase + IdentifierID));
80 else if (IdentifierID) {
82 IdentifierInfo *II = PTHMgr.GetIdentifierInfo(IdentifierID-1);
84 Tok.setIdentifierInfo(II);
86 // Change the kind of this identifier to the appropriate token kind, e.g.
87 // turning "for" into a keyword.
88 Tok.setKind(II->getTokenID());
90 if (II->isHandleIdentifierCase())
91 return PP->HandleIdentifier(Tok);
96 //===--------------------------------------==//
98 //===--------------------------------------==//
99 if (TKind == tok::eof) {
100 // Save the end-of-file token.
103 assert(!ParsingPreprocessorDirective);
104 assert(!LexingRawMode);
106 return LexEndOfFile(Tok);
109 if (TKind == tok::hash && Tok.isAtStartOfLine()) {
110 LastHashTokPtr = CurPtr - DISK_TOKEN_SIZE;
111 assert(!LexingRawMode);
112 PP->HandleDirective(Tok);
117 if (TKind == tok::eod) {
118 assert(ParsingPreprocessorDirective);
119 ParsingPreprocessorDirective = false;
127 bool PTHLexer::LexEndOfFile(Token &Result) {
128 // If we hit the end of the file while parsing a preprocessor directive,
129 // end the preprocessor directive first. The next token returned will
130 // then be the end of file.
131 if (ParsingPreprocessorDirective) {
132 ParsingPreprocessorDirective = false; // Done parsing the "line".
133 return true; // Have a token.
136 assert(!LexingRawMode);
138 // If we are in a #if directive, emit an error.
139 while (!ConditionalStack.empty()) {
140 if (PP->getCodeCompletionFileLoc() != FileStartLoc)
141 PP->Diag(ConditionalStack.back().IfLoc,
142 diag::err_pp_unterminated_conditional);
143 ConditionalStack.pop_back();
146 // Finally, let the preprocessor handle this.
147 return PP->HandleEndOfFile(Result);
150 // FIXME: We can just grab the last token instead of storing a copy
152 void PTHLexer::getEOF(Token& Tok) {
153 assert(EofToken.is(tok::eof));
157 void PTHLexer::DiscardToEndOfLine() {
158 assert(ParsingPreprocessorDirective && ParsingFilename == false &&
159 "Must be in a preprocessing directive!");
161 // We assume that if the preprocessor wishes to discard to the end of
162 // the line that it also means to end the current preprocessor directive.
163 ParsingPreprocessorDirective = false;
165 // Skip tokens by only peeking at their token kind and the flags.
166 // We don't need to actually reconstruct full tokens from the token buffer.
167 // This saves some copies and it also reduces IdentifierInfo* lookup.
168 const unsigned char* p = CurPtr;
170 // Read the token kind. Are we at the end of the file?
171 tok::TokenKind x = (tok::TokenKind) (uint8_t) *p;
172 if (x == tok::eof) break;
174 // Read the token flags. Are we at the start of the next line?
175 Token::TokenFlags y = (Token::TokenFlags) (uint8_t) p[1];
176 if (y & Token::StartOfLine) break;
178 // Skip to the next token.
179 p += DISK_TOKEN_SIZE;
185 /// SkipBlock - Used by Preprocessor to skip the current conditional block.
186 bool PTHLexer::SkipBlock() {
187 assert(CurPPCondPtr && "No cached PP conditional information.");
188 assert(LastHashTokPtr && "No known '#' token.");
190 const unsigned char* HashEntryI = 0;
194 // Read the token offset from the side-table.
195 uint32_t Offset = ReadLE32(CurPPCondPtr);
197 // Read the target table index from the side-table.
198 TableIdx = ReadLE32(CurPPCondPtr);
200 // Compute the actual memory address of the '#' token data for this entry.
201 HashEntryI = TokBuf + Offset;
203 // Optmization: "Sibling jumping". #if...#else...#endif blocks can
204 // contain nested blocks. In the side-table we can jump over these
205 // nested blocks instead of doing a linear search if the next "sibling"
206 // entry is not at a location greater than LastHashTokPtr.
207 if (HashEntryI < LastHashTokPtr && TableIdx) {
208 // In the side-table we are still at an entry for a '#' token that
209 // is earlier than the last one we saw. Check if the location we would
210 // stride gets us closer.
211 const unsigned char* NextPPCondPtr =
212 PPCond + TableIdx*(sizeof(uint32_t)*2);
213 assert(NextPPCondPtr >= CurPPCondPtr);
214 // Read where we should jump to.
215 const unsigned char* HashEntryJ = TokBuf + ReadLE32(NextPPCondPtr);
217 if (HashEntryJ <= LastHashTokPtr) {
218 // Jump directly to the next entry in the side table.
219 HashEntryI = HashEntryJ;
220 TableIdx = ReadLE32(NextPPCondPtr);
221 CurPPCondPtr = NextPPCondPtr;
225 while (HashEntryI < LastHashTokPtr);
226 assert(HashEntryI == LastHashTokPtr && "No PP-cond entry found for '#'");
227 assert(TableIdx && "No jumping from #endifs.");
229 // Update our side-table iterator.
230 const unsigned char* NextPPCondPtr = PPCond + TableIdx*(sizeof(uint32_t)*2);
231 assert(NextPPCondPtr >= CurPPCondPtr);
232 CurPPCondPtr = NextPPCondPtr;
234 // Read where we should jump to.
235 HashEntryI = TokBuf + ReadLE32(NextPPCondPtr);
236 uint32_t NextIdx = ReadLE32(NextPPCondPtr);
238 // By construction NextIdx will be zero if this is a #endif. This is useful
239 // to know to obviate lexing another token.
240 bool isEndif = NextIdx == 0;
242 // This case can occur when we see something like this:
245 // /* a comment or nothing */
248 // If we are skipping the first #if block it will be the case that CurPtr
249 // already points 'elif'. Just return.
251 if (CurPtr > HashEntryI) {
252 assert(CurPtr == HashEntryI + DISK_TOKEN_SIZE);
253 // Did we reach a #endif? If so, go ahead and consume that token as well.
255 CurPtr += DISK_TOKEN_SIZE*2;
257 LastHashTokPtr = HashEntryI;
262 // Otherwise, we need to advance. Update CurPtr to point to the '#' token.
265 // Update the location of the last observed '#'. This is useful if we
266 // are skipping multiple blocks.
267 LastHashTokPtr = CurPtr;
269 // Skip the '#' token.
270 assert(((tok::TokenKind)*CurPtr) == tok::hash);
271 CurPtr += DISK_TOKEN_SIZE;
273 // Did we reach a #endif? If so, go ahead and consume that token as well.
274 if (isEndif) { CurPtr += DISK_TOKEN_SIZE*2; }
279 SourceLocation PTHLexer::getSourceLocation() {
280 // getSourceLocation is not on the hot path. It is used to get the location
281 // of the next token when transitioning back to this lexer when done
282 // handling a #included file. Just read the necessary data from the token
283 // data buffer to construct the SourceLocation object.
284 // NOTE: This is a virtual function; hence it is defined out-of-line.
285 const unsigned char *OffsetPtr = CurPtr + (DISK_TOKEN_SIZE - 4);
286 uint32_t Offset = ReadLE32(OffsetPtr);
287 return FileStartLoc.getLocWithOffset(Offset);
290 //===----------------------------------------------------------------------===//
291 // PTH file lookup: map from strings to file data.
292 //===----------------------------------------------------------------------===//
294 /// PTHFileLookup - This internal data structure is used by the PTHManager
295 /// to map from FileEntry objects managed by FileManager to offsets within
299 const uint32_t TokenOff;
300 const uint32_t PPCondOff;
302 PTHFileData(uint32_t tokenOff, uint32_t ppCondOff)
303 : TokenOff(tokenOff), PPCondOff(ppCondOff) {}
305 uint32_t getTokenOffset() const { return TokenOff; }
306 uint32_t getPPCondOffset() const { return PPCondOff; }
310 class PTHFileLookupCommonTrait {
312 typedef std::pair<unsigned char, const char*> internal_key_type;
314 static unsigned ComputeHash(internal_key_type x) {
315 return llvm::HashString(x.second);
318 static std::pair<unsigned, unsigned>
319 ReadKeyDataLength(const unsigned char*& d) {
320 unsigned keyLen = (unsigned) ReadUnalignedLE16(d);
321 unsigned dataLen = (unsigned) *(d++);
322 return std::make_pair(keyLen, dataLen);
325 static internal_key_type ReadKey(const unsigned char* d, unsigned) {
326 unsigned char k = *(d++); // Read the entry kind.
327 return std::make_pair(k, (const char*) d);
331 class PTHFileLookupTrait : public PTHFileLookupCommonTrait {
333 typedef const FileEntry* external_key_type;
334 typedef PTHFileData data_type;
336 static internal_key_type GetInternalKey(const FileEntry* FE) {
337 return std::make_pair((unsigned char) 0x1, FE->getName());
340 static bool EqualKey(internal_key_type a, internal_key_type b) {
341 return a.first == b.first && strcmp(a.second, b.second) == 0;
344 static PTHFileData ReadData(const internal_key_type& k,
345 const unsigned char* d, unsigned) {
346 assert(k.first == 0x1 && "Only file lookups can match!");
347 uint32_t x = ::ReadUnalignedLE32(d);
348 uint32_t y = ::ReadUnalignedLE32(d);
349 return PTHFileData(x, y);
353 class PTHStringLookupTrait {
358 typedef const std::pair<const char*, unsigned>
361 typedef external_key_type internal_key_type;
363 static bool EqualKey(const internal_key_type& a,
364 const internal_key_type& b) {
365 return (a.second == b.second) ? memcmp(a.first, b.first, a.second) == 0
369 static unsigned ComputeHash(const internal_key_type& a) {
370 return llvm::HashString(StringRef(a.first, a.second));
373 // This hopefully will just get inlined and removed by the optimizer.
374 static const internal_key_type&
375 GetInternalKey(const external_key_type& x) { return x; }
377 static std::pair<unsigned, unsigned>
378 ReadKeyDataLength(const unsigned char*& d) {
379 return std::make_pair((unsigned) ReadUnalignedLE16(d), sizeof(uint32_t));
382 static std::pair<const char*, unsigned>
383 ReadKey(const unsigned char* d, unsigned n) {
384 assert(n >= 2 && d[n-1] == '\0');
385 return std::make_pair((const char*) d, n-1);
388 static uint32_t ReadData(const internal_key_type& k, const unsigned char* d,
390 return ::ReadUnalignedLE32(d);
394 } // end anonymous namespace
396 typedef OnDiskChainedHashTable<PTHFileLookupTrait> PTHFileLookup;
397 typedef OnDiskChainedHashTable<PTHStringLookupTrait> PTHStringIdLookup;
399 //===----------------------------------------------------------------------===//
400 // PTHManager methods.
401 //===----------------------------------------------------------------------===//
403 PTHManager::PTHManager(const llvm::MemoryBuffer* buf, void* fileLookup,
404 const unsigned char* idDataTable,
405 IdentifierInfo** perIDCache,
406 void* stringIdLookup, unsigned numIds,
407 const unsigned char* spellingBase,
408 const char* originalSourceFile)
409 : Buf(buf), PerIDCache(perIDCache), FileLookup(fileLookup),
410 IdDataTable(idDataTable), StringIdLookup(stringIdLookup),
411 NumIds(numIds), PP(0), SpellingBase(spellingBase),
412 OriginalSourceFile(originalSourceFile) {}
414 PTHManager::~PTHManager() {
416 delete (PTHFileLookup*) FileLookup;
417 delete (PTHStringIdLookup*) StringIdLookup;
421 static void InvalidPTH(DiagnosticsEngine &Diags, const char *Msg) {
422 Diags.Report(Diags.getCustomDiagID(DiagnosticsEngine::Error, Msg));
425 PTHManager *PTHManager::Create(const std::string &file,
426 DiagnosticsEngine &Diags) {
427 // Memory map the PTH file.
428 OwningPtr<llvm::MemoryBuffer> File;
430 if (llvm::MemoryBuffer::getFile(file, File)) {
431 // FIXME: Add ec.message() to this diag.
432 Diags.Report(diag::err_invalid_pth_file) << file;
436 // Get the buffer ranges and check if there are at least three 32-bit
437 // words at the end of the file.
438 const unsigned char *BufBeg = (const unsigned char*)File->getBufferStart();
439 const unsigned char *BufEnd = (const unsigned char*)File->getBufferEnd();
441 // Check the prologue of the file.
442 if ((BufEnd - BufBeg) < (signed)(sizeof("cfe-pth") + 4 + 4) ||
443 memcmp(BufBeg, "cfe-pth", sizeof("cfe-pth")) != 0) {
444 Diags.Report(diag::err_invalid_pth_file) << file;
448 // Read the PTH version.
449 const unsigned char *p = BufBeg + (sizeof("cfe-pth"));
450 unsigned Version = ReadLE32(p);
452 if (Version < PTHManager::Version) {
454 Version < PTHManager::Version
455 ? "PTH file uses an older PTH format that is no longer supported"
456 : "PTH file uses a newer PTH format that cannot be read");
460 // Compute the address of the index table at the end of the PTH file.
461 const unsigned char *PrologueOffset = p;
463 if (PrologueOffset >= BufEnd) {
464 Diags.Report(diag::err_invalid_pth_file) << file;
468 // Construct the file lookup table. This will be used for mapping from
469 // FileEntry*'s to cached tokens.
470 const unsigned char* FileTableOffset = PrologueOffset + sizeof(uint32_t)*2;
471 const unsigned char* FileTable = BufBeg + ReadLE32(FileTableOffset);
473 if (!(FileTable > BufBeg && FileTable < BufEnd)) {
474 Diags.Report(diag::err_invalid_pth_file) << file;
475 return 0; // FIXME: Proper error diagnostic?
478 OwningPtr<PTHFileLookup> FL(PTHFileLookup::Create(FileTable, BufBeg));
480 // Warn if the PTH file is empty. We still want to create a PTHManager
481 // as the PTH could be used with -include-pth.
483 InvalidPTH(Diags, "PTH file contains no cached source data");
485 // Get the location of the table mapping from persistent ids to the
486 // data needed to reconstruct identifiers.
487 const unsigned char* IDTableOffset = PrologueOffset + sizeof(uint32_t)*0;
488 const unsigned char* IData = BufBeg + ReadLE32(IDTableOffset);
490 if (!(IData >= BufBeg && IData < BufEnd)) {
491 Diags.Report(diag::err_invalid_pth_file) << file;
495 // Get the location of the hashtable mapping between strings and
497 const unsigned char* StringIdTableOffset = PrologueOffset + sizeof(uint32_t)*1;
498 const unsigned char* StringIdTable = BufBeg + ReadLE32(StringIdTableOffset);
499 if (!(StringIdTable >= BufBeg && StringIdTable < BufEnd)) {
500 Diags.Report(diag::err_invalid_pth_file) << file;
504 OwningPtr<PTHStringIdLookup> SL(PTHStringIdLookup::Create(StringIdTable,
507 // Get the location of the spelling cache.
508 const unsigned char* spellingBaseOffset = PrologueOffset + sizeof(uint32_t)*3;
509 const unsigned char* spellingBase = BufBeg + ReadLE32(spellingBaseOffset);
510 if (!(spellingBase >= BufBeg && spellingBase < BufEnd)) {
511 Diags.Report(diag::err_invalid_pth_file) << file;
515 // Get the number of IdentifierInfos and pre-allocate the identifier cache.
516 uint32_t NumIds = ReadLE32(IData);
518 // Pre-allocate the persistent ID -> IdentifierInfo* cache. We use calloc()
519 // so that we in the best case only zero out memory once when the OS returns
521 IdentifierInfo** PerIDCache = 0;
524 PerIDCache = (IdentifierInfo**)calloc(NumIds, sizeof(*PerIDCache));
526 InvalidPTH(Diags, "Could not allocate memory for processing PTH file");
531 // Compute the address of the original source file.
532 const unsigned char* originalSourceBase = PrologueOffset + sizeof(uint32_t)*4;
533 unsigned len = ReadUnalignedLE16(originalSourceBase);
534 if (!len) originalSourceBase = 0;
536 // Create the new PTHManager.
537 return new PTHManager(File.take(), FL.take(), IData, PerIDCache,
538 SL.take(), NumIds, spellingBase,
539 (const char*) originalSourceBase);
542 IdentifierInfo* PTHManager::LazilyCreateIdentifierInfo(unsigned PersistentID) {
543 // Look in the PTH file for the string data for the IdentifierInfo object.
544 const unsigned char* TableEntry = IdDataTable + sizeof(uint32_t)*PersistentID;
545 const unsigned char* IDData =
546 (const unsigned char*)Buf->getBufferStart() + ReadLE32(TableEntry);
547 assert(IDData < (const unsigned char*)Buf->getBufferEnd());
549 // Allocate the object.
550 std::pair<IdentifierInfo,const unsigned char*> *Mem =
551 Alloc.Allocate<std::pair<IdentifierInfo,const unsigned char*> >();
553 Mem->second = IDData;
554 assert(IDData[0] != '\0');
555 IdentifierInfo *II = new ((void*) Mem) IdentifierInfo();
557 // Store the new IdentifierInfo in the cache.
558 PerIDCache[PersistentID] = II;
559 assert(II->getNameStart() && II->getNameStart()[0] != '\0');
563 IdentifierInfo* PTHManager::get(StringRef Name) {
564 PTHStringIdLookup& SL = *((PTHStringIdLookup*)StringIdLookup);
565 // Double check our assumption that the last character isn't '\0'.
566 assert(Name.empty() || Name.back() != '\0');
567 PTHStringIdLookup::iterator I = SL.find(std::make_pair(Name.data(),
569 if (I == SL.end()) // No identifier found?
572 // Match found. Return the identifier!
574 return GetIdentifierInfo(*I-1);
577 PTHLexer *PTHManager::CreateLexer(FileID FID) {
578 const FileEntry *FE = PP->getSourceManager().getFileEntryForID(FID);
582 // Lookup the FileEntry object in our file lookup data structure. It will
583 // return a variant that indicates whether or not there is an offset within
584 // the PTH file that contains cached tokens.
585 PTHFileLookup& PFL = *((PTHFileLookup*)FileLookup);
586 PTHFileLookup::iterator I = PFL.find(FE);
588 if (I == PFL.end()) // No tokens available?
591 const PTHFileData& FileData = *I;
593 const unsigned char *BufStart = (const unsigned char *)Buf->getBufferStart();
594 // Compute the offset of the token data within the buffer.
595 const unsigned char* data = BufStart + FileData.getTokenOffset();
597 // Get the location of pp-conditional table.
598 const unsigned char* ppcond = BufStart + FileData.getPPCondOffset();
599 uint32_t Len = ReadLE32(ppcond);
600 if (Len == 0) ppcond = 0;
602 assert(PP && "No preprocessor set yet!");
603 return new PTHLexer(*PP, FID, data, ppcond, *this);
606 //===----------------------------------------------------------------------===//
608 //===----------------------------------------------------------------------===//
616 llvm::sys::fs::UniqueID UniqueID;
619 PTHStatData(uint64_t Size, time_t ModTime, llvm::sys::fs::UniqueID UniqueID,
621 : HasData(true), Size(Size), ModTime(ModTime), UniqueID(UniqueID),
622 IsDirectory(IsDirectory) {}
624 PTHStatData() : HasData(false) {}
627 class PTHStatLookupTrait : public PTHFileLookupCommonTrait {
629 typedef const char* external_key_type; // const char*
630 typedef PTHStatData data_type;
632 static internal_key_type GetInternalKey(const char *path) {
633 // The key 'kind' doesn't matter here because it is ignored in EqualKey.
634 return std::make_pair((unsigned char) 0x0, path);
637 static bool EqualKey(internal_key_type a, internal_key_type b) {
638 // When doing 'stat' lookups we don't care about the kind of 'a' and 'b',
640 return strcmp(a.second, b.second) == 0;
643 static data_type ReadData(const internal_key_type& k, const unsigned char* d,
646 if (k.first /* File or Directory */) {
647 bool IsDirectory = true;
648 if (k.first == 0x1 /* File */) {
650 d += 4 * 2; // Skip the first 2 words.
653 uint64_t File = ReadUnalignedLE64(d);
654 uint64_t Device = ReadUnalignedLE64(d);
655 llvm::sys::fs::UniqueID UniqueID(File, Device);
656 time_t ModTime = ReadUnalignedLE64(d);
657 uint64_t Size = ReadUnalignedLE64(d);
658 return data_type(Size, ModTime, UniqueID, IsDirectory);
661 // Negative stat. Don't read anything.
666 class PTHStatCache : public FileSystemStatCache {
667 typedef OnDiskChainedHashTable<PTHStatLookupTrait> CacheTy;
671 PTHStatCache(PTHFileLookup &FL) :
672 Cache(FL.getNumBuckets(), FL.getNumEntries(), FL.getBuckets(),
677 LookupResult getStat(const char *Path, FileData &Data, bool isFile,
678 int *FileDescriptor) {
679 // Do the lookup for the file's data in the PTH file.
680 CacheTy::iterator I = Cache.find(Path);
682 // If we don't get a hit in the PTH file just forward to 'stat'.
683 if (I == Cache.end())
684 return statChained(Path, Data, isFile, FileDescriptor);
686 const PTHStatData &D = *I;
692 Data.ModTime = D.ModTime;
693 Data.UniqueID = D.UniqueID;
694 Data.IsDirectory = D.IsDirectory;
695 Data.IsNamedPipe = false;
701 } // end anonymous namespace
703 FileSystemStatCache *PTHManager::createStatCache() {
704 return new PTHStatCache(*((PTHFileLookup*) FileLookup));