1 //===- VirtualFileSystem.cpp - Virtual File System Layer ------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file implements the VirtualFileSystem interface.
12 //===----------------------------------------------------------------------===//
14 #include "llvm/Support/VirtualFileSystem.h"
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/DenseMap.h"
17 #include "llvm/ADT/IntrusiveRefCntPtr.h"
18 #include "llvm/ADT/None.h"
19 #include "llvm/ADT/Optional.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/SmallString.h"
22 #include "llvm/ADT/SmallVector.h"
23 #include "llvm/ADT/StringRef.h"
24 #include "llvm/ADT/StringSet.h"
25 #include "llvm/ADT/Twine.h"
26 #include "llvm/ADT/iterator_range.h"
27 #include "llvm/Config/llvm-config.h"
28 #include "llvm/Support/Casting.h"
29 #include "llvm/Support/Chrono.h"
30 #include "llvm/Support/Compiler.h"
31 #include "llvm/Support/Debug.h"
32 #include "llvm/Support/Errc.h"
33 #include "llvm/Support/ErrorHandling.h"
34 #include "llvm/Support/ErrorOr.h"
35 #include "llvm/Support/FileSystem.h"
36 #include "llvm/Support/MemoryBuffer.h"
37 #include "llvm/Support/Path.h"
38 #include "llvm/Support/Process.h"
39 #include "llvm/Support/SMLoc.h"
40 #include "llvm/Support/SourceMgr.h"
41 #include "llvm/Support/YAMLParser.h"
42 #include "llvm/Support/raw_ostream.h"
53 #include <system_error>
58 using namespace llvm::vfs;
60 using llvm::sys::fs::file_status;
61 using llvm::sys::fs::file_type;
62 using llvm::sys::fs::perms;
63 using llvm::sys::fs::UniqueID;
65 Status::Status(const file_status &Status)
66 : UID(Status.getUniqueID()), MTime(Status.getLastModificationTime()),
67 User(Status.getUser()), Group(Status.getGroup()), Size(Status.getSize()),
68 Type(Status.type()), Perms(Status.permissions()) {}
70 Status::Status(StringRef Name, UniqueID UID, sys::TimePoint<> MTime,
71 uint32_t User, uint32_t Group, uint64_t Size, file_type Type,
73 : Name(Name), UID(UID), MTime(MTime), User(User), Group(Group), Size(Size),
74 Type(Type), Perms(Perms) {}
76 Status Status::copyWithNewName(const Status &In, StringRef NewName) {
77 return Status(NewName, In.getUniqueID(), In.getLastModificationTime(),
78 In.getUser(), In.getGroup(), In.getSize(), In.getType(),
82 Status Status::copyWithNewName(const file_status &In, StringRef NewName) {
83 return Status(NewName, In.getUniqueID(), In.getLastModificationTime(),
84 In.getUser(), In.getGroup(), In.getSize(), In.type(),
88 bool Status::equivalent(const Status &Other) const {
89 assert(isStatusKnown() && Other.isStatusKnown());
90 return getUniqueID() == Other.getUniqueID();
93 bool Status::isDirectory() const { return Type == file_type::directory_file; }
95 bool Status::isRegularFile() const { return Type == file_type::regular_file; }
97 bool Status::isOther() const {
98 return exists() && !isRegularFile() && !isDirectory() && !isSymlink();
101 bool Status::isSymlink() const { return Type == file_type::symlink_file; }
103 bool Status::isStatusKnown() const { return Type != file_type::status_error; }
105 bool Status::exists() const {
106 return isStatusKnown() && Type != file_type::file_not_found;
109 File::~File() = default;
111 FileSystem::~FileSystem() = default;
113 ErrorOr<std::unique_ptr<MemoryBuffer>>
114 FileSystem::getBufferForFile(const llvm::Twine &Name, int64_t FileSize,
115 bool RequiresNullTerminator, bool IsVolatile) {
116 auto F = openFileForRead(Name);
120 return (*F)->getBuffer(Name, FileSize, RequiresNullTerminator, IsVolatile);
123 std::error_code FileSystem::makeAbsolute(SmallVectorImpl<char> &Path) const {
124 if (llvm::sys::path::is_absolute(Path))
127 auto WorkingDir = getCurrentWorkingDirectory();
129 return WorkingDir.getError();
131 llvm::sys::fs::make_absolute(WorkingDir.get(), Path);
135 std::error_code FileSystem::getRealPath(const Twine &Path,
136 SmallVectorImpl<char> &Output) const {
137 return errc::operation_not_permitted;
140 std::error_code FileSystem::isLocal(const Twine &Path, bool &Result) {
141 return errc::operation_not_permitted;
144 bool FileSystem::exists(const Twine &Path) {
145 auto Status = status(Path);
146 return Status && Status->exists();
150 static bool isTraversalComponent(StringRef Component) {
151 return Component.equals("..") || Component.equals(".");
154 static bool pathHasTraversal(StringRef Path) {
155 using namespace llvm::sys;
157 for (StringRef Comp : llvm::make_range(path::begin(Path), path::end(Path)))
158 if (isTraversalComponent(Comp))
164 //===-----------------------------------------------------------------------===/
165 // RealFileSystem implementation
166 //===-----------------------------------------------------------------------===/
170 /// Wrapper around a raw file descriptor.
171 class RealFile : public File {
172 friend class RealFileSystem;
176 std::string RealName;
178 RealFile(int FD, StringRef NewName, StringRef NewRealPathName)
179 : FD(FD), S(NewName, {}, {}, {}, {}, {},
180 llvm::sys::fs::file_type::status_error, {}),
181 RealName(NewRealPathName.str()) {
182 assert(FD >= 0 && "Invalid or inactive file descriptor");
186 ~RealFile() override;
188 ErrorOr<Status> status() override;
189 ErrorOr<std::string> getName() override;
190 ErrorOr<std::unique_ptr<MemoryBuffer>> getBuffer(const Twine &Name,
192 bool RequiresNullTerminator,
193 bool IsVolatile) override;
194 std::error_code close() override;
199 RealFile::~RealFile() { close(); }
201 ErrorOr<Status> RealFile::status() {
202 assert(FD != -1 && "cannot stat closed file");
203 if (!S.isStatusKnown()) {
204 file_status RealStatus;
205 if (std::error_code EC = sys::fs::status(FD, RealStatus))
207 S = Status::copyWithNewName(RealStatus, S.getName());
212 ErrorOr<std::string> RealFile::getName() {
213 return RealName.empty() ? S.getName().str() : RealName;
216 ErrorOr<std::unique_ptr<MemoryBuffer>>
217 RealFile::getBuffer(const Twine &Name, int64_t FileSize,
218 bool RequiresNullTerminator, bool IsVolatile) {
219 assert(FD != -1 && "cannot get buffer for closed file");
220 return MemoryBuffer::getOpenFile(FD, Name, FileSize, RequiresNullTerminator,
224 std::error_code RealFile::close() {
225 std::error_code EC = sys::Process::SafelyCloseFileDescriptor(FD);
232 /// The file system according to your operating system.
233 class RealFileSystem : public FileSystem {
235 ErrorOr<Status> status(const Twine &Path) override;
236 ErrorOr<std::unique_ptr<File>> openFileForRead(const Twine &Path) override;
237 directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override;
239 llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override;
240 std::error_code setCurrentWorkingDirectory(const Twine &Path) override;
241 std::error_code isLocal(const Twine &Path, bool &Result) override;
242 std::error_code getRealPath(const Twine &Path,
243 SmallVectorImpl<char> &Output) const override;
246 mutable std::mutex CWDMutex;
247 mutable std::string CWDCache;
252 ErrorOr<Status> RealFileSystem::status(const Twine &Path) {
253 sys::fs::file_status RealStatus;
254 if (std::error_code EC = sys::fs::status(Path, RealStatus))
256 return Status::copyWithNewName(RealStatus, Path.str());
259 ErrorOr<std::unique_ptr<File>>
260 RealFileSystem::openFileForRead(const Twine &Name) {
262 SmallString<256> RealName;
263 if (std::error_code EC =
264 sys::fs::openFileForRead(Name, FD, sys::fs::OF_None, &RealName))
266 return std::unique_ptr<File>(new RealFile(FD, Name.str(), RealName.str()));
269 llvm::ErrorOr<std::string> RealFileSystem::getCurrentWorkingDirectory() const {
270 std::lock_guard<std::mutex> Lock(CWDMutex);
271 if (!CWDCache.empty())
273 SmallString<256> Dir;
274 if (std::error_code EC = llvm::sys::fs::current_path(Dir))
276 CWDCache = Dir.str();
280 std::error_code RealFileSystem::setCurrentWorkingDirectory(const Twine &Path) {
281 // FIXME: chdir is thread hostile; on the other hand, creating the same
282 // behavior as chdir is complex: chdir resolves the path once, thus
283 // guaranteeing that all subsequent relative path operations work
284 // on the same path the original chdir resulted in. This makes a
285 // difference for example on network filesystems, where symlinks might be
286 // switched during runtime of the tool. Fixing this depends on having a
287 // file system abstraction that allows openat() style interactions.
288 if (auto EC = llvm::sys::fs::set_current_path(Path))
292 std::lock_guard<std::mutex> Lock(CWDMutex);
294 return std::error_code();
297 std::error_code RealFileSystem::isLocal(const Twine &Path, bool &Result) {
298 return llvm::sys::fs::is_local(Path, Result);
302 RealFileSystem::getRealPath(const Twine &Path,
303 SmallVectorImpl<char> &Output) const {
304 return llvm::sys::fs::real_path(Path, Output);
307 IntrusiveRefCntPtr<FileSystem> vfs::getRealFileSystem() {
308 static IntrusiveRefCntPtr<FileSystem> FS = new RealFileSystem();
314 class RealFSDirIter : public llvm::vfs::detail::DirIterImpl {
315 llvm::sys::fs::directory_iterator Iter;
318 RealFSDirIter(const Twine &Path, std::error_code &EC) : Iter(Path, EC) {
319 if (Iter != llvm::sys::fs::directory_iterator())
320 CurrentEntry = directory_entry(Iter->path(), Iter->type());
323 std::error_code increment() override {
326 CurrentEntry = (Iter == llvm::sys::fs::directory_iterator())
328 : directory_entry(Iter->path(), Iter->type());
335 directory_iterator RealFileSystem::dir_begin(const Twine &Dir,
336 std::error_code &EC) {
337 return directory_iterator(std::make_shared<RealFSDirIter>(Dir, EC));
340 //===-----------------------------------------------------------------------===/
341 // OverlayFileSystem implementation
342 //===-----------------------------------------------------------------------===/
344 OverlayFileSystem::OverlayFileSystem(IntrusiveRefCntPtr<FileSystem> BaseFS) {
345 FSList.push_back(std::move(BaseFS));
348 void OverlayFileSystem::pushOverlay(IntrusiveRefCntPtr<FileSystem> FS) {
349 FSList.push_back(FS);
350 // Synchronize added file systems by duplicating the working directory from
351 // the first one in the list.
352 FS->setCurrentWorkingDirectory(getCurrentWorkingDirectory().get());
355 ErrorOr<Status> OverlayFileSystem::status(const Twine &Path) {
356 // FIXME: handle symlinks that cross file systems
357 for (iterator I = overlays_begin(), E = overlays_end(); I != E; ++I) {
358 ErrorOr<Status> Status = (*I)->status(Path);
359 if (Status || Status.getError() != llvm::errc::no_such_file_or_directory)
362 return make_error_code(llvm::errc::no_such_file_or_directory);
365 ErrorOr<std::unique_ptr<File>>
366 OverlayFileSystem::openFileForRead(const llvm::Twine &Path) {
367 // FIXME: handle symlinks that cross file systems
368 for (iterator I = overlays_begin(), E = overlays_end(); I != E; ++I) {
369 auto Result = (*I)->openFileForRead(Path);
370 if (Result || Result.getError() != llvm::errc::no_such_file_or_directory)
373 return make_error_code(llvm::errc::no_such_file_or_directory);
376 llvm::ErrorOr<std::string>
377 OverlayFileSystem::getCurrentWorkingDirectory() const {
378 // All file systems are synchronized, just take the first working directory.
379 return FSList.front()->getCurrentWorkingDirectory();
383 OverlayFileSystem::setCurrentWorkingDirectory(const Twine &Path) {
384 for (auto &FS : FSList)
385 if (std::error_code EC = FS->setCurrentWorkingDirectory(Path))
390 std::error_code OverlayFileSystem::isLocal(const Twine &Path, bool &Result) {
391 for (auto &FS : FSList)
392 if (FS->exists(Path))
393 return FS->isLocal(Path, Result);
394 return errc::no_such_file_or_directory;
398 OverlayFileSystem::getRealPath(const Twine &Path,
399 SmallVectorImpl<char> &Output) const {
400 for (auto &FS : FSList)
401 if (FS->exists(Path))
402 return FS->getRealPath(Path, Output);
403 return errc::no_such_file_or_directory;
406 llvm::vfs::detail::DirIterImpl::~DirIterImpl() = default;
410 class OverlayFSDirIterImpl : public llvm::vfs::detail::DirIterImpl {
411 OverlayFileSystem &Overlays;
413 OverlayFileSystem::iterator CurrentFS;
414 directory_iterator CurrentDirIter;
415 llvm::StringSet<> SeenNames;
417 std::error_code incrementFS() {
418 assert(CurrentFS != Overlays.overlays_end() && "incrementing past end");
420 for (auto E = Overlays.overlays_end(); CurrentFS != E; ++CurrentFS) {
422 CurrentDirIter = (*CurrentFS)->dir_begin(Path, EC);
423 if (EC && EC != errc::no_such_file_or_directory)
425 if (CurrentDirIter != directory_iterator())
431 std::error_code incrementDirIter(bool IsFirstTime) {
432 assert((IsFirstTime || CurrentDirIter != directory_iterator()) &&
433 "incrementing past end");
436 CurrentDirIter.increment(EC);
437 if (!EC && CurrentDirIter == directory_iterator())
442 std::error_code incrementImpl(bool IsFirstTime) {
444 std::error_code EC = incrementDirIter(IsFirstTime);
445 if (EC || CurrentDirIter == directory_iterator()) {
446 CurrentEntry = directory_entry();
449 CurrentEntry = *CurrentDirIter;
450 StringRef Name = llvm::sys::path::filename(CurrentEntry.path());
451 if (SeenNames.insert(Name).second)
452 return EC; // name not seen before
454 llvm_unreachable("returned above");
458 OverlayFSDirIterImpl(const Twine &Path, OverlayFileSystem &FS,
460 : Overlays(FS), Path(Path.str()), CurrentFS(Overlays.overlays_begin()) {
461 CurrentDirIter = (*CurrentFS)->dir_begin(Path, EC);
462 EC = incrementImpl(true);
465 std::error_code increment() override { return incrementImpl(false); }
470 directory_iterator OverlayFileSystem::dir_begin(const Twine &Dir,
471 std::error_code &EC) {
472 return directory_iterator(
473 std::make_shared<OverlayFSDirIterImpl>(Dir, *this, EC));
476 void ProxyFileSystem::anchor() {}
483 enum InMemoryNodeKind { IME_File, IME_Directory, IME_HardLink };
485 /// The in memory file system is a tree of Nodes. Every node can either be a
486 /// file , hardlink or a directory.
488 InMemoryNodeKind Kind;
489 std::string FileName;
492 InMemoryNode(llvm::StringRef FileName, InMemoryNodeKind Kind)
493 : Kind(Kind), FileName(llvm::sys::path::filename(FileName)) {}
494 virtual ~InMemoryNode() = default;
496 /// Get the filename of this node (the name without the directory part).
497 StringRef getFileName() const { return FileName; }
498 InMemoryNodeKind getKind() const { return Kind; }
499 virtual std::string toString(unsigned Indent) const = 0;
502 class InMemoryFile : public InMemoryNode {
504 std::unique_ptr<llvm::MemoryBuffer> Buffer;
507 InMemoryFile(Status Stat, std::unique_ptr<llvm::MemoryBuffer> Buffer)
508 : InMemoryNode(Stat.getName(), IME_File), Stat(std::move(Stat)),
509 Buffer(std::move(Buffer)) {}
511 /// Return the \p Status for this node. \p RequestedName should be the name
512 /// through which the caller referred to this node. It will override
513 /// \p Status::Name in the return value, to mimic the behavior of \p RealFile.
514 Status getStatus(StringRef RequestedName) const {
515 return Status::copyWithNewName(Stat, RequestedName);
517 llvm::MemoryBuffer *getBuffer() const { return Buffer.get(); }
519 std::string toString(unsigned Indent) const override {
520 return (std::string(Indent, ' ') + Stat.getName() + "\n").str();
523 static bool classof(const InMemoryNode *N) {
524 return N->getKind() == IME_File;
530 class InMemoryHardLink : public InMemoryNode {
531 const InMemoryFile &ResolvedFile;
534 InMemoryHardLink(StringRef Path, const InMemoryFile &ResolvedFile)
535 : InMemoryNode(Path, IME_HardLink), ResolvedFile(ResolvedFile) {}
536 const InMemoryFile &getResolvedFile() const { return ResolvedFile; }
538 std::string toString(unsigned Indent) const override {
539 return std::string(Indent, ' ') + "HardLink to -> " +
540 ResolvedFile.toString(0);
543 static bool classof(const InMemoryNode *N) {
544 return N->getKind() == IME_HardLink;
548 /// Adapt a InMemoryFile for VFS' File interface. The goal is to make
549 /// \p InMemoryFileAdaptor mimic as much as possible the behavior of
551 class InMemoryFileAdaptor : public File {
552 const InMemoryFile &Node;
553 /// The name to use when returning a Status for this file.
554 std::string RequestedName;
557 explicit InMemoryFileAdaptor(const InMemoryFile &Node,
558 std::string RequestedName)
559 : Node(Node), RequestedName(std::move(RequestedName)) {}
561 llvm::ErrorOr<Status> status() override {
562 return Node.getStatus(RequestedName);
565 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
566 getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator,
567 bool IsVolatile) override {
568 llvm::MemoryBuffer *Buf = Node.getBuffer();
569 return llvm::MemoryBuffer::getMemBuffer(
570 Buf->getBuffer(), Buf->getBufferIdentifier(), RequiresNullTerminator);
573 std::error_code close() override { return {}; }
577 class InMemoryDirectory : public InMemoryNode {
579 llvm::StringMap<std::unique_ptr<InMemoryNode>> Entries;
582 InMemoryDirectory(Status Stat)
583 : InMemoryNode(Stat.getName(), IME_Directory), Stat(std::move(Stat)) {}
585 /// Return the \p Status for this node. \p RequestedName should be the name
586 /// through which the caller referred to this node. It will override
587 /// \p Status::Name in the return value, to mimic the behavior of \p RealFile.
588 Status getStatus(StringRef RequestedName) const {
589 return Status::copyWithNewName(Stat, RequestedName);
591 InMemoryNode *getChild(StringRef Name) {
592 auto I = Entries.find(Name);
593 if (I != Entries.end())
594 return I->second.get();
598 InMemoryNode *addChild(StringRef Name, std::unique_ptr<InMemoryNode> Child) {
599 return Entries.insert(make_pair(Name, std::move(Child)))
600 .first->second.get();
603 using const_iterator = decltype(Entries)::const_iterator;
605 const_iterator begin() const { return Entries.begin(); }
606 const_iterator end() const { return Entries.end(); }
608 std::string toString(unsigned Indent) const override {
610 (std::string(Indent, ' ') + Stat.getName() + "\n").str();
611 for (const auto &Entry : Entries)
612 Result += Entry.second->toString(Indent + 2);
616 static bool classof(const InMemoryNode *N) {
617 return N->getKind() == IME_Directory;
622 Status getNodeStatus(const InMemoryNode *Node, StringRef RequestedName) {
623 if (auto Dir = dyn_cast<detail::InMemoryDirectory>(Node))
624 return Dir->getStatus(RequestedName);
625 if (auto File = dyn_cast<detail::InMemoryFile>(Node))
626 return File->getStatus(RequestedName);
627 if (auto Link = dyn_cast<detail::InMemoryHardLink>(Node))
628 return Link->getResolvedFile().getStatus(RequestedName);
629 llvm_unreachable("Unknown node type");
632 } // namespace detail
634 InMemoryFileSystem::InMemoryFileSystem(bool UseNormalizedPaths)
635 : Root(new detail::InMemoryDirectory(
636 Status("", getNextVirtualUniqueID(), llvm::sys::TimePoint<>(), 0, 0,
637 0, llvm::sys::fs::file_type::directory_file,
638 llvm::sys::fs::perms::all_all))),
639 UseNormalizedPaths(UseNormalizedPaths) {}
641 InMemoryFileSystem::~InMemoryFileSystem() = default;
643 std::string InMemoryFileSystem::toString() const {
644 return Root->toString(/*Indent=*/0);
647 bool InMemoryFileSystem::addFile(const Twine &P, time_t ModificationTime,
648 std::unique_ptr<llvm::MemoryBuffer> Buffer,
649 Optional<uint32_t> User,
650 Optional<uint32_t> Group,
651 Optional<llvm::sys::fs::file_type> Type,
652 Optional<llvm::sys::fs::perms> Perms,
653 const detail::InMemoryFile *HardLinkTarget) {
654 SmallString<128> Path;
657 // Fix up relative paths. This just prepends the current working directory.
658 std::error_code EC = makeAbsolute(Path);
662 if (useNormalizedPaths())
663 llvm::sys::path::remove_dots(Path, /*remove_dot_dot=*/true);
668 detail::InMemoryDirectory *Dir = Root.get();
669 auto I = llvm::sys::path::begin(Path), E = sys::path::end(Path);
670 const auto ResolvedUser = User.getValueOr(0);
671 const auto ResolvedGroup = Group.getValueOr(0);
672 const auto ResolvedType = Type.getValueOr(sys::fs::file_type::regular_file);
673 const auto ResolvedPerms = Perms.getValueOr(sys::fs::all_all);
674 assert(!(HardLinkTarget && Buffer) && "HardLink cannot have a buffer");
675 // Any intermediate directories we create should be accessible by
676 // the owner, even if Perms says otherwise for the final path.
677 const auto NewDirectoryPerms = ResolvedPerms | sys::fs::owner_all;
680 detail::InMemoryNode *Node = Dir->getChild(Name);
685 std::unique_ptr<detail::InMemoryNode> Child;
687 Child.reset(new detail::InMemoryHardLink(P.str(), *HardLinkTarget));
689 // Create a new file or directory.
690 Status Stat(P.str(), getNextVirtualUniqueID(),
691 llvm::sys::toTimePoint(ModificationTime), ResolvedUser,
692 ResolvedGroup, Buffer->getBufferSize(), ResolvedType,
694 if (ResolvedType == sys::fs::file_type::directory_file) {
695 Child.reset(new detail::InMemoryDirectory(std::move(Stat)));
698 new detail::InMemoryFile(std::move(Stat), std::move(Buffer)));
701 Dir->addChild(Name, std::move(Child));
705 // Create a new directory. Use the path up to here.
707 StringRef(Path.str().begin(), Name.end() - Path.str().begin()),
708 getNextVirtualUniqueID(), llvm::sys::toTimePoint(ModificationTime),
709 ResolvedUser, ResolvedGroup, 0, sys::fs::file_type::directory_file,
711 Dir = cast<detail::InMemoryDirectory>(Dir->addChild(
712 Name, llvm::make_unique<detail::InMemoryDirectory>(std::move(Stat))));
716 if (auto *NewDir = dyn_cast<detail::InMemoryDirectory>(Node)) {
719 assert((isa<detail::InMemoryFile>(Node) ||
720 isa<detail::InMemoryHardLink>(Node)) &&
721 "Must be either file, hardlink or directory!");
723 // Trying to insert a directory in place of a file.
727 // Return false only if the new file is different from the existing one.
728 if (auto Link = dyn_cast<detail::InMemoryHardLink>(Node)) {
729 return Link->getResolvedFile().getBuffer()->getBuffer() ==
732 return cast<detail::InMemoryFile>(Node)->getBuffer()->getBuffer() ==
738 bool InMemoryFileSystem::addFile(const Twine &P, time_t ModificationTime,
739 std::unique_ptr<llvm::MemoryBuffer> Buffer,
740 Optional<uint32_t> User,
741 Optional<uint32_t> Group,
742 Optional<llvm::sys::fs::file_type> Type,
743 Optional<llvm::sys::fs::perms> Perms) {
744 return addFile(P, ModificationTime, std::move(Buffer), User, Group, Type,
745 Perms, /*HardLinkTarget=*/nullptr);
748 bool InMemoryFileSystem::addFileNoOwn(const Twine &P, time_t ModificationTime,
749 llvm::MemoryBuffer *Buffer,
750 Optional<uint32_t> User,
751 Optional<uint32_t> Group,
752 Optional<llvm::sys::fs::file_type> Type,
753 Optional<llvm::sys::fs::perms> Perms) {
754 return addFile(P, ModificationTime,
755 llvm::MemoryBuffer::getMemBuffer(
756 Buffer->getBuffer(), Buffer->getBufferIdentifier()),
757 std::move(User), std::move(Group), std::move(Type),
761 static ErrorOr<const detail::InMemoryNode *>
762 lookupInMemoryNode(const InMemoryFileSystem &FS, detail::InMemoryDirectory *Dir,
764 SmallString<128> Path;
767 // Fix up relative paths. This just prepends the current working directory.
768 std::error_code EC = FS.makeAbsolute(Path);
772 if (FS.useNormalizedPaths())
773 llvm::sys::path::remove_dots(Path, /*remove_dot_dot=*/true);
778 auto I = llvm::sys::path::begin(Path), E = llvm::sys::path::end(Path);
780 detail::InMemoryNode *Node = Dir->getChild(*I);
783 return errc::no_such_file_or_directory;
785 // Return the file if it's at the end of the path.
786 if (auto File = dyn_cast<detail::InMemoryFile>(Node)) {
789 return errc::no_such_file_or_directory;
792 // If Node is HardLink then return the resolved file.
793 if (auto File = dyn_cast<detail::InMemoryHardLink>(Node)) {
795 return &File->getResolvedFile();
796 return errc::no_such_file_or_directory;
798 // Traverse directories.
799 Dir = cast<detail::InMemoryDirectory>(Node);
805 bool InMemoryFileSystem::addHardLink(const Twine &FromPath,
806 const Twine &ToPath) {
807 auto FromNode = lookupInMemoryNode(*this, Root.get(), FromPath);
808 auto ToNode = lookupInMemoryNode(*this, Root.get(), ToPath);
809 // FromPath must not have been added before. ToPath must have been added
810 // before. Resolved ToPath must be a File.
811 if (!ToNode || FromNode || !isa<detail::InMemoryFile>(*ToNode))
813 return this->addFile(FromPath, 0, nullptr, None, None, None, None,
814 cast<detail::InMemoryFile>(*ToNode));
817 llvm::ErrorOr<Status> InMemoryFileSystem::status(const Twine &Path) {
818 auto Node = lookupInMemoryNode(*this, Root.get(), Path);
820 return detail::getNodeStatus(*Node, Path.str());
821 return Node.getError();
824 llvm::ErrorOr<std::unique_ptr<File>>
825 InMemoryFileSystem::openFileForRead(const Twine &Path) {
826 auto Node = lookupInMemoryNode(*this, Root.get(), Path);
828 return Node.getError();
830 // When we have a file provide a heap-allocated wrapper for the memory buffer
831 // to match the ownership semantics for File.
832 if (auto *F = dyn_cast<detail::InMemoryFile>(*Node))
833 return std::unique_ptr<File>(
834 new detail::InMemoryFileAdaptor(*F, Path.str()));
836 // FIXME: errc::not_a_file?
837 return make_error_code(llvm::errc::invalid_argument);
842 /// Adaptor from InMemoryDir::iterator to directory_iterator.
843 class InMemoryDirIterator : public llvm::vfs::detail::DirIterImpl {
844 detail::InMemoryDirectory::const_iterator I;
845 detail::InMemoryDirectory::const_iterator E;
846 std::string RequestedDirName;
848 void setCurrentEntry() {
850 SmallString<256> Path(RequestedDirName);
851 llvm::sys::path::append(Path, I->second->getFileName());
852 sys::fs::file_type Type;
853 switch (I->second->getKind()) {
854 case detail::IME_File:
855 case detail::IME_HardLink:
856 Type = sys::fs::file_type::regular_file;
858 case detail::IME_Directory:
859 Type = sys::fs::file_type::directory_file;
862 CurrentEntry = directory_entry(Path.str(), Type);
864 // When we're at the end, make CurrentEntry invalid and DirIterImpl will
866 CurrentEntry = directory_entry();
871 InMemoryDirIterator() = default;
873 explicit InMemoryDirIterator(const detail::InMemoryDirectory &Dir,
874 std::string RequestedDirName)
875 : I(Dir.begin()), E(Dir.end()),
876 RequestedDirName(std::move(RequestedDirName)) {
880 std::error_code increment() override {
889 directory_iterator InMemoryFileSystem::dir_begin(const Twine &Dir,
890 std::error_code &EC) {
891 auto Node = lookupInMemoryNode(*this, Root.get(), Dir);
893 EC = Node.getError();
894 return directory_iterator(std::make_shared<InMemoryDirIterator>());
897 if (auto *DirNode = dyn_cast<detail::InMemoryDirectory>(*Node))
898 return directory_iterator(
899 std::make_shared<InMemoryDirIterator>(*DirNode, Dir.str()));
901 EC = make_error_code(llvm::errc::not_a_directory);
902 return directory_iterator(std::make_shared<InMemoryDirIterator>());
905 std::error_code InMemoryFileSystem::setCurrentWorkingDirectory(const Twine &P) {
906 SmallString<128> Path;
909 // Fix up relative paths. This just prepends the current working directory.
910 std::error_code EC = makeAbsolute(Path);
914 if (useNormalizedPaths())
915 llvm::sys::path::remove_dots(Path, /*remove_dot_dot=*/true);
918 WorkingDirectory = Path.str();
923 InMemoryFileSystem::getRealPath(const Twine &Path,
924 SmallVectorImpl<char> &Output) const {
925 auto CWD = getCurrentWorkingDirectory();
926 if (!CWD || CWD->empty())
927 return errc::operation_not_permitted;
928 Path.toVector(Output);
929 if (auto EC = makeAbsolute(Output))
931 llvm::sys::path::remove_dots(Output, /*remove_dot_dot=*/true);
935 std::error_code InMemoryFileSystem::isLocal(const Twine &Path, bool &Result) {
943 //===-----------------------------------------------------------------------===/
944 // RedirectingFileSystem implementation
945 //===-----------------------------------------------------------------------===/
947 // FIXME: reuse implementation common with OverlayFSDirIterImpl as these
948 // iterators are conceptually similar.
949 class llvm::vfs::VFSFromYamlDirIterImpl
950 : public llvm::vfs::detail::DirIterImpl {
952 RedirectingFileSystem::RedirectingDirectoryEntry::iterator Current, End;
954 // To handle 'fallthrough' mode we need to iterate at first through
955 // RedirectingDirectoryEntry and then through ExternalFS. These operations are
956 // done sequentially, we just need to keep a track of what kind of iteration
957 // we are currently performing.
959 /// Flag telling if we should iterate through ExternalFS or stop at the last
960 /// RedirectingDirectoryEntry::iterator.
961 bool IterateExternalFS;
962 /// Flag telling if we have switched to iterating through ExternalFS.
963 bool IsExternalFSCurrent = false;
964 FileSystem &ExternalFS;
965 directory_iterator ExternalDirIter;
966 llvm::StringSet<> SeenNames;
968 /// To combine multiple iterations, different methods are responsible for
969 /// different iteration steps.
972 /// Responsible for dispatching between RedirectingDirectoryEntry iteration
973 /// and ExternalFS iteration.
974 std::error_code incrementImpl(bool IsFirstTime);
975 /// Responsible for RedirectingDirectoryEntry iteration.
976 std::error_code incrementContent(bool IsFirstTime);
977 /// Responsible for ExternalFS iteration.
978 std::error_code incrementExternal();
982 VFSFromYamlDirIterImpl(
984 RedirectingFileSystem::RedirectingDirectoryEntry::iterator Begin,
985 RedirectingFileSystem::RedirectingDirectoryEntry::iterator End,
986 bool IterateExternalFS, FileSystem &ExternalFS, std::error_code &EC);
988 std::error_code increment() override;
991 llvm::ErrorOr<std::string>
992 RedirectingFileSystem::getCurrentWorkingDirectory() const {
993 return ExternalFS->getCurrentWorkingDirectory();
997 RedirectingFileSystem::setCurrentWorkingDirectory(const Twine &Path) {
998 return ExternalFS->setCurrentWorkingDirectory(Path);
1001 std::error_code RedirectingFileSystem::isLocal(const Twine &Path,
1003 return ExternalFS->isLocal(Path, Result);
1006 directory_iterator RedirectingFileSystem::dir_begin(const Twine &Dir,
1007 std::error_code &EC) {
1008 ErrorOr<RedirectingFileSystem::Entry *> E = lookupPath(Dir);
1011 if (IsFallthrough && EC == errc::no_such_file_or_directory)
1012 return ExternalFS->dir_begin(Dir, EC);
1015 ErrorOr<Status> S = status(Dir, *E);
1020 if (!S->isDirectory()) {
1021 EC = std::error_code(static_cast<int>(errc::not_a_directory),
1022 std::system_category());
1026 auto *D = cast<RedirectingFileSystem::RedirectingDirectoryEntry>(*E);
1027 return directory_iterator(std::make_shared<VFSFromYamlDirIterImpl>(
1028 Dir, D->contents_begin(), D->contents_end(),
1029 /*IterateExternalFS=*/IsFallthrough, *ExternalFS, EC));
1032 void RedirectingFileSystem::setExternalContentsPrefixDir(StringRef PrefixDir) {
1033 ExternalContentsPrefixDir = PrefixDir.str();
1036 StringRef RedirectingFileSystem::getExternalContentsPrefixDir() const {
1037 return ExternalContentsPrefixDir;
1040 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1041 LLVM_DUMP_METHOD void RedirectingFileSystem::dump() const {
1042 for (const auto &Root : Roots)
1043 dumpEntry(Root.get());
1046 LLVM_DUMP_METHOD void
1047 RedirectingFileSystem::dumpEntry(RedirectingFileSystem::Entry *E,
1048 int NumSpaces) const {
1049 StringRef Name = E->getName();
1050 for (int i = 0, e = NumSpaces; i < e; ++i)
1052 dbgs() << "'" << Name.str().c_str() << "'"
1055 if (E->getKind() == RedirectingFileSystem::EK_Directory) {
1056 auto *DE = dyn_cast<RedirectingFileSystem::RedirectingDirectoryEntry>(E);
1057 assert(DE && "Should be a directory");
1059 for (std::unique_ptr<Entry> &SubEntry :
1060 llvm::make_range(DE->contents_begin(), DE->contents_end()))
1061 dumpEntry(SubEntry.get(), NumSpaces + 2);
1066 /// A helper class to hold the common YAML parsing state.
1067 class llvm::vfs::RedirectingFileSystemParser {
1068 yaml::Stream &Stream;
1070 void error(yaml::Node *N, const Twine &Msg) { Stream.printError(N, Msg); }
1073 bool parseScalarString(yaml::Node *N, StringRef &Result,
1074 SmallVectorImpl<char> &Storage) {
1075 const auto *S = dyn_cast<yaml::ScalarNode>(N);
1078 error(N, "expected string");
1081 Result = S->getValue(Storage);
1086 bool parseScalarBool(yaml::Node *N, bool &Result) {
1087 SmallString<5> Storage;
1089 if (!parseScalarString(N, Value, Storage))
1092 if (Value.equals_lower("true") || Value.equals_lower("on") ||
1093 Value.equals_lower("yes") || Value == "1") {
1096 } else if (Value.equals_lower("false") || Value.equals_lower("off") ||
1097 Value.equals_lower("no") || Value == "0") {
1102 error(N, "expected boolean value");
1110 KeyStatus(bool Required = false) : Required(Required) {}
1113 using KeyStatusPair = std::pair<StringRef, KeyStatus>;
1116 bool checkDuplicateOrUnknownKey(yaml::Node *KeyNode, StringRef Key,
1117 DenseMap<StringRef, KeyStatus> &Keys) {
1118 if (!Keys.count(Key)) {
1119 error(KeyNode, "unknown key");
1122 KeyStatus &S = Keys[Key];
1124 error(KeyNode, Twine("duplicate key '") + Key + "'");
1132 bool checkMissingKeys(yaml::Node *Obj, DenseMap<StringRef, KeyStatus> &Keys) {
1133 for (const auto &I : Keys) {
1134 if (I.second.Required && !I.second.Seen) {
1135 error(Obj, Twine("missing key '") + I.first + "'");
1142 RedirectingFileSystem::Entry *
1143 lookupOrCreateEntry(RedirectingFileSystem *FS, StringRef Name,
1144 RedirectingFileSystem::Entry *ParentEntry = nullptr) {
1145 if (!ParentEntry) { // Look for a existent root
1146 for (const auto &Root : FS->Roots) {
1147 if (Name.equals(Root->getName())) {
1148 ParentEntry = Root.get();
1152 } else { // Advance to the next component
1153 auto *DE = dyn_cast<RedirectingFileSystem::RedirectingDirectoryEntry>(
1155 for (std::unique_ptr<RedirectingFileSystem::Entry> &Content :
1156 llvm::make_range(DE->contents_begin(), DE->contents_end())) {
1158 dyn_cast<RedirectingFileSystem::RedirectingDirectoryEntry>(
1160 if (DirContent && Name.equals(Content->getName()))
1165 // ... or create a new one
1166 std::unique_ptr<RedirectingFileSystem::Entry> E =
1167 llvm::make_unique<RedirectingFileSystem::RedirectingDirectoryEntry>(
1168 Name, Status("", getNextVirtualUniqueID(),
1169 std::chrono::system_clock::now(), 0, 0, 0,
1170 file_type::directory_file, sys::fs::all_all));
1172 if (!ParentEntry) { // Add a new root to the overlay
1173 FS->Roots.push_back(std::move(E));
1174 ParentEntry = FS->Roots.back().get();
1179 dyn_cast<RedirectingFileSystem::RedirectingDirectoryEntry>(ParentEntry);
1180 DE->addContent(std::move(E));
1181 return DE->getLastContent();
1184 void uniqueOverlayTree(RedirectingFileSystem *FS,
1185 RedirectingFileSystem::Entry *SrcE,
1186 RedirectingFileSystem::Entry *NewParentE = nullptr) {
1187 StringRef Name = SrcE->getName();
1188 switch (SrcE->getKind()) {
1189 case RedirectingFileSystem::EK_Directory: {
1191 dyn_cast<RedirectingFileSystem::RedirectingDirectoryEntry>(SrcE);
1192 assert(DE && "Must be a directory");
1193 // Empty directories could be present in the YAML as a way to
1194 // describe a file for a current directory after some of its subdir
1195 // is parsed. This only leads to redundant walks, ignore it.
1197 NewParentE = lookupOrCreateEntry(FS, Name, NewParentE);
1198 for (std::unique_ptr<RedirectingFileSystem::Entry> &SubEntry :
1199 llvm::make_range(DE->contents_begin(), DE->contents_end()))
1200 uniqueOverlayTree(FS, SubEntry.get(), NewParentE);
1203 case RedirectingFileSystem::EK_File: {
1204 auto *FE = dyn_cast<RedirectingFileSystem::RedirectingFileEntry>(SrcE);
1205 assert(FE && "Must be a file");
1206 assert(NewParentE && "Parent entry must exist");
1207 auto *DE = dyn_cast<RedirectingFileSystem::RedirectingDirectoryEntry>(
1210 llvm::make_unique<RedirectingFileSystem::RedirectingFileEntry>(
1211 Name, FE->getExternalContentsPath(), FE->getUseName()));
1217 std::unique_ptr<RedirectingFileSystem::Entry>
1218 parseEntry(yaml::Node *N, RedirectingFileSystem *FS, bool IsRootEntry) {
1219 auto *M = dyn_cast<yaml::MappingNode>(N);
1221 error(N, "expected mapping node for file or directory entry");
1225 KeyStatusPair Fields[] = {
1226 KeyStatusPair("name", true),
1227 KeyStatusPair("type", true),
1228 KeyStatusPair("contents", false),
1229 KeyStatusPair("external-contents", false),
1230 KeyStatusPair("use-external-name", false),
1233 DenseMap<StringRef, KeyStatus> Keys(std::begin(Fields), std::end(Fields));
1235 bool HasContents = false; // external or otherwise
1236 std::vector<std::unique_ptr<RedirectingFileSystem::Entry>>
1238 std::string ExternalContentsPath;
1240 yaml::Node *NameValueNode;
1241 auto UseExternalName =
1242 RedirectingFileSystem::RedirectingFileEntry::NK_NotSet;
1243 RedirectingFileSystem::EntryKind Kind;
1245 for (auto &I : *M) {
1247 // Reuse the buffer for key and value, since we don't look at key after
1249 SmallString<256> Buffer;
1250 if (!parseScalarString(I.getKey(), Key, Buffer))
1253 if (!checkDuplicateOrUnknownKey(I.getKey(), Key, Keys))
1257 if (Key == "name") {
1258 if (!parseScalarString(I.getValue(), Value, Buffer))
1261 NameValueNode = I.getValue();
1262 if (FS->UseCanonicalizedPaths) {
1263 SmallString<256> Path(Value);
1264 // Guarantee that old YAML files containing paths with ".." and "."
1265 // are properly canonicalized before read into the VFS.
1266 Path = sys::path::remove_leading_dotslash(Path);
1267 sys::path::remove_dots(Path, /*remove_dot_dot=*/true);
1272 } else if (Key == "type") {
1273 if (!parseScalarString(I.getValue(), Value, Buffer))
1275 if (Value == "file")
1276 Kind = RedirectingFileSystem::EK_File;
1277 else if (Value == "directory")
1278 Kind = RedirectingFileSystem::EK_Directory;
1280 error(I.getValue(), "unknown value for 'type'");
1283 } else if (Key == "contents") {
1286 "entry already has 'contents' or 'external-contents'");
1290 auto *Contents = dyn_cast<yaml::SequenceNode>(I.getValue());
1292 // FIXME: this is only for directories, what about files?
1293 error(I.getValue(), "expected array");
1297 for (auto &I : *Contents) {
1298 if (std::unique_ptr<RedirectingFileSystem::Entry> E =
1299 parseEntry(&I, FS, /*IsRootEntry*/ false))
1300 EntryArrayContents.push_back(std::move(E));
1304 } else if (Key == "external-contents") {
1307 "entry already has 'contents' or 'external-contents'");
1311 if (!parseScalarString(I.getValue(), Value, Buffer))
1314 SmallString<256> FullPath;
1315 if (FS->IsRelativeOverlay) {
1316 FullPath = FS->getExternalContentsPrefixDir();
1317 assert(!FullPath.empty() &&
1318 "External contents prefix directory must exist");
1319 llvm::sys::path::append(FullPath, Value);
1324 if (FS->UseCanonicalizedPaths) {
1325 // Guarantee that old YAML files containing paths with ".." and "."
1326 // are properly canonicalized before read into the VFS.
1327 FullPath = sys::path::remove_leading_dotslash(FullPath);
1328 sys::path::remove_dots(FullPath, /*remove_dot_dot=*/true);
1330 ExternalContentsPath = FullPath.str();
1331 } else if (Key == "use-external-name") {
1333 if (!parseScalarBool(I.getValue(), Val))
1336 Val ? RedirectingFileSystem::RedirectingFileEntry::NK_External
1337 : RedirectingFileSystem::RedirectingFileEntry::NK_Virtual;
1339 llvm_unreachable("key missing from Keys");
1343 if (Stream.failed())
1346 // check for missing keys
1348 error(N, "missing key 'contents' or 'external-contents'");
1351 if (!checkMissingKeys(N, Keys))
1354 // check invalid configuration
1355 if (Kind == RedirectingFileSystem::EK_Directory &&
1357 RedirectingFileSystem::RedirectingFileEntry::NK_NotSet) {
1358 error(N, "'use-external-name' is not supported for directories");
1362 if (IsRootEntry && !sys::path::is_absolute(Name)) {
1363 assert(NameValueNode && "Name presence should be checked earlier");
1364 error(NameValueNode,
1365 "entry with relative path at the root level is not discoverable");
1369 // Remove trailing slash(es), being careful not to remove the root path
1370 StringRef Trimmed(Name);
1371 size_t RootPathLen = sys::path::root_path(Trimmed).size();
1372 while (Trimmed.size() > RootPathLen &&
1373 sys::path::is_separator(Trimmed.back()))
1374 Trimmed = Trimmed.slice(0, Trimmed.size() - 1);
1375 // Get the last component
1376 StringRef LastComponent = sys::path::filename(Trimmed);
1378 std::unique_ptr<RedirectingFileSystem::Entry> Result;
1380 case RedirectingFileSystem::EK_File:
1381 Result = llvm::make_unique<RedirectingFileSystem::RedirectingFileEntry>(
1382 LastComponent, std::move(ExternalContentsPath), UseExternalName);
1384 case RedirectingFileSystem::EK_Directory:
1386 llvm::make_unique<RedirectingFileSystem::RedirectingDirectoryEntry>(
1387 LastComponent, std::move(EntryArrayContents),
1388 Status("", getNextVirtualUniqueID(),
1389 std::chrono::system_clock::now(), 0, 0, 0,
1390 file_type::directory_file, sys::fs::all_all));
1394 StringRef Parent = sys::path::parent_path(Trimmed);
1398 // if 'name' contains multiple components, create implicit directory entries
1399 for (sys::path::reverse_iterator I = sys::path::rbegin(Parent),
1400 E = sys::path::rend(Parent);
1402 std::vector<std::unique_ptr<RedirectingFileSystem::Entry>> Entries;
1403 Entries.push_back(std::move(Result));
1405 llvm::make_unique<RedirectingFileSystem::RedirectingDirectoryEntry>(
1406 *I, std::move(Entries),
1407 Status("", getNextVirtualUniqueID(),
1408 std::chrono::system_clock::now(), 0, 0, 0,
1409 file_type::directory_file, sys::fs::all_all));
1415 RedirectingFileSystemParser(yaml::Stream &S) : Stream(S) {}
1418 bool parse(yaml::Node *Root, RedirectingFileSystem *FS) {
1419 auto *Top = dyn_cast<yaml::MappingNode>(Root);
1421 error(Root, "expected mapping node");
1425 KeyStatusPair Fields[] = {
1426 KeyStatusPair("version", true),
1427 KeyStatusPair("case-sensitive", false),
1428 KeyStatusPair("use-external-names", false),
1429 KeyStatusPair("overlay-relative", false),
1430 KeyStatusPair("fallthrough", false),
1431 KeyStatusPair("roots", true),
1434 DenseMap<StringRef, KeyStatus> Keys(std::begin(Fields), std::end(Fields));
1435 std::vector<std::unique_ptr<RedirectingFileSystem::Entry>> RootEntries;
1437 // Parse configuration and 'roots'
1438 for (auto &I : *Top) {
1439 SmallString<10> KeyBuffer;
1441 if (!parseScalarString(I.getKey(), Key, KeyBuffer))
1444 if (!checkDuplicateOrUnknownKey(I.getKey(), Key, Keys))
1447 if (Key == "roots") {
1448 auto *Roots = dyn_cast<yaml::SequenceNode>(I.getValue());
1450 error(I.getValue(), "expected array");
1454 for (auto &I : *Roots) {
1455 if (std::unique_ptr<RedirectingFileSystem::Entry> E =
1456 parseEntry(&I, FS, /*IsRootEntry*/ true))
1457 RootEntries.push_back(std::move(E));
1461 } else if (Key == "version") {
1462 StringRef VersionString;
1463 SmallString<4> Storage;
1464 if (!parseScalarString(I.getValue(), VersionString, Storage))
1467 if (VersionString.getAsInteger<int>(10, Version)) {
1468 error(I.getValue(), "expected integer");
1472 error(I.getValue(), "invalid version number");
1476 error(I.getValue(), "version mismatch, expected 0");
1479 } else if (Key == "case-sensitive") {
1480 if (!parseScalarBool(I.getValue(), FS->CaseSensitive))
1482 } else if (Key == "overlay-relative") {
1483 if (!parseScalarBool(I.getValue(), FS->IsRelativeOverlay))
1485 } else if (Key == "use-external-names") {
1486 if (!parseScalarBool(I.getValue(), FS->UseExternalNames))
1488 } else if (Key == "fallthrough") {
1489 if (!parseScalarBool(I.getValue(), FS->IsFallthrough))
1492 llvm_unreachable("key missing from Keys");
1496 if (Stream.failed())
1499 if (!checkMissingKeys(Top, Keys))
1502 // Now that we sucessefully parsed the YAML file, canonicalize the internal
1503 // representation to a proper directory tree so that we can search faster
1505 for (auto &E : RootEntries)
1506 uniqueOverlayTree(FS, E.get());
1512 RedirectingFileSystem *
1513 RedirectingFileSystem::create(std::unique_ptr<MemoryBuffer> Buffer,
1514 SourceMgr::DiagHandlerTy DiagHandler,
1515 StringRef YAMLFilePath, void *DiagContext,
1516 IntrusiveRefCntPtr<FileSystem> ExternalFS) {
1518 yaml::Stream Stream(Buffer->getMemBufferRef(), SM);
1520 SM.setDiagHandler(DiagHandler, DiagContext);
1521 yaml::document_iterator DI = Stream.begin();
1522 yaml::Node *Root = DI->getRoot();
1523 if (DI == Stream.end() || !Root) {
1524 SM.PrintMessage(SMLoc(), SourceMgr::DK_Error, "expected root node");
1528 RedirectingFileSystemParser P(Stream);
1530 std::unique_ptr<RedirectingFileSystem> FS(
1531 new RedirectingFileSystem(std::move(ExternalFS)));
1533 if (!YAMLFilePath.empty()) {
1534 // Use the YAML path from -ivfsoverlay to compute the dir to be prefixed
1535 // to each 'external-contents' path.
1538 // -ivfsoverlay dummy.cache/vfs/vfs.yaml
1540 // FS->ExternalContentsPrefixDir => /<absolute_path_to>/dummy.cache/vfs
1542 SmallString<256> OverlayAbsDir = sys::path::parent_path(YAMLFilePath);
1543 std::error_code EC = llvm::sys::fs::make_absolute(OverlayAbsDir);
1544 assert(!EC && "Overlay dir final path must be absolute");
1546 FS->setExternalContentsPrefixDir(OverlayAbsDir);
1549 if (!P.parse(Root, FS.get()))
1552 return FS.release();
1555 ErrorOr<RedirectingFileSystem::Entry *>
1556 RedirectingFileSystem::lookupPath(const Twine &Path_) const {
1557 SmallString<256> Path;
1558 Path_.toVector(Path);
1560 // Handle relative paths
1561 if (std::error_code EC = makeAbsolute(Path))
1564 // Canonicalize path by removing ".", "..", "./", etc components. This is
1565 // a VFS request, do bot bother about symlinks in the path components
1566 // but canonicalize in order to perform the correct entry search.
1567 if (UseCanonicalizedPaths) {
1568 Path = sys::path::remove_leading_dotslash(Path);
1569 sys::path::remove_dots(Path, /*remove_dot_dot=*/true);
1573 return make_error_code(llvm::errc::invalid_argument);
1575 sys::path::const_iterator Start = sys::path::begin(Path);
1576 sys::path::const_iterator End = sys::path::end(Path);
1577 for (const auto &Root : Roots) {
1578 ErrorOr<RedirectingFileSystem::Entry *> Result =
1579 lookupPath(Start, End, Root.get());
1580 if (Result || Result.getError() != llvm::errc::no_such_file_or_directory)
1583 return make_error_code(llvm::errc::no_such_file_or_directory);
1586 ErrorOr<RedirectingFileSystem::Entry *>
1587 RedirectingFileSystem::lookupPath(sys::path::const_iterator Start,
1588 sys::path::const_iterator End,
1589 RedirectingFileSystem::Entry *From) const {
1591 assert(!isTraversalComponent(*Start) &&
1592 !isTraversalComponent(From->getName()) &&
1593 "Paths should not contain traversal components");
1595 // FIXME: this is here to support windows, remove it once canonicalized
1596 // paths become globally default.
1597 if (Start->equals("."))
1601 StringRef FromName = From->getName();
1603 // Forward the search to the next component in case this is an empty one.
1604 if (!FromName.empty()) {
1605 if (CaseSensitive ? !Start->equals(FromName)
1606 : !Start->equals_lower(FromName))
1608 return make_error_code(llvm::errc::no_such_file_or_directory);
1618 auto *DE = dyn_cast<RedirectingFileSystem::RedirectingDirectoryEntry>(From);
1620 return make_error_code(llvm::errc::not_a_directory);
1622 for (const std::unique_ptr<RedirectingFileSystem::Entry> &DirEntry :
1623 llvm::make_range(DE->contents_begin(), DE->contents_end())) {
1624 ErrorOr<RedirectingFileSystem::Entry *> Result =
1625 lookupPath(Start, End, DirEntry.get());
1626 if (Result || Result.getError() != llvm::errc::no_such_file_or_directory)
1629 return make_error_code(llvm::errc::no_such_file_or_directory);
1632 static Status getRedirectedFileStatus(const Twine &Path, bool UseExternalNames,
1633 Status ExternalStatus) {
1634 Status S = ExternalStatus;
1635 if (!UseExternalNames)
1636 S = Status::copyWithNewName(S, Path.str());
1637 S.IsVFSMapped = true;
1641 ErrorOr<Status> RedirectingFileSystem::status(const Twine &Path,
1642 RedirectingFileSystem::Entry *E) {
1643 assert(E != nullptr);
1644 if (auto *F = dyn_cast<RedirectingFileSystem::RedirectingFileEntry>(E)) {
1645 ErrorOr<Status> S = ExternalFS->status(F->getExternalContentsPath());
1646 assert(!S || S->getName() == F->getExternalContentsPath());
1648 return getRedirectedFileStatus(Path, F->useExternalName(UseExternalNames),
1651 } else { // directory
1652 auto *DE = cast<RedirectingFileSystem::RedirectingDirectoryEntry>(E);
1653 return Status::copyWithNewName(DE->getStatus(), Path.str());
1657 ErrorOr<Status> RedirectingFileSystem::status(const Twine &Path) {
1658 ErrorOr<RedirectingFileSystem::Entry *> Result = lookupPath(Path);
1660 if (IsFallthrough &&
1661 Result.getError() == llvm::errc::no_such_file_or_directory) {
1662 return ExternalFS->status(Path);
1664 return Result.getError();
1666 return status(Path, *Result);
1671 /// Provide a file wrapper with an overriden status.
1672 class FileWithFixedStatus : public File {
1673 std::unique_ptr<File> InnerFile;
1677 FileWithFixedStatus(std::unique_ptr<File> InnerFile, Status S)
1678 : InnerFile(std::move(InnerFile)), S(std::move(S)) {}
1680 ErrorOr<Status> status() override { return S; }
1681 ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
1683 getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator,
1684 bool IsVolatile) override {
1685 return InnerFile->getBuffer(Name, FileSize, RequiresNullTerminator,
1689 std::error_code close() override { return InnerFile->close(); }
1694 ErrorOr<std::unique_ptr<File>>
1695 RedirectingFileSystem::openFileForRead(const Twine &Path) {
1696 ErrorOr<RedirectingFileSystem::Entry *> E = lookupPath(Path);
1698 if (IsFallthrough &&
1699 E.getError() == llvm::errc::no_such_file_or_directory) {
1700 return ExternalFS->openFileForRead(Path);
1702 return E.getError();
1705 auto *F = dyn_cast<RedirectingFileSystem::RedirectingFileEntry>(*E);
1706 if (!F) // FIXME: errc::not_a_file?
1707 return make_error_code(llvm::errc::invalid_argument);
1709 auto Result = ExternalFS->openFileForRead(F->getExternalContentsPath());
1713 auto ExternalStatus = (*Result)->status();
1714 if (!ExternalStatus)
1715 return ExternalStatus.getError();
1717 // FIXME: Update the status with the name and VFSMapped.
1718 Status S = getRedirectedFileStatus(Path, F->useExternalName(UseExternalNames),
1720 return std::unique_ptr<File>(
1721 llvm::make_unique<FileWithFixedStatus>(std::move(*Result), S));
1725 RedirectingFileSystem::getRealPath(const Twine &Path,
1726 SmallVectorImpl<char> &Output) const {
1727 ErrorOr<RedirectingFileSystem::Entry *> Result = lookupPath(Path);
1729 if (IsFallthrough &&
1730 Result.getError() == llvm::errc::no_such_file_or_directory) {
1731 return ExternalFS->getRealPath(Path, Output);
1733 return Result.getError();
1737 dyn_cast<RedirectingFileSystem::RedirectingFileEntry>(*Result)) {
1738 return ExternalFS->getRealPath(F->getExternalContentsPath(), Output);
1740 // Even if there is a directory entry, fall back to ExternalFS if allowed,
1741 // because directories don't have a single external contents path.
1742 return IsFallthrough ? ExternalFS->getRealPath(Path, Output)
1743 : llvm::errc::invalid_argument;
1746 IntrusiveRefCntPtr<FileSystem>
1747 vfs::getVFSFromYAML(std::unique_ptr<MemoryBuffer> Buffer,
1748 SourceMgr::DiagHandlerTy DiagHandler,
1749 StringRef YAMLFilePath, void *DiagContext,
1750 IntrusiveRefCntPtr<FileSystem> ExternalFS) {
1751 return RedirectingFileSystem::create(std::move(Buffer), DiagHandler,
1752 YAMLFilePath, DiagContext,
1753 std::move(ExternalFS));
1756 static void getVFSEntries(RedirectingFileSystem::Entry *SrcE,
1757 SmallVectorImpl<StringRef> &Path,
1758 SmallVectorImpl<YAMLVFSEntry> &Entries) {
1759 auto Kind = SrcE->getKind();
1760 if (Kind == RedirectingFileSystem::EK_Directory) {
1761 auto *DE = dyn_cast<RedirectingFileSystem::RedirectingDirectoryEntry>(SrcE);
1762 assert(DE && "Must be a directory");
1763 for (std::unique_ptr<RedirectingFileSystem::Entry> &SubEntry :
1764 llvm::make_range(DE->contents_begin(), DE->contents_end())) {
1765 Path.push_back(SubEntry->getName());
1766 getVFSEntries(SubEntry.get(), Path, Entries);
1772 assert(Kind == RedirectingFileSystem::EK_File && "Must be a EK_File");
1773 auto *FE = dyn_cast<RedirectingFileSystem::RedirectingFileEntry>(SrcE);
1774 assert(FE && "Must be a file");
1775 SmallString<128> VPath;
1776 for (auto &Comp : Path)
1777 llvm::sys::path::append(VPath, Comp);
1778 Entries.push_back(YAMLVFSEntry(VPath.c_str(), FE->getExternalContentsPath()));
1781 void vfs::collectVFSFromYAML(std::unique_ptr<MemoryBuffer> Buffer,
1782 SourceMgr::DiagHandlerTy DiagHandler,
1783 StringRef YAMLFilePath,
1784 SmallVectorImpl<YAMLVFSEntry> &CollectedEntries,
1786 IntrusiveRefCntPtr<FileSystem> ExternalFS) {
1787 RedirectingFileSystem *VFS = RedirectingFileSystem::create(
1788 std::move(Buffer), DiagHandler, YAMLFilePath, DiagContext,
1789 std::move(ExternalFS));
1790 ErrorOr<RedirectingFileSystem::Entry *> RootE = VFS->lookupPath("/");
1793 SmallVector<StringRef, 8> Components;
1794 Components.push_back("/");
1795 getVFSEntries(*RootE, Components, CollectedEntries);
1798 UniqueID vfs::getNextVirtualUniqueID() {
1799 static std::atomic<unsigned> UID;
1800 unsigned ID = ++UID;
1801 // The following assumes that uint64_t max will never collide with a real
1802 // dev_t value from the OS.
1803 return UniqueID(std::numeric_limits<uint64_t>::max(), ID);
1806 void YAMLVFSWriter::addFileMapping(StringRef VirtualPath, StringRef RealPath) {
1807 assert(sys::path::is_absolute(VirtualPath) && "virtual path not absolute");
1808 assert(sys::path::is_absolute(RealPath) && "real path not absolute");
1809 assert(!pathHasTraversal(VirtualPath) && "path traversal is not supported");
1810 Mappings.emplace_back(VirtualPath, RealPath);
1816 llvm::raw_ostream &OS;
1817 SmallVector<StringRef, 16> DirStack;
1819 unsigned getDirIndent() { return 4 * DirStack.size(); }
1820 unsigned getFileIndent() { return 4 * (DirStack.size() + 1); }
1821 bool containedIn(StringRef Parent, StringRef Path);
1822 StringRef containedPart(StringRef Parent, StringRef Path);
1823 void startDirectory(StringRef Path);
1824 void endDirectory();
1825 void writeEntry(StringRef VPath, StringRef RPath);
1828 JSONWriter(llvm::raw_ostream &OS) : OS(OS) {}
1830 void write(ArrayRef<YAMLVFSEntry> Entries, Optional<bool> UseExternalNames,
1831 Optional<bool> IsCaseSensitive, Optional<bool> IsOverlayRelative,
1832 StringRef OverlayDir);
1837 bool JSONWriter::containedIn(StringRef Parent, StringRef Path) {
1838 using namespace llvm::sys;
1840 // Compare each path component.
1841 auto IParent = path::begin(Parent), EParent = path::end(Parent);
1842 for (auto IChild = path::begin(Path), EChild = path::end(Path);
1843 IParent != EParent && IChild != EChild; ++IParent, ++IChild) {
1844 if (*IParent != *IChild)
1847 // Have we exhausted the parent path?
1848 return IParent == EParent;
1851 StringRef JSONWriter::containedPart(StringRef Parent, StringRef Path) {
1852 assert(!Parent.empty());
1853 assert(containedIn(Parent, Path));
1854 return Path.slice(Parent.size() + 1, StringRef::npos);
1857 void JSONWriter::startDirectory(StringRef Path) {
1859 DirStack.empty() ? Path : containedPart(DirStack.back(), Path);
1860 DirStack.push_back(Path);
1861 unsigned Indent = getDirIndent();
1862 OS.indent(Indent) << "{\n";
1863 OS.indent(Indent + 2) << "'type': 'directory',\n";
1864 OS.indent(Indent + 2) << "'name': \"" << llvm::yaml::escape(Name) << "\",\n";
1865 OS.indent(Indent + 2) << "'contents': [\n";
1868 void JSONWriter::endDirectory() {
1869 unsigned Indent = getDirIndent();
1870 OS.indent(Indent + 2) << "]\n";
1871 OS.indent(Indent) << "}";
1873 DirStack.pop_back();
1876 void JSONWriter::writeEntry(StringRef VPath, StringRef RPath) {
1877 unsigned Indent = getFileIndent();
1878 OS.indent(Indent) << "{\n";
1879 OS.indent(Indent + 2) << "'type': 'file',\n";
1880 OS.indent(Indent + 2) << "'name': \"" << llvm::yaml::escape(VPath) << "\",\n";
1881 OS.indent(Indent + 2) << "'external-contents': \""
1882 << llvm::yaml::escape(RPath) << "\"\n";
1883 OS.indent(Indent) << "}";
1886 void JSONWriter::write(ArrayRef<YAMLVFSEntry> Entries,
1887 Optional<bool> UseExternalNames,
1888 Optional<bool> IsCaseSensitive,
1889 Optional<bool> IsOverlayRelative,
1890 StringRef OverlayDir) {
1891 using namespace llvm::sys;
1895 if (IsCaseSensitive.hasValue())
1896 OS << " 'case-sensitive': '"
1897 << (IsCaseSensitive.getValue() ? "true" : "false") << "',\n";
1898 if (UseExternalNames.hasValue())
1899 OS << " 'use-external-names': '"
1900 << (UseExternalNames.getValue() ? "true" : "false") << "',\n";
1901 bool UseOverlayRelative = false;
1902 if (IsOverlayRelative.hasValue()) {
1903 UseOverlayRelative = IsOverlayRelative.getValue();
1904 OS << " 'overlay-relative': '" << (UseOverlayRelative ? "true" : "false")
1907 OS << " 'roots': [\n";
1909 if (!Entries.empty()) {
1910 const YAMLVFSEntry &Entry = Entries.front();
1911 startDirectory(path::parent_path(Entry.VPath));
1913 StringRef RPath = Entry.RPath;
1914 if (UseOverlayRelative) {
1915 unsigned OverlayDirLen = OverlayDir.size();
1916 assert(RPath.substr(0, OverlayDirLen) == OverlayDir &&
1917 "Overlay dir must be contained in RPath");
1918 RPath = RPath.slice(OverlayDirLen, RPath.size());
1921 writeEntry(path::filename(Entry.VPath), RPath);
1923 for (const auto &Entry : Entries.slice(1)) {
1924 StringRef Dir = path::parent_path(Entry.VPath);
1925 if (Dir == DirStack.back())
1928 while (!DirStack.empty() && !containedIn(DirStack.back(), Dir)) {
1933 startDirectory(Dir);
1935 StringRef RPath = Entry.RPath;
1936 if (UseOverlayRelative) {
1937 unsigned OverlayDirLen = OverlayDir.size();
1938 assert(RPath.substr(0, OverlayDirLen) == OverlayDir &&
1939 "Overlay dir must be contained in RPath");
1940 RPath = RPath.slice(OverlayDirLen, RPath.size());
1942 writeEntry(path::filename(Entry.VPath), RPath);
1945 while (!DirStack.empty()) {
1956 void YAMLVFSWriter::write(llvm::raw_ostream &OS) {
1957 llvm::sort(Mappings, [](const YAMLVFSEntry &LHS, const YAMLVFSEntry &RHS) {
1958 return LHS.VPath < RHS.VPath;
1961 JSONWriter(OS).write(Mappings, UseExternalNames, IsCaseSensitive,
1962 IsOverlayRelative, OverlayDir);
1965 VFSFromYamlDirIterImpl::VFSFromYamlDirIterImpl(
1967 RedirectingFileSystem::RedirectingDirectoryEntry::iterator Begin,
1968 RedirectingFileSystem::RedirectingDirectoryEntry::iterator End,
1969 bool IterateExternalFS, FileSystem &ExternalFS, std::error_code &EC)
1970 : Dir(_Path.str()), Current(Begin), End(End),
1971 IterateExternalFS(IterateExternalFS), ExternalFS(ExternalFS) {
1972 EC = incrementImpl(/*IsFirstTime=*/true);
1975 std::error_code VFSFromYamlDirIterImpl::increment() {
1976 return incrementImpl(/*IsFirstTime=*/false);
1979 std::error_code VFSFromYamlDirIterImpl::incrementExternal() {
1980 assert(!(IsExternalFSCurrent && ExternalDirIter == directory_iterator()) &&
1981 "incrementing past end");
1983 if (IsExternalFSCurrent) {
1984 ExternalDirIter.increment(EC);
1985 } else if (IterateExternalFS) {
1986 ExternalDirIter = ExternalFS.dir_begin(Dir, EC);
1987 IsExternalFSCurrent = true;
1988 if (EC && EC != errc::no_such_file_or_directory)
1992 if (EC || ExternalDirIter == directory_iterator()) {
1993 CurrentEntry = directory_entry();
1995 CurrentEntry = *ExternalDirIter;
2000 std::error_code VFSFromYamlDirIterImpl::incrementContent(bool IsFirstTime) {
2001 assert((IsFirstTime || Current != End) && "cannot iterate past end");
2004 while (Current != End) {
2005 SmallString<128> PathStr(Dir);
2006 llvm::sys::path::append(PathStr, (*Current)->getName());
2007 sys::fs::file_type Type;
2008 switch ((*Current)->getKind()) {
2009 case RedirectingFileSystem::EK_Directory:
2010 Type = sys::fs::file_type::directory_file;
2012 case RedirectingFileSystem::EK_File:
2013 Type = sys::fs::file_type::regular_file;
2016 CurrentEntry = directory_entry(PathStr.str(), Type);
2019 return incrementExternal();
2022 std::error_code VFSFromYamlDirIterImpl::incrementImpl(bool IsFirstTime) {
2024 std::error_code EC = IsExternalFSCurrent ? incrementExternal()
2025 : incrementContent(IsFirstTime);
2026 if (EC || CurrentEntry.path().empty())
2028 StringRef Name = llvm::sys::path::filename(CurrentEntry.path());
2029 if (SeenNames.insert(Name).second)
2030 return EC; // name not seen before
2032 llvm_unreachable("returned above");
2035 vfs::recursive_directory_iterator::recursive_directory_iterator(
2036 FileSystem &FS_, const Twine &Path, std::error_code &EC)
2038 directory_iterator I = FS->dir_begin(Path, EC);
2039 if (I != directory_iterator()) {
2040 State = std::make_shared<detail::RecDirIterState>();
2041 State->Stack.push(I);
2045 vfs::recursive_directory_iterator &
2046 recursive_directory_iterator::increment(std::error_code &EC) {
2047 assert(FS && State && !State->Stack.empty() && "incrementing past end");
2048 assert(!State->Stack.top()->path().empty() && "non-canonical end iterator");
2049 vfs::directory_iterator End;
2051 if (State->HasNoPushRequest)
2052 State->HasNoPushRequest = false;
2054 if (State->Stack.top()->type() == sys::fs::file_type::directory_file) {
2055 vfs::directory_iterator I = FS->dir_begin(State->Stack.top()->path(), EC);
2057 State->Stack.push(I);
2063 while (!State->Stack.empty() && State->Stack.top().increment(EC) == End)
2066 if (State->Stack.empty())
2067 State.reset(); // end iterator