From a5037eb436c36555de6bc709a7822c1998940a11 Mon Sep 17 00:00:00 2001 From: Andrey Shulzhenko Date: Tue, 13 Mar 2018 17:43:38 +0300 Subject: [PATCH] Usd source modified to support in-memory-file read/write (#20) * [USD] changed FILE* to ArchFile* for binary save/load * Fixes for Debug build * Memory Storage (working but not final version) * Memory storage fixes & cleanups * Implemented file mapping for ReadOnly memory storage * Fix for array diff in USD text format * Test for usd diff refactored (added command args parsing and write to diff.usda) --- .../usd/src/pxr/base/lib/arch/fileSystem.cpp | 498 ++++++++++++++---- .../usd/src/pxr/base/lib/arch/fileSystem.h | 150 ++++-- .../usd/src/pxr/base/lib/arch/stackTrace.cpp | 6 +- .../usd/src/pxr/base/lib/tf/envSetting.cpp | 2 +- .../usd/src/pxr/base/lib/tf/pathUtils.cpp | 2 +- .../usd/src/pxr/base/lib/tf/pyInterpreter.cpp | 2 +- .../src/pxr/base/lib/tf/safeOutputFile.cpp | 14 +- .../usd/src/pxr/base/lib/tf/safeOutputFile.h | 8 +- .../pxr/base/lib/tf/scriptModuleLoader.cpp | 2 +- .../src/pxr/usd/lib/sdf/textFileFormat.cpp | 4 +- .../usd/src/pxr/usd/lib/usd/crateFile.cpp | 44 +- .../usd/src/pxr/usd/lib/usd/crateFile.h | 6 +- 12 files changed, 564 insertions(+), 174 deletions(-) diff --git a/third_party/usd/src/pxr/base/lib/arch/fileSystem.cpp b/third_party/usd/src/pxr/base/lib/arch/fileSystem.cpp index b1f7016ba0..444d51620a 100644 --- a/third_party/usd/src/pxr/base/lib/arch/fileSystem.cpp +++ b/third_party/usd/src/pxr/base/lib/arch/fileSystem.cpp @@ -68,9 +68,392 @@ static inline HANDLE _FileToWinHANDLE(FILE *file) } #endif // ARCH_OS_WINDOWS -FILE* ArchOpenFile(char const* fileName, char const* mode) +class ArchMemFixedStorage : public ArchMemStorage { - return fopen(fileName, mode); +public: + ArchMemFixedStorage(size_t length, const uint8_t* data) : _length(length), _data(data) {} + + virtual size_t GetLength() const override { return _length; } + + virtual size_t Read(uint8_t* data, size_t count, size_t offset) override + { + size_t offsetEnd = offset + count; + if (offsetEnd > _length) offsetEnd = _length; + + if (offsetEnd > offset) + { + const size_t copyCount = (offsetEnd - offset); + memcpy(data, _data + offset, copyCount); + return copyCount; + } + else + { + return 0; + } + } + virtual size_t Write(const uint8_t* data, size_t count, size_t offset) override + { + return 0; + } + + virtual const void* GetPtrForMapping() const override { return _data; } + +private: + size_t _length; + const uint8_t* _data; +}; + + +class ArchMemGrowingStorage : public ArchMemStorage +{ +public: + static const size_t ChunkSize = 256 * 1024; // 256 Kb (8 bits) + + ArchMemGrowingStorage() : _length(0) {} + + virtual size_t GetLength() const override { return _length; } + + virtual size_t Read(uint8_t* data, size_t count, size_t offset) override + { + size_t offsetEnd = offset + count; + if (offsetEnd > _length) offsetEnd = _length; + + if (offsetEnd > offset) + { + const size_t begChunkIdx = offset / ChunkSize; + const size_t endChunkIdx = (offsetEnd + ChunkSize - 1) / ChunkSize; + + _CopyData(data, begChunkIdx, endChunkIdx, offset, offsetEnd, [](uint8_t* dst, const uint8_t* src, size_t size) { memcpy(dst, src, size); }); + + return (offsetEnd - offset); + } + else + { + return 0; + } + } + + virtual size_t Write(const uint8_t* data, size_t count, size_t offset) override + { + if (count > 0) + { + const size_t offsetEnd = offset + count; + const size_t begChunkIdx = offset / ChunkSize; + const size_t endChunkIdx = (offsetEnd + ChunkSize - 1) / ChunkSize; + + const size_t endChunkIdx0 = _chunks.size(); + if (endChunkIdx > endChunkIdx0) + { + //allocate new chunks + _chunks.resize(endChunkIdx); + for (auto idx = endChunkIdx0; idx < endChunkIdx; ++idx) + { + _chunks[idx] = Chunk_t(new uint8_t[ChunkSize]); + } + } + + _CopyData(data, begChunkIdx, endChunkIdx, offset, offsetEnd, [](const uint8_t* src, uint8_t* dst, size_t size) { memcpy(dst, src, size); }); + + if (_length < offsetEnd) _length = offsetEnd; + } + return count; + } + + virtual const void* GetPtrForMapping() const override { return nullptr; } + +private: + size_t _length; + + typedef std::unique_ptr Chunk_t; + std::vector _chunks; + + template + Ptr _CopyData(Ptr data, size_t begChunkIdx, size_t endChunkIdx, size_t offsetBeg, size_t offsetEnd, Func copyFunc) + { + if (begChunkIdx == endChunkIdx - 1) + { + const size_t offsetInChunk = (offsetBeg - begChunkIdx * ChunkSize); + copyFunc(data, _chunks[begChunkIdx].get() + offsetInChunk, offsetEnd - offsetBeg); + } + else + { + { + const size_t offsetInChunk = (offsetBeg - begChunkIdx * ChunkSize); + const size_t copyCount = ChunkSize - offsetInChunk; + copyFunc(data, _chunks[begChunkIdx].get() + offsetInChunk, copyCount); + data += copyCount; + } + auto curChunkIdx = begChunkIdx; + for (; curChunkIdx < endChunkIdx - 1; ++curChunkIdx) + { + copyFunc(data, _chunks[curChunkIdx].get(), ChunkSize); + data += ChunkSize; + } + { + const size_t copyCount = (offsetEnd - curChunkIdx * ChunkSize); + copyFunc(data, _chunks[curChunkIdx].get(), copyCount); + data += copyCount; + } + } + return data; + } +}; + + +namespace +{ + class ArchMemStorageRegistry + { + struct CompareStr + { + using is_transparent = void; + + bool operator()(const std::string& str1, const std::string& str2) const + { + return str1 < str2; + } + bool operator()(const char* str1, const std::string& str2) const + { + return ::strcmp(str1, str2.c_str()) < 0; + } + bool operator()(const std::string& str1, const char* str2) const + { + return ::strcmp(str1.c_str(), str2) < 0; + } + }; + + std::map, CompareStr> _map; + + ArchMemStorageRegistry() {} + + public: + static ArchMemStorageRegistry* GetInst() + { + static ArchMemStorageRegistry inst; + return &inst; + } + + void Set(const std::string& path, const std::shared_ptr& ptr) + { + _map[path] = ptr; + } + + std::shared_ptr Get(const char* path) + { + auto it = _map.find(path); + return it != _map.end() ? it->second.lock() : std::shared_ptr(); + } + }; +} + + +std::shared_ptr ArchCreateMemStorageRO(const std::string& path, const void* data, size_t size) +{ + auto result = std::shared_ptr(new ArchMemFixedStorage(size, static_cast(data))); + ArchMemStorageRegistry::GetInst()->Set(path, result); + return result; +} + +std::shared_ptr ArchCreateMemStorageRW(const std::string& path) +{ + auto result = std::shared_ptr(new ArchMemGrowingStorage()); + ArchMemStorageRegistry::GetInst()->Set(path, result); + return result; +} + +class ArchMappingMemImpl : public ArchMappingImpl +{ + std::shared_ptr _memStorage; + + ArchMappingMemImpl(const std::shared_ptr& memStorage) : _memStorage(memStorage) {} + +public: + static ArchMappingMemImpl* Create(const std::shared_ptr& memStorage) + { + return (memStorage->GetPtrForMapping() != nullptr) ? new ArchMappingMemImpl(memStorage) : nullptr; + } + + virtual ~ArchMappingMemImpl() override {} + + virtual size_t GetLength() const override { return _memStorage->GetLength(); } + virtual void* GetPtr() const override { return const_cast(_memStorage->GetPtrForMapping()); } +}; + +class ArchMemFile : public ArchFile +{ + std::shared_ptr _memStorage; + + ArchMemFile(const std::shared_ptr& memStorage) : _memStorage(memStorage) {} + +public: + static ArchMemFile* Open(const char* fileName) + { + auto memStorage = ArchMemStorageRegistry::GetInst()->Get(fileName); + return memStorage ? new ArchMemFile(memStorage) : nullptr; + } + + virtual int64_t GetFileLength() override + { + return _memStorage->GetLength(); + } + + virtual ArchConstFileMapping MapFileReadOnly(std::string *errMsg) override + { + return ArchConstFileMapping(ArchMappingMemImpl::Create(_memStorage)); + } + virtual ArchMutableFileMapping MapFileReadWrite(std::string *errMsg) override + { + return ArchMutableFileMapping(); + } + + virtual int64_t PRead(void *buffer, size_t count, int64_t offset) override + { + return _memStorage->Read(static_cast(buffer), count, offset); + } + virtual int64_t PWrite(void const *bytes, size_t count, int64_t offset) override + { + return _memStorage->Write(static_cast(bytes), count, offset); + } + + virtual void FileAdvise(int64_t offset, size_t count, ArchFileAdvice adv) override + { + //do nothing + } + +}; + + +class ArchMappingDiscImpl : public ArchMappingImpl +{ + void* _ptr; + size_t _length; + + ArchMappingDiscImpl(void* ptr, size_t length) : _ptr(ptr), _length(length) {} + +public: + static ArchMappingDiscImpl* Create(FILE* file, std::string *errMsg, bool isConst) + { + auto length = ArchGetFileLength(file); + if (length < 0) + return nullptr; + +#if defined(ARCH_OS_WINDOWS) + uint64_t unsignedLength = length; + DWORD maxSizeHigh = static_cast(unsignedLength >> 32); + DWORD maxSizeLow = static_cast(unsignedLength); + HANDLE hFileMap = CreateFileMapping( + _FileToWinHANDLE(file), NULL, + PAGE_READONLY /* allow read-only or copy-on-write */, + maxSizeHigh, maxSizeLow, NULL); + if (hFileMap == NULL) + return nullptr; + void* ptr = MapViewOfFile(hFileMap, isConst ? FILE_MAP_READ : FILE_MAP_COPY, + /*offsetHigh=*/ 0, /*offsetLow=*/0, unsignedLength); + // Close the mapping handle, and return the view pointer. + CloseHandle(hFileMap); + return new ArchMappingDiscImpl(ptr, length); +#else // Assume POSIX + auto m = mmap(nullptr, length, + isConst ? PROT_READ : PROT_READ | PROT_WRITE, + MAP_PRIVATE, fileno(file), 0); + if (m == MAP_FAILED) { + if (errMsg) { + int err = errno; + if (err == EINVAL) { + *errMsg = "bad arguments to mmap()"; + } + else if (err == EMFILE || err == ENOMEM) { + *errMsg = "system limit on mapped regions exceeded, " + "or out of memory"; + } + else { + *errMsg = ArchStrerror(); + } + } + return nullptr; + } + return new ArchMappingDiscImpl(m, length); +#endif + } + + virtual ~ArchMappingDiscImpl() override + { + if (_ptr) + { +#if defined(ARCH_OS_WINDOWS) + UnmapViewOfFile(_ptr); +#else // assume POSIX + munmap(_ptr, _length); +#endif + } + } + + virtual size_t GetLength() const override { return _length; } + virtual void* GetPtr() const override { return _ptr; } +}; + + +class ArchDiscFile : public ArchFile +{ + FILE* _file; + + ArchDiscFile(FILE* file) : _file(file) {} + +public: + static ArchDiscFile* Open(char const* fileName, char const* mode) + { + FILE* file = fopen(fileName, mode); + return file ? new ArchDiscFile(file) : nullptr; + } + + static ArchDiscFile* Open(int fd, char const* mode) + { + FILE* file = ArchFdOpen(fd, mode); + return file ? new ArchDiscFile(file) : nullptr; + } + + virtual ~ArchDiscFile() + { + fclose(_file); + } + + inline FILE* GetFilePtr() { return _file; } + + virtual int64_t GetFileLength() override; + + virtual ArchConstFileMapping MapFileReadOnly(std::string *errMsg) override + { + return ArchConstFileMapping(ArchMappingDiscImpl::Create(_file, errMsg, true)); + } + virtual ArchMutableFileMapping MapFileReadWrite(std::string *errMsg) override + { + return ArchMutableFileMapping(ArchMappingDiscImpl::Create(_file, errMsg, false)); + } + + virtual int64_t PRead(void *buffer, size_t count, int64_t offset) override; + virtual int64_t PWrite(void const *bytes, size_t count, int64_t offset) override; + + virtual void FileAdvise(int64_t offset, size_t count, ArchFileAdvice adv) override; +}; + + +ArchFile* ArchOpenFile(char const* fileName, char const* mode) +{ + if (ArchIsMemoryPath(fileName)) + { + return ArchMemFile::Open(fileName); + } + return ArchDiscFile::Open(fileName, mode); +} + +ArchFile* ArchOpenFile(int fd, char const* mode) +{ + return ArchDiscFile::Open(fd, mode); +} + +void ArchReleaseFile(ArchFile* file) +{ + if (file) delete file; } #if defined(ARCH_OS_WINDOWS) @@ -190,14 +573,13 @@ _GetFileLength(HANDLE handle) #endif -int64_t -ArchGetFileLength(FILE *file) +int64_t ArchGetFileLength(FILE* file) { if (!file) return -1; #if defined (ARCH_OS_LINUX) || defined (ARCH_OS_DARWIN) struct stat buf; - return fstat(fileno(file), &buf) < 0 ? -1 : + return fstat(fileno(_file), &buf) < 0 ? -1 : static_cast(buf.st_size); #elif defined (ARCH_OS_WINDOWS) return _GetFileLength(_FileToWinHANDLE(file)); @@ -206,6 +588,11 @@ ArchGetFileLength(FILE *file) #endif } +int64_t ArchDiscFile::GetFileLength() +{ + return ArchGetFileLength(_file); +} + int64_t ArchGetFileLength(const char* fileName) { @@ -418,86 +805,6 @@ ArchGetTmpDir() return _TmpDir; } -void -Arch_Unmapper::operator()(char const *mapStart) const -{ - void *ptr = static_cast(const_cast(mapStart)); - if (!ptr) - return; -#if defined(ARCH_OS_WINDOWS) - UnmapViewOfFile(ptr); -#else // assume POSIX - munmap(ptr, _length); -#endif -} - -void -Arch_Unmapper::operator()(char *mapStart) const -{ - (*this)(static_cast(mapStart)); -} - -template -static inline Mapping -Arch_MapFileImpl(FILE *file, std::string *errMsg) -{ - using PtrType = typename Mapping::pointer; - constexpr bool isConst = - std::is_const::value; - - auto length = ArchGetFileLength(file); - if (length < 0) - return Mapping(); - -#if defined(ARCH_OS_WINDOWS) - uint64_t unsignedLength = length; - DWORD maxSizeHigh = static_cast(unsignedLength >> 32); - DWORD maxSizeLow = static_cast(unsignedLength); - HANDLE hFileMap = CreateFileMapping( - _FileToWinHANDLE(file), NULL, - PAGE_READONLY /* allow read-only or copy-on-write */, - maxSizeHigh, maxSizeLow, NULL); - if (hFileMap == NULL) - return Mapping(); - auto ptr = static_cast( - MapViewOfFile(hFileMap, isConst ? FILE_MAP_READ : FILE_MAP_COPY, - /*offsetHigh=*/ 0, /*offsetLow=*/0, unsignedLength)); - // Close the mapping handle, and return the view pointer. - CloseHandle(hFileMap); - return Mapping(ptr, Arch_Unmapper(length)); -#else // Assume POSIX - auto m = mmap(nullptr, length, - isConst ? PROT_READ : PROT_READ | PROT_WRITE, - MAP_PRIVATE, fileno(file), 0); - Mapping ret(m == MAP_FAILED ? nullptr : static_cast(m), - Arch_Unmapper(length)); - if (!ret && errMsg) { - int err = errno; - if (err == EINVAL) { - *errMsg = "bad arguments to mmap()"; - } else if (err == EMFILE || err == ENOMEM) { - *errMsg = "system limit on mapped regions exceeded, " - "or out of memory"; - } else { - *errMsg = ArchStrerror(); - } - } - return ret; -#endif -} - -ArchConstFileMapping -ArchMapFileReadOnly(FILE *file, std::string *errMsg) -{ - return Arch_MapFileImpl(file, errMsg); -} - -ArchMutableFileMapping -ArchMapFileReadWrite(FILE *file, std::string *errMsg) -{ - return Arch_MapFileImpl(file, errMsg); -} - ARCH_API void ArchMemAdvise(void const *addr, size_t len, ArchMemAdvice adv) { @@ -548,13 +855,13 @@ ArchQueryMappedMemoryResidency( } int64_t -ArchPRead(FILE *file, void *buffer, size_t count, int64_t offset) +ArchDiscFile::PRead(void *buffer, size_t count, int64_t offset) { if (count == 0) return 0; #if defined(ARCH_OS_WINDOWS) - HANDLE hFile = _FileToWinHANDLE(file); + HANDLE hFile = _FileToWinHANDLE(_file); OVERLAPPED overlapped; memset(&overlapped, 0, sizeof(overlapped)); @@ -571,7 +878,7 @@ ArchPRead(FILE *file, void *buffer, size_t count, int64_t offset) return -1; #else // assume POSIX // Read and check if all got read (most common case). - int fd = fileno(file); + int fd = fileno(_file); // Convert to signed so we can compare the result of pread with count // without the compiler complaining. This conversion is implementation // defined if count is larger than what's representable by int64_t, and @@ -603,13 +910,13 @@ ArchPRead(FILE *file, void *buffer, size_t count, int64_t offset) } int64_t -ArchPWrite(FILE *file, void const *bytes, size_t count, int64_t offset) +ArchDiscFile::PWrite(void const *bytes, size_t count, int64_t offset) { if (offset < 0) return -1; #if defined(ARCH_OS_WINDOWS) - HANDLE hFile = _FileToWinHANDLE(file); + HANDLE hFile = _FileToWinHANDLE(_file); OVERLAPPED overlapped; memset(&overlapped, 0, sizeof(overlapped)); @@ -630,7 +937,7 @@ ArchPWrite(FILE *file, void const *bytes, size_t count, int64_t offset) // written, or we get an error return. // Write and check if all got written (most common case). - int fd = fileno(file); + int fd = fileno(_file); // Convert to signed so we can compare the result of pwrite with count // without the compiler complaining. This conversion is implementation // defined if count is larger than what's representable by int64_t, and @@ -917,9 +1224,8 @@ ArchReadLink(const char* path) #endif -ARCH_API -void ArchFileAdvise( - FILE *file, int64_t offset, size_t count, ArchFileAdvice adv) + +void ArchDiscFile::FileAdvise(int64_t offset, size_t count, ArchFileAdvice adv) { #if defined(ARCH_OS_WINDOWS) // No windows implementation yet. Not clear what's equivalent. @@ -933,12 +1239,12 @@ void ArchFileAdvise( /* ArchFileAdviceDontNeed = */ POSIX_FADV_DONTNEED, /* ArchFileAdviceRandomAccess = */ POSIX_FADV_RANDOM }; - int rval = posix_fadvise(fileno(file), offset, static_cast(count), + int rval = posix_fadvise(fileno(_file), offset, static_cast(count), adviceMap[adv]); if (rval != 0) { fprintf(stderr, "failed call to posix_fadvise(%d, %zd, %zd)" "ret=%d, errno=%d '%s'\n", - fileno(file), offset, static_cast(count), + fileno(_file), offset, static_cast(count), rval, errno, ArchStrerror().c_str()); } #endif diff --git a/third_party/usd/src/pxr/base/lib/arch/fileSystem.h b/third_party/usd/src/pxr/base/lib/arch/fileSystem.h index 7b9252e6f6..76a4642db9 100644 --- a/third_party/usd/src/pxr/base/lib/arch/fileSystem.h +++ b/third_party/usd/src/pxr/base/lib/arch/fileSystem.h @@ -118,6 +118,85 @@ typedef struct __stat64 ArchStatType; typedef struct stat ArchStatType; #endif +enum ArchFileAdvice { + ArchFileAdviceNormal, // Treat range with default behavior. + ArchFileAdviceWillNeed, // OS may prefetch this range. + ArchFileAdviceDontNeed, // OS may free resources related to this range. + ArchFileAdviceRandomAccess, // Prefetching may not be beneficial. +}; + +class ArchMappingImpl +{ +protected: + ArchMappingImpl() = default; +public: + virtual ~ArchMappingImpl() = default; + virtual size_t GetLength() const = 0; + virtual void* GetPtr() const = 0; +}; + +template +class ArchMapping +{ + ArchMappingImpl* _impl; + +public: + ArchMapping() noexcept : _impl(nullptr) {} + explicit ArchMapping(ArchMappingImpl* impl) noexcept : _impl(impl) {} + + ~ArchMapping() noexcept { if (_impl) delete _impl; } + + ArchMapping(const ArchMapping&) = delete; + ArchMapping& operator=(const ArchMapping&) = delete; + + ArchMapping(ArchMapping&& other) noexcept : _impl(other._impl) { other._impl = nullptr; } + ArchMapping& operator=(ArchMapping&& other) noexcept + { + if (_impl != other._impl) + { + if (_impl) delete _impl; + _impl = other._impl; + other._impl = nullptr; + } + return *this; + } + + T* get() const noexcept { return _impl ? static_cast(_impl->GetPtr()) : nullptr; } + + size_t length() const noexcept { return _impl ? _impl->GetLength() : 0; } + + explicit operator bool() const noexcept { return _impl != nullptr; } +}; + + +/// ArchConstFileMapping and ArchMutableFileMapping are std::unique_ptr and std::unique_ptr respectively. The functions +/// ArchMapFileReadOnly() and ArchMapFileReadWrite() return them and provide +/// access to memory-mapped file contents. +using ArchConstFileMapping = ArchMapping; +using ArchMutableFileMapping = ArchMapping; + +// ArchFile abstract class +class ArchFile +{ +protected: + ArchFile() {} + +public: + virtual ~ArchFile() = 0 {} + + virtual int64_t GetFileLength() = 0; + + virtual ArchConstFileMapping MapFileReadOnly(std::string *errMsg) = 0; + virtual ArchMutableFileMapping MapFileReadWrite(std::string *errMsg) = 0; + + virtual int64_t PRead(void *buffer, size_t count, int64_t offset) = 0; + virtual int64_t PWrite(void const *bytes, size_t count, int64_t offset) = 0; + + virtual void FileAdvise(int64_t offset, size_t count, ArchFileAdvice adv) = 0; +}; + + /// \file fileSystem.h /// Architecture dependent file system access /// \ingroup group_arch_SystemFunctions @@ -128,9 +207,14 @@ typedef struct stat ArchStatType; /// Opens the file that is specified by filename. /// Returning true if the file was opened successfully; false otherwise. /// -ARCH_API FILE* +ARCH_API ArchFile* ArchOpenFile(char const* fileName, char const* mode); +ARCH_API ArchFile* +ArchOpenFile(int fd, char const* mode); + +ARCH_API void ArchReleaseFile(ArchFile* file); + #if defined(ARCH_OS_WINDOWS) # define ArchChmod(path, mode) _chmod(path, mode) #else @@ -183,7 +267,8 @@ ArchOpenFile(char const* fileName, char const* mode); /// /// Returns -1 if the file cannot be opened/read. ARCH_API int64_t ArchGetFileLength(const char* fileName); -ARCH_API int64_t ArchGetFileLength(FILE *file); +ARCH_API int64_t ArchGetFileLength(FILE* fileName); +ARCH_API inline int64_t ArchGetFileLength(ArchFile *file) { return file->GetFileLength(); } /// Returns true if the data in \c stat struct \p st indicates that the target /// file or directory is writable. @@ -277,34 +362,17 @@ ARCH_API std::string ArchMakeTmpSubdir(const std::string& tmpdir, const std::string& prefix); -// Helper 'deleter' for use with std::unique_ptr for file mappings. -struct Arch_Unmapper { - Arch_Unmapper() : _length(~0) {} - explicit Arch_Unmapper(size_t length) : _length(length) {} - ARCH_API void operator()(char *mapStart) const; - ARCH_API void operator()(char const *mapStart) const; - size_t GetLength() const { return _length; } -private: - size_t _length; -}; - -/// ArchConstFileMapping and ArchMutableFileMapping are std::unique_ptr and std::unique_ptr respectively. The functions -/// ArchMapFileReadOnly() and ArchMapFileReadWrite() return them and provide -/// access to memory-mapped file contents. -using ArchConstFileMapping = std::unique_ptr; -using ArchMutableFileMapping = std::unique_ptr; /// Return the length of an ArchConstFileMapping. inline size_t ArchGetFileMappingLength(ArchConstFileMapping const &m) { - return m.get_deleter().GetLength(); + return m.length(); } /// Return the length of an ArchMutableFileMapping. inline size_t ArchGetFileMappingLength(ArchMutableFileMapping const &m) { - return m.get_deleter().GetLength(); + return m.length(); } /// Privately map the passed \p file into memory and return a unique_ptr to the @@ -313,7 +381,7 @@ ArchGetFileMappingLength(ArchMutableFileMapping const &m) { /// information about the failure. ARCH_API ArchConstFileMapping -ArchMapFileReadOnly(FILE *file, std::string *errMsg=nullptr); +inline ArchMapFileReadOnly(ArchFile *file, std::string *errMsg = nullptr) { return file->MapFileReadOnly(errMsg); } /// Privately map the passed \p file into memory and return a unique_ptr to the /// copy-on-write mapped contents. If modified, the affected pages are @@ -323,7 +391,7 @@ ArchMapFileReadOnly(FILE *file, std::string *errMsg=nullptr); /// with information about the failure. ARCH_API ArchMutableFileMapping -ArchMapFileReadWrite(FILE *file, std::string *errMsg=nullptr); +inline ArchMapFileReadWrite(ArchFile *file, std::string *errMsg=nullptr) { return file->MapFileReadWrite(errMsg); } enum ArchMemAdvice { ArchMemAdviceNormal, // Treat range with default behavior. @@ -361,34 +429,48 @@ ArchQueryMappedMemoryResidency( /// bytes read, or zero if at end of file. Return -1 in case of an error, with /// errno set appropriately. ARCH_API -int64_t ArchPRead(FILE *file, void *buffer, size_t count, int64_t offset); +inline int64_t ArchPRead(ArchFile *file, void *buffer, size_t count, int64_t offset) { return file->PRead(buffer, count, offset); } /// Write up to \p count bytes from \p buffer to \p file at \p offset. The file /// position indicator for \p file is not changed. Return the number of bytes /// written, possibly zero if none written. Return -1 in case of an error, with /// errno set appropriately. ARCH_API -int64_t ArchPWrite(FILE *file, void const *bytes, size_t count, int64_t offset); +inline int64_t ArchPWrite(ArchFile *file, void const *bytes, size_t count, int64_t offset) { return file->PWrite(bytes, count, offset); } /// Returns the value of the symbolic link at \p path. Returns the empty /// string on error or if \p path does not refer to a symbolic link. ARCH_API std::string ArchReadLink(const char* path); -enum ArchFileAdvice { - ArchFileAdviceNormal, // Treat range with default behavior. - ArchFileAdviceWillNeed, // OS may prefetch this range. - ArchFileAdviceDontNeed, // OS may free resources related to this range. - ArchFileAdviceRandomAccess, // Prefetching may not be beneficial. -}; - /// Advise the OS regarding how the application intends to access a range of /// bytes in a file. See ArchFileAdvice. This call does not change program /// semantics. It is only an optimization hint to the OS, and may be a no-op on /// some systems. ARCH_API -void ArchFileAdvise(FILE *file, int64_t offset, size_t count, - ArchFileAdvice adv); +inline void ArchFileAdvise(ArchFile *file, int64_t offset, size_t count, ArchFileAdvice adv) { file->FileAdvise(offset, count, adv); } + + +ARCH_API inline bool ArchIsMemoryPath(const char* path) { return (strncmp(path, "mem://", 6) == 0); } + +class ArchMemStorage +{ +protected: + ArchMemStorage() {} + +public: + virtual ~ArchMemStorage() {} + + virtual size_t GetLength() const = 0; + + virtual size_t Read(uint8_t* data, size_t count, size_t offset) = 0; + virtual size_t Write(const uint8_t* data, size_t count, size_t offset) = 0; + + virtual const void* GetPtrForMapping() const = 0; +}; + +ARCH_API std::shared_ptr ArchCreateMemStorageRO(const std::string& path, const void* data, size_t size); +ARCH_API std::shared_ptr ArchCreateMemStorageRW(const std::string& path); ///@} diff --git a/third_party/usd/src/pxr/base/lib/arch/stackTrace.cpp b/third_party/usd/src/pxr/base/lib/arch/stackTrace.cpp index b1f0ba6b21..d18fbac2f8 100644 --- a/third_party/usd/src/pxr/base/lib/arch/stackTrace.cpp +++ b/third_party/usd/src/pxr/base/lib/arch/stackTrace.cpp @@ -830,8 +830,8 @@ _FinishLoggingFatalStackTrace(const char *progname, const char *stackTrace, { if (!crashingHard && sessionLog) { // If we were given a session log, cat it to the end of the stack. - if (FILE* stackFd = ArchOpenFile(stackTrace, "a")) { - if (FILE* sessionLogFd = ArchOpenFile(sessionLog, "r")) { + if (FILE* stackFd = fopen(stackTrace, "a")) { + if (FILE* sessionLogFd = fopen(sessionLog, "r")) { fputs("\n\n********** Session Log **********\n\n", stackFd); // Cat the sesion log char line[4096]; @@ -907,7 +907,7 @@ ArchLogPostMortem(const char* reason, } // Write reason for stack trace to logfile. - if (FILE* stackFd = ArchOpenFile(logfile, "a")) { + if (FILE* stackFd = fopen(logfile, "a")) { if (reason) { fputs("This stack trace was requested because: ", stackFd); fputs(reason, stackFd); diff --git a/third_party/usd/src/pxr/base/lib/tf/envSetting.cpp b/third_party/usd/src/pxr/base/lib/tf/envSetting.cpp index 704a08fcba..0431464910 100644 --- a/third_party/usd/src/pxr/base/lib/tf/envSetting.cpp +++ b/third_party/usd/src/pxr/base/lib/tf/envSetting.cpp @@ -57,7 +57,7 @@ class Tf_EnvSettingRegistry { Tf_EnvSettingRegistry() { string fileName = TfGetenv("PIXAR_TF_ENV_SETTING_FILE", ""); - if (FILE* fp = ArchOpenFile(fileName.c_str(), "r")) { + if (FILE* fp = fopen(fileName.c_str(), "r")) { char buffer[1024]; #ifdef PXR_PYTHON_SUPPORT_ENABLED diff --git a/third_party/usd/src/pxr/base/lib/tf/pathUtils.cpp b/third_party/usd/src/pxr/base/lib/tf/pathUtils.cpp index cdb04488fe..a86604502a 100644 --- a/third_party/usd/src/pxr/base/lib/tf/pathUtils.cpp +++ b/third_party/usd/src/pxr/base/lib/tf/pathUtils.cpp @@ -429,7 +429,7 @@ TfNormPath(string const &inPath) string TfAbsPath(string const& path) { - if (path.empty()) { + if (path.empty() || ArchIsMemoryPath(path.c_str())) { return path; } diff --git a/third_party/usd/src/pxr/base/lib/tf/pyInterpreter.cpp b/third_party/usd/src/pxr/base/lib/tf/pyInterpreter.cpp index 4c6b486270..1950e37d4c 100644 --- a/third_party/usd/src/pxr/base/lib/tf/pyInterpreter.cpp +++ b/third_party/usd/src/pxr/base/lib/tf/pyInterpreter.cpp @@ -169,7 +169,7 @@ boost::python::handle<> TfPyRunFile(const std::string &filename, int start, object const &globals, object const &locals) { - FILE *f = ArchOpenFile(filename.c_str(), "r"); + FILE *f = fopen(filename.c_str(), "r"); if (!f) { TF_CODING_ERROR("Could not open file '%s'!", filename.c_str()); return handle<>(); diff --git a/third_party/usd/src/pxr/base/lib/tf/safeOutputFile.cpp b/third_party/usd/src/pxr/base/lib/tf/safeOutputFile.cpp index 9fa0a1eaee..606e511afe 100644 --- a/third_party/usd/src/pxr/base/lib/tf/safeOutputFile.cpp +++ b/third_party/usd/src/pxr/base/lib/tf/safeOutputFile.cpp @@ -52,7 +52,7 @@ TfSafeOutputFile::IsOpenForUpdate() const return _file && _tempFileName.empty(); } -FILE * +ArchFile * TfSafeOutputFile::ReleaseUpdatedFile() { if (!IsOpenForUpdate()) { @@ -60,7 +60,7 @@ TfSafeOutputFile::ReleaseUpdatedFile() "replace)"); return nullptr; } - FILE *ret = _file; + ArchFile *ret = _file; _file = nullptr; _tempFileName.clear(); _targetFileName.clear(); @@ -74,7 +74,7 @@ TfSafeOutputFile::Close() return; // Close the file. - fclose(_file); + ArchReleaseFile(_file); _file = nullptr; // If this was for update, we have nothing else to do. @@ -95,7 +95,7 @@ TfSafeOutputFile::Update(std::string const &fileName) { TfSafeOutputFile result; result._targetFileName = fileName; - FILE *file = ArchOpenFile(fileName.c_str(), "r+"); + ArchFile *file = ArchOpenFile(fileName.c_str(), "r+"); if (!file) { TF_RUNTIME_ERROR("Unable to open file '%s' for writing", fileName.c_str()); @@ -108,6 +108,10 @@ TfSafeOutputFile::Update(std::string const &fileName) TfSafeOutputFile TfSafeOutputFile::Replace(std::string const &fileName) { + if (ArchIsMemoryPath(fileName.c_str())) { + return TfSafeOutputFile::Update(fileName); + } + TfSafeOutputFile result; std::string error; int tmpFd = Tf_CreateSiblingTempFile(fileName, @@ -120,7 +124,7 @@ TfSafeOutputFile::Replace(std::string const &fileName) } // Obtain a FILE *. - result._file = ArchFdOpen(tmpFd, "w"); + result._file = ArchOpenFile(tmpFd, "w"); if (!result._file) { TF_RUNTIME_ERROR("Unable to obtain writable FILE pointer: %s", ArchStrerror(errno).c_str()); diff --git a/third_party/usd/src/pxr/base/lib/tf/safeOutputFile.h b/third_party/usd/src/pxr/base/lib/tf/safeOutputFile.h index 139be2f31d..7ea45ec737 100644 --- a/third_party/usd/src/pxr/base/lib/tf/safeOutputFile.h +++ b/third_party/usd/src/pxr/base/lib/tf/safeOutputFile.h @@ -36,6 +36,8 @@ PXR_NAMESPACE_OPEN_SCOPE +class ArchFile; + /// \class TfSafeOutputFile /// /// Opens a file for output, either for update "r+" or to completely replace @@ -83,19 +85,19 @@ class TfSafeOutputFile TF_API void Close(); /// Return the opened FILE *. - FILE *Get() const { return _file; } + ArchFile *Get() const { return _file; } /// If the underlying file was opened by Update(), return it. The caller /// takes responsibility for closing the file later. It is an error to call /// this for files opened for Replace. - TF_API FILE *ReleaseUpdatedFile(); + TF_API ArchFile *ReleaseUpdatedFile(); /// Return true if this TfSafeOutputFile was created by a call to Update(), /// false otherwise. TF_API bool IsOpenForUpdate() const; private: - FILE *_file = nullptr; + ArchFile *_file = nullptr; std::string _targetFileName; std::string _tempFileName; }; diff --git a/third_party/usd/src/pxr/base/lib/tf/scriptModuleLoader.cpp b/third_party/usd/src/pxr/base/lib/tf/scriptModuleLoader.cpp index 175ac266cb..2d7b38d920 100644 --- a/third_party/usd/src/pxr/base/lib/tf/scriptModuleLoader.cpp +++ b/third_party/usd/src/pxr/base/lib/tf/scriptModuleLoader.cpp @@ -169,7 +169,7 @@ TfScriptModuleLoader::GetModulesDict() const void TfScriptModuleLoader::WriteDotFile(string const &file) const { - FILE *out = ArchOpenFile(file.c_str(), "w"); + FILE *out = fopen(file.c_str(), "w"); if (!out) { TF_RUNTIME_ERROR("Could not open '%s' for writing.\n", file.c_str()); return; diff --git a/third_party/usd/src/pxr/usd/lib/sdf/textFileFormat.cpp b/third_party/usd/src/pxr/usd/lib/sdf/textFileFormat.cpp index f809a6c603..8ca436caf6 100644 --- a/third_party/usd/src/pxr/usd/lib/sdf/textFileFormat.cpp +++ b/third_party/usd/src/pxr/usd/lib/sdf/textFileFormat.cpp @@ -105,7 +105,7 @@ SdfTextFileFormat::CanRead(const string& filePath) const TRACE_FUNCTION(); bool canRead = false; - if (FILE *f = ArchOpenFile(filePath.c_str(), "rb")) { + if (FILE *f = fopen(filePath.c_str(), "rb")) { canRead = _CanReadImpl(f); fclose(f); } @@ -125,7 +125,7 @@ class Sdf_ScopedFilePointer : boost::noncopyable { public: explicit Sdf_ScopedFilePointer(const string& filePath) - : _fp(ArchOpenFile(filePath.c_str(), "rb")) + : _fp(fopen(filePath.c_str(), "rb")) { } ~Sdf_ScopedFilePointer() { diff --git a/third_party/usd/src/pxr/usd/lib/usd/crateFile.cpp b/third_party/usd/src/pxr/usd/lib/usd/crateFile.cpp index 27b017ff8a..9a438adc5f 100644 --- a/third_party/usd/src/pxr/usd/lib/usd/crateFile.cpp +++ b/third_party/usd/src/pxr/usd/lib/usd/crateFile.cpp @@ -124,7 +124,7 @@ static int _GetMMapPrefetchKB() // Write nbytes bytes to fd at pos. static inline int64_t -WriteToFd(FILE *file, void const *bytes, int64_t nbytes, int64_t pos) { +WriteToFd(ArchFile *file, void const *bytes, int64_t nbytes, int64_t pos) { int64_t nwritten = ArchPWrite(file, bytes, nbytes, pos); if (ARCH_UNLIKELY(nwritten < 0)) { TF_RUNTIME_ERROR("Failed writing usdc data: %s", @@ -463,7 +463,7 @@ struct _MmapStream { }; struct _PreadStream { - explicit _PreadStream(FILE *file) : _cur(0), _file(file) {} + explicit _PreadStream(ArchFile *file) : _cur(0), _file(file) {} inline void Read(void *dest, size_t nBytes) { _cur += ArchPRead(_file, dest, nBytes, _cur); } @@ -475,7 +475,7 @@ struct _PreadStream { private: int64_t _cur; - FILE *_file; + ArchFile *_file; }; //////////////////////////////////////////////////////////////////////// @@ -521,7 +521,7 @@ class CrateFile::_BufferedOutput int64_t size = 0; }; - explicit _BufferedOutput(FILE *file) + explicit _BufferedOutput(ArchFile *file) : _filePos(0) , _file(file) , _bufferPos(0) @@ -630,7 +630,7 @@ class CrateFile::_BufferedOutput // Write head in the file. Always inside the buffer region. int64_t _filePos; - FILE *_file; + ArchFile *_file; // Start of current buffer is at this file offset. int64_t _bufferPos; @@ -1404,11 +1404,11 @@ CrateFile::CreateNew() /* static */ ArchConstFileMapping -CrateFile::_MmapFile(char const *fileName, FILE *file) +CrateFile::_MmapFile(char const *fileName, ArchFile *file) { ArchConstFileMapping map = ArchMapFileReadOnly(file); if (!map) - TF_RUNTIME_ERROR("Couldn't map file '%s'", fileName); + TF_DIAGNOSTIC_WARNING("Couldn't map file '%s'", fileName); return map; } @@ -1428,9 +1428,9 @@ CrateFile::Open(string const &fileName) return result; } - if (!TfGetenvBool("USDC_USE_PREAD", false)) { + ArchConstFileMapping mapStart; + if (!TfGetenvBool("USDC_USE_PREAD", false) && (mapStart = _MmapFile(fileName.c_str(), inputFile.get()))) { // Map the file. - auto mapStart = _MmapFile(fileName.c_str(), inputFile.get()); result.reset(new CrateFile(fileName, std::move(mapStart))); } else { result.reset(new CrateFile(fileName, std::move(inputFile))); @@ -1519,6 +1519,7 @@ CrateFile::CrateFile(string const &fileName, _UniqueFILE inputFile) , _useMmap(false) { _DoAllTypeRegistrations(); + _InitPread(); } void @@ -1542,7 +1543,7 @@ CrateFile::~CrateFile() static std::mutex outputMutex; // Dump a debug page map if requested. - if (_useMmap && _mapStart && _debugPageMap) { + if (_mapStart && _debugPageMap) { int64_t length = ArchGetFileMappingLength(_mapStart); int64_t npages = (length + PAGESIZE-1) / PAGESIZE; std::unique_ptr mincoreMap(new unsigned char[npages]); @@ -1660,11 +1661,8 @@ CrateFile::Packer::Close() // Reset the mapping or file so we can read values from the newly // written file. - if (_crate->_useMmap) { + if (_crate->_useMmap && (_crate->_mapStart = _MmapFile(_crate->_fileName.c_str(), file.get()))) { // Must remap the file. - _crate->_mapStart = _MmapFile(_crate->_fileName.c_str(), file.get()); - if (!_crate->_mapStart) - return false; _crate->_InitMMap(); } else { // Must adopt the file handle if we don't already have one. @@ -1921,7 +1919,7 @@ CrateFile::_GetTimeSampleValueImpl(TimeSamples const &ts, size_t i) const { // Need to read the rep from the file for index i. auto offset = ts.valuesFileOffset + i * sizeof(ValueRep); - if (_useMmap) { + if (_mapStart) { auto reader = _MakeReader(_MmapStream(_mapStart, _debugPageMap.get())); reader.Seek(offset); return VtValue(reader.Read()); @@ -1937,7 +1935,7 @@ CrateFile::_MakeTimeSampleValuesMutableImpl(TimeSamples &ts) const { // Read out the reps into the vector. ts.values.resize(ts.times.Get().size()); - if (_useMmap) { + if (_mapStart) { auto reader = _MakeReader(_MmapStream(_mapStart, _debugPageMap.get())); reader.Seek(ts.valuesFileOffset); for (size_t i = 0, n = ts.times.Get().size(); i != n; ++i) @@ -2809,7 +2807,7 @@ CrateFile::_BuildDecompressedPathsImpl( void CrateFile::_ReadRawBytes(int64_t start, int64_t size, char *buf) const { - if (_useMmap) { + if (_mapStart) { auto reader = _MakeReader(_MmapStream(_mapStart, _debugPageMap.get())); reader.Seek(start); reader.template ReadContiguous(buf, size); @@ -2971,7 +2969,7 @@ void CrateFile::_UnpackValue(ValueRep rep, T *out) const { auto const &h = _GetValueHandler(); - if (_useMmap) { + if (_mapStart) { h.Unpack(_MakeReader(_MmapStream(_mapStart, _debugPageMap.get())), rep, out); } else { @@ -2983,7 +2981,7 @@ template void CrateFile::_UnpackValue(ValueRep rep, VtArray *out) const { auto const &h = _GetValueHandler(); - if (_useMmap) { + if (_mapStart) { h.UnpackArray(_MakeReader(_MmapStream(_mapStart, _debugPageMap.get())), rep, out); } else { @@ -3001,7 +2999,7 @@ CrateFile::_UnpackValue(ValueRep rep, VtValue *result) const { return; } auto index = static_cast(repType); - if (_useMmap) { + if (_mapStart) { _unpackValueFunctionsMmap[index](rep, result); } else { _unpackValueFunctionsPread[index](rep, result); @@ -3110,11 +3108,9 @@ CrateFile::_IsKnownSection(char const *name) { } void -CrateFile::_Fcloser::operator()(FILE *f) const +CrateFile::_Fcloser::operator()(ArchFile *f) const { - if (f) { - fclose(f); - } + ArchReleaseFile(f); } CrateFile::Spec::Spec(Spec_0_0_1 const &s) diff --git a/third_party/usd/src/pxr/usd/lib/usd/crateFile.h b/third_party/usd/src/pxr/usd/lib/usd/crateFile.h index 39567a4932..217620ef75 100644 --- a/third_party/usd/src/pxr/usd/lib/usd/crateFile.h +++ b/third_party/usd/src/pxr/usd/lib/usd/crateFile.h @@ -269,9 +269,9 @@ class CrateFile private: struct _Fcloser { - void operator()(FILE *f) const; + void operator()(ArchFile *f) const; }; - typedef std::unique_ptr _UniqueFILE; + typedef std::unique_ptr _UniqueFILE; //////////////////////////////////////////////////////////////////////// @@ -530,7 +530,7 @@ class CrateFile void _InitMMap(); void _InitPread(); - static ArchConstFileMapping _MmapFile(char const *fileName, FILE *file); + static ArchConstFileMapping _MmapFile(char const *fileName, ArchFile *file); class _Writer; class _BufferedOutput;