Usd source modified to support in-memory-file read/write (PixarAnimat…

…ionStudios#20)

* [USD] changed FILE* to ArchFile* for binary save/load

* Fixes for Debug build

* Memory Storage (working but not final version)

* Memory storage fixes & cleanups

* Implemented file mapping for ReadOnly memory storage

* Fix for array diff in USD text format

* Test for usd diff refactored (added command args parsing and write to diff.usda)

third_party/usd/src/pxr/base/lib/arch/fileSystem.h

@@ -118,6 +118,85 @@ typedef struct __stat64 ArchStatType;
 typedef struct stat ArchStatType;
 #endif
 
+enum ArchFileAdvice {
+	ArchFileAdviceNormal,       // Treat range with default behavior.
+	ArchFileAdviceWillNeed,     // OS may prefetch this range.
+	ArchFileAdviceDontNeed,     // OS may free resources related to this range.
+	ArchFileAdviceRandomAccess, // Prefetching may not be beneficial.
+};
+
+class ArchMappingImpl
+{
+protected:
+	ArchMappingImpl() = default;
+public:
+	virtual ~ArchMappingImpl() = default;
+	virtual size_t GetLength() const = 0;
+	virtual void* GetPtr() const = 0;
+};
+
+template <typename T>
+class ArchMapping
+{
+	ArchMappingImpl* _impl;
+
+public:
+	ArchMapping() noexcept : _impl(nullptr) {}
+	explicit ArchMapping(ArchMappingImpl* impl) noexcept : _impl(impl) {}
+
+	~ArchMapping() noexcept { if (_impl) delete _impl; }
+
+	ArchMapping(const ArchMapping&) = delete;
+	ArchMapping& operator=(const ArchMapping&) = delete;
+
+	ArchMapping(ArchMapping&& other) noexcept : _impl(other._impl) { other._impl = nullptr; }
+	ArchMapping& operator=(ArchMapping&& other) noexcept
+	{
+		if (_impl != other._impl)
+		{
+			if (_impl) delete _impl;
+			_impl = other._impl;
+			other._impl = nullptr;
+		}
+		return *this;
+	}
+
+	T* get() const noexcept { return _impl ? static_cast<T*>(_impl->GetPtr()) : nullptr; }
+
+	size_t length() const noexcept { return _impl ? _impl->GetLength() : 0; }
+
+	explicit operator bool() const noexcept { return _impl != nullptr; }
+};
+
+
+/// ArchConstFileMapping and ArchMutableFileMapping are std::unique_ptr<char
+/// const *, ...> and std::unique_ptr<char *, ...> respectively.  The functions
+/// ArchMapFileReadOnly() and ArchMapFileReadWrite() return them and provide
+/// access to memory-mapped file contents.
+using ArchConstFileMapping = ArchMapping<char const>;
+using ArchMutableFileMapping = ArchMapping<char>;
+
+// ArchFile abstract class
+class ArchFile
+{
+protected:
+	ArchFile() {}
+
+public:
+	virtual ~ArchFile() = 0 {}
+
+	virtual int64_t GetFileLength() = 0;
+
+	virtual ArchConstFileMapping MapFileReadOnly(std::string *errMsg) = 0;
+	virtual ArchMutableFileMapping MapFileReadWrite(std::string *errMsg) = 0;
+
+	virtual int64_t PRead(void *buffer, size_t count, int64_t offset) = 0;
+	virtual int64_t PWrite(void const *bytes, size_t count, int64_t offset) = 0;
+
+	virtual void FileAdvise(int64_t offset, size_t count, ArchFileAdvice adv) = 0;
+};
+
+
 /// \file fileSystem.h
 /// Architecture dependent file system access
 /// \ingroup group_arch_SystemFunctions
@@ -128,9 +207,14 @@ typedef struct stat ArchStatType;
 /// Opens the file that is specified by filename.
 /// Returning true if the file was opened successfully; false otherwise.
 ///
-ARCH_API FILE*
+ARCH_API ArchFile*
 ArchOpenFile(char const* fileName, char const* mode);
 
+ARCH_API ArchFile*
+ArchOpenFile(int fd, char const* mode);
+
+ARCH_API void ArchReleaseFile(ArchFile* file);
+
 #if defined(ARCH_OS_WINDOWS)
 #   define ArchChmod(path, mode)        _chmod(path, mode)
 #else
@@ -183,7 +267,8 @@ ArchOpenFile(char const* fileName, char const* mode);
 ///
 /// Returns -1 if the file cannot be opened/read.
 ARCH_API int64_t ArchGetFileLength(const char* fileName);
-ARCH_API int64_t ArchGetFileLength(FILE *file);
+ARCH_API int64_t ArchGetFileLength(FILE* fileName);
+ARCH_API inline int64_t ArchGetFileLength(ArchFile *file) { return file->GetFileLength(); }
 
 /// Returns true if the data in \c stat struct \p st indicates that the target
 /// file or directory is writable.
@@ -277,34 +362,17 @@ ARCH_API
 std::string ArchMakeTmpSubdir(const std::string& tmpdir,
                               const std::string& prefix);
 
-// Helper 'deleter' for use with std::unique_ptr for file mappings.
-struct Arch_Unmapper {
-    Arch_Unmapper() : _length(~0) {}
-    explicit Arch_Unmapper(size_t length) : _length(length) {}
-    ARCH_API void operator()(char *mapStart) const;
-    ARCH_API void operator()(char const *mapStart) const;
-    size_t GetLength() const { return _length; }
-private:
-    size_t _length;
-};
-
-/// ArchConstFileMapping and ArchMutableFileMapping are std::unique_ptr<char
-/// const *, ...> and std::unique_ptr<char *, ...> respectively.  The functions
-/// ArchMapFileReadOnly() and ArchMapFileReadWrite() return them and provide
-/// access to memory-mapped file contents.
-using ArchConstFileMapping = std::unique_ptr<char const, Arch_Unmapper>;
-using ArchMutableFileMapping = std::unique_ptr<char, Arch_Unmapper>;
 
 /// Return the length of an ArchConstFileMapping.
 inline size_t
 ArchGetFileMappingLength(ArchConstFileMapping const &m) {
-    return m.get_deleter().GetLength();
+    return m.length();
 }
 
 /// Return the length of an ArchMutableFileMapping.
 inline size_t
 ArchGetFileMappingLength(ArchMutableFileMapping const &m) {
-    return m.get_deleter().GetLength();
+    return m.length();
 }
 
 /// Privately map the passed \p file into memory and return a unique_ptr to the
@@ -313,7 +381,7 @@ ArchGetFileMappingLength(ArchMutableFileMapping const &m) {
 /// information about the failure.
 ARCH_API
 ArchConstFileMapping
-ArchMapFileReadOnly(FILE *file, std::string *errMsg=nullptr);
+inline ArchMapFileReadOnly(ArchFile *file, std::string *errMsg = nullptr) { return file->MapFileReadOnly(errMsg); }
 
 /// Privately map the passed \p file into memory and return a unique_ptr to the
 /// copy-on-write mapped contents.  If modified, the affected pages are
@@ -323,7 +391,7 @@ ArchMapFileReadOnly(FILE *file, std::string *errMsg=nullptr);
 /// with information about the failure.
 ARCH_API
 ArchMutableFileMapping
-ArchMapFileReadWrite(FILE *file, std::string *errMsg=nullptr);
+inline ArchMapFileReadWrite(ArchFile *file, std::string *errMsg=nullptr) { return file->MapFileReadWrite(errMsg); }
 
 enum ArchMemAdvice {
     ArchMemAdviceNormal,       // Treat range with default behavior.
@@ -361,34 +429,48 @@ ArchQueryMappedMemoryResidency(
 /// bytes read, or zero if at end of file.  Return -1 in case of an error, with
 /// errno set appropriately.
 ARCH_API
-int64_t ArchPRead(FILE *file, void *buffer, size_t count, int64_t offset);
+inline int64_t ArchPRead(ArchFile *file, void *buffer, size_t count, int64_t offset) { return file->PRead(buffer, count, offset); }
 
 /// Write up to \p count bytes from \p buffer to \p file at \p offset.  The file
 /// position indicator for \p file is not changed.  Return the number of bytes
 /// written, possibly zero if none written.  Return -1 in case of an error, with
 /// errno set appropriately.
 ARCH_API
-int64_t ArchPWrite(FILE *file, void const *bytes, size_t count, int64_t offset);
+inline int64_t ArchPWrite(ArchFile *file, void const *bytes, size_t count, int64_t offset) { return file->PWrite(bytes, count, offset); }
 
 /// Returns the value of the symbolic link at \p path.  Returns the empty
 /// string on error or if \p path does not refer to a symbolic link.
 ARCH_API
 std::string ArchReadLink(const char* path);
 
-enum ArchFileAdvice {
-    ArchFileAdviceNormal,       // Treat range with default behavior.
-    ArchFileAdviceWillNeed,     // OS may prefetch this range.
-    ArchFileAdviceDontNeed,     // OS may free resources related to this range.
-    ArchFileAdviceRandomAccess, // Prefetching may not be beneficial.
-};
-
 /// Advise the OS regarding how the application intends to access a range of
 /// bytes in a file.  See ArchFileAdvice.  This call does not change program
 /// semantics.  It is only an optimization hint to the OS, and may be a no-op on
 /// some systems.
 ARCH_API
-void ArchFileAdvise(FILE *file, int64_t offset, size_t count,
-                    ArchFileAdvice adv);
+inline void ArchFileAdvise(ArchFile *file, int64_t offset, size_t count, ArchFileAdvice adv) { file->FileAdvise(offset, count, adv); }
+
+
+ARCH_API inline bool ArchIsMemoryPath(const char* path) { return (strncmp(path, "mem://", 6) == 0); }
+
+class ArchMemStorage
+{
+protected:
+	ArchMemStorage() {}
+
+public:
+	virtual ~ArchMemStorage() {}
+
+	virtual size_t GetLength() const = 0;
+
+	virtual size_t Read(uint8_t* data, size_t count, size_t offset) = 0;
+	virtual size_t Write(const uint8_t* data, size_t count, size_t offset) = 0;
+
+	virtual const void* GetPtrForMapping() const = 0;
+};
+
+ARCH_API std::shared_ptr<ArchMemStorage> ArchCreateMemStorageRO(const std::string& path, const void* data, size_t size);
+ARCH_API std::shared_ptr<ArchMemStorage> ArchCreateMemStorageRW(const std::string& path);
 
 ///@}
 

third_party/usd/src/pxr/base/lib/arch/stackTrace.cpp

@@ -830,8 +830,8 @@ _FinishLoggingFatalStackTrace(const char *progname, const char *stackTrace,
 {
     if (!crashingHard && sessionLog) {
         // If we were given a session log, cat it to the end of the stack.
-        if (FILE* stackFd = ArchOpenFile(stackTrace, "a")) {
-            if (FILE* sessionLogFd = ArchOpenFile(sessionLog, "r")) {
+        if (FILE* stackFd = fopen(stackTrace, "a")) {
+            if (FILE* sessionLogFd = fopen(sessionLog, "r")) {
                 fputs("\n\n********** Session Log **********\n\n", stackFd);
                 // Cat the sesion log
                 char line[4096];
@@ -907,7 +907,7 @@ ArchLogPostMortem(const char* reason,
     }
 
     // Write reason for stack trace to logfile.
-    if (FILE* stackFd = ArchOpenFile(logfile, "a")) {
+    if (FILE* stackFd = fopen(logfile, "a")) {
         if (reason) {
             fputs("This stack trace was requested because: ", stackFd);
             fputs(reason, stackFd);

third_party/usd/src/pxr/base/lib/tf/envSetting.cpp

@@ -57,7 +57,7 @@ class Tf_EnvSettingRegistry {
 
     Tf_EnvSettingRegistry() {
         string fileName = TfGetenv("PIXAR_TF_ENV_SETTING_FILE", "");
-        if (FILE* fp = ArchOpenFile(fileName.c_str(), "r")) {
+        if (FILE* fp = fopen(fileName.c_str(), "r")) {
             char buffer[1024];
 
 #ifdef PXR_PYTHON_SUPPORT_ENABLED

third_party/usd/src/pxr/base/lib/tf/pathUtils.cpp

@@ -429,7 +429,7 @@ TfNormPath(string const &inPath)
 string
 TfAbsPath(string const& path)
 {
-    if (path.empty()) {
+    if (path.empty() || ArchIsMemoryPath(path.c_str())) {
         return path;
     }
 

third_party/usd/src/pxr/base/lib/tf/pyInterpreter.cpp

@@ -169,7 +169,7 @@ boost::python::handle<>
 TfPyRunFile(const std::string &filename, int start,
             object const &globals, object const &locals)
 {
-    FILE *f = ArchOpenFile(filename.c_str(), "r");
+    FILE *f = fopen(filename.c_str(), "r");
     if (!f) {
         TF_CODING_ERROR("Could not open file '%s'!", filename.c_str());
         return handle<>();

third_party/usd/src/pxr/base/lib/tf/safeOutputFile.cpp

@@ -52,15 +52,15 @@ TfSafeOutputFile::IsOpenForUpdate() const
     return _file && _tempFileName.empty();
 }
 
-FILE *
+ArchFile *
 TfSafeOutputFile::ReleaseUpdatedFile()
 {
     if (!IsOpenForUpdate()) {
         TF_CODING_ERROR("Invalid output file (failed to open, or opened for "
                         "replace)");
         return nullptr;
     }
-    FILE *ret = _file;
+	ArchFile *ret = _file;
     _file = nullptr;
     _tempFileName.clear();
     _targetFileName.clear();
@@ -74,7 +74,7 @@ TfSafeOutputFile::Close()
         return;
 
     // Close the file.
-    fclose(_file);
+    ArchReleaseFile(_file);
     _file = nullptr;
 
     // If this was for update, we have nothing else to do.
@@ -95,7 +95,7 @@ TfSafeOutputFile::Update(std::string const &fileName)
 {
     TfSafeOutputFile result;
     result._targetFileName = fileName;
-    FILE *file = ArchOpenFile(fileName.c_str(), "r+");
+    ArchFile *file = ArchOpenFile(fileName.c_str(), "r+");
     if (!file) {
         TF_RUNTIME_ERROR("Unable to open file '%s' for writing",
                          fileName.c_str());
@@ -108,6 +108,10 @@ TfSafeOutputFile::Update(std::string const &fileName)
 TfSafeOutputFile
 TfSafeOutputFile::Replace(std::string const &fileName)
 {
+	if (ArchIsMemoryPath(fileName.c_str())) {
+		return TfSafeOutputFile::Update(fileName);
+	}
+
     TfSafeOutputFile result;
     std::string error;
     int tmpFd = Tf_CreateSiblingTempFile(fileName,
@@ -120,7 +124,7 @@ TfSafeOutputFile::Replace(std::string const &fileName)
     }
 
     // Obtain a FILE *.
-    result._file = ArchFdOpen(tmpFd, "w");
+    result._file = ArchOpenFile(tmpFd, "w");
     if (!result._file) {
         TF_RUNTIME_ERROR("Unable to obtain writable FILE pointer: %s",
                          ArchStrerror(errno).c_str());

third_party/usd/src/pxr/base/lib/tf/safeOutputFile.h

@@ -36,6 +36,8 @@
 
 PXR_NAMESPACE_OPEN_SCOPE
 
+class ArchFile;
+
 /// \class TfSafeOutputFile
 ///
 /// Opens a file for output, either for update "r+" or to completely replace
@@ -83,19 +85,19 @@ class TfSafeOutputFile
     TF_API void Close();
 
     /// Return the opened FILE *.
-    FILE *Get() const { return _file; }
+    ArchFile *Get() const { return _file; }
 
     /// If the underlying file was opened by Update(), return it.  The caller
     /// takes responsibility for closing the file later.  It is an error to call
     /// this for files opened for Replace.
-    TF_API FILE *ReleaseUpdatedFile();
+    TF_API ArchFile *ReleaseUpdatedFile();
 
     /// Return true if this TfSafeOutputFile was created by a call to Update(),
     /// false otherwise.
     TF_API bool IsOpenForUpdate() const;
 
 private:
-    FILE *_file = nullptr;
+    ArchFile *_file = nullptr;
     std::string _targetFileName;
     std::string _tempFileName;
 };

third_party/usd/src/pxr/base/lib/tf/scriptModuleLoader.cpp

@@ -169,7 +169,7 @@ TfScriptModuleLoader::GetModulesDict() const
 void
 TfScriptModuleLoader::WriteDotFile(string const &file) const
 {
-    FILE *out = ArchOpenFile(file.c_str(), "w");
+    FILE *out = fopen(file.c_str(), "w");
     if (!out) {
         TF_RUNTIME_ERROR("Could not open '%s' for writing.\n", file.c_str());
         return;

third_party/usd/src/pxr/usd/lib/sdf/textFileFormat.cpp

@@ -105,7 +105,7 @@ SdfTextFileFormat::CanRead(const string& filePath) const
     TRACE_FUNCTION();
 
     bool canRead = false;
-    if (FILE *f = ArchOpenFile(filePath.c_str(), "rb")) {
+    if (FILE *f = fopen(filePath.c_str(), "rb")) {
         canRead = _CanReadImpl(f);
         fclose(f);
     }
@@ -125,7 +125,7 @@ class Sdf_ScopedFilePointer : boost::noncopyable
 {
 public:
     explicit Sdf_ScopedFilePointer(const string& filePath)
-        : _fp(ArchOpenFile(filePath.c_str(), "rb"))
+        : _fp(fopen(filePath.c_str(), "rb"))
     { }
 
     ~Sdf_ScopedFilePointer() {