diff --git a/source/adios2/common/ADIOSTypes.h b/source/adios2/common/ADIOSTypes.h index 090733e399..fad51e8fd9 100644 --- a/source/adios2/common/ADIOSTypes.h +++ b/source/adios2/common/ADIOSTypes.h @@ -21,7 +21,6 @@ #include #include #include -#include #include #include #include //std::pair @@ -385,48 +384,6 @@ struct UserOptions SST sst; }; -/** Host access protocols */ -enum class HostAccessProtocol -{ - Invalid, - SSH, - XRootD, - S3 -}; - -/** Host authentication protocols */ -enum class HostAuthProtocol -{ - Invalid, - Password, - X509 -}; - -struct HostConfig -{ - std::string name; // Connection Option name - HostAccessProtocol protocol = HostAccessProtocol::Invalid; - - /* ssh and xrootd parameters */ - uint16_t port = 0; - uint16_t localPort = 0; - std::string hostname = ""; - std::string username = ""; - std::string remoteServerPath = ""; - HostAuthProtocol authentication = HostAuthProtocol::Invalid; - - /* s3 parameters */ - std::string endpoint = ""; - std::string awsProfile = "default"; // profile name in ~/.aws/credentials - bool isAWS_EC2 = false; - bool recheckMetadata = true; - - int verbose = 0; -}; - -/** HostOptions holds all user options from ~/.config/adios2/hosts.yaml */ -using HostOptions = std::map>; - /** * os << [adios2_type] enables output of adios2 enums/classes directly * to output streams (e.g. std::cout), if ToString() can handle [adios2_type]. diff --git a/source/adios2/core/ADIOS.cpp b/source/adios2/core/ADIOS.cpp index bc113dba62..f7d248d2ee 100644 --- a/source/adios2/core/ADIOS.cpp +++ b/source/adios2/core/ADIOS.cpp @@ -110,9 +110,6 @@ static std::atomic_uint adios_count(0); // total adios objects during runtime /** User defined options from ~/.config/adios2/adios2.yaml if it exists */ const adios2::UserOptions &ADIOS::GetUserOptions() { return m_UserOptions; }; -/** A constant reference to the host options from ~/.config/adios2/hosts.yaml */ -const adios2::HostOptions &ADIOS::GetHostOptions() { return m_HostOptions; }; - ADIOS::ADIOS(const std::string configFile, helper::Comm comm, const std::string hostLanguage) : m_HostLanguage(hostLanguage), m_Comm(std::move(comm)), m_ConfigFile(configFile), m_CampaignManager(m_Comm) @@ -132,7 +129,6 @@ ADIOS::ADIOS(const std::string configFile, helper::Comm comm, const std::string } #endif ProcessUserConfig(); - ProcessHostConfig(); if (!configFile.empty()) { if (!adios2sys::SystemTools::FileExists(configFile)) @@ -215,22 +211,6 @@ void ADIOS::ProcessUserConfig() } } -void ADIOS::ProcessHostConfig() -{ - // read config parameters from config file - std::string homePath; -#ifdef _WIN32 - homePath = getenv("HOMEPATH"); -#else - homePath = getenv("HOME"); -#endif - const std::string cfgFile = homePath + "/.config/adios2/hosts.yaml"; - if (adios2sys::SystemTools::FileExists(cfgFile)) - { - helper::ParseHostOptionsFile(m_Comm, cfgFile, m_HostOptions, homePath); - } -} - IO &ADIOS::DeclareIO(const std::string name, const ArrayOrdering ArrayOrder) { auto itIO = m_IOs.find(name); diff --git a/source/adios2/core/ADIOS.h b/source/adios2/core/ADIOS.h index 1dcdc9b58a..00bff3f89c 100644 --- a/source/adios2/core/ADIOS.h +++ b/source/adios2/core/ADIOS.h @@ -167,9 +167,6 @@ class ADIOS /** A constant reference to the user options from ~/.config/adios2/adios2.yaml */ const adios2::UserOptions &GetUserOptions(); - /** A constant reference to the host options from ~/.config/adios2/hosts.yaml */ - const adios2::HostOptions &GetHostOptions(); - private: /** Communicator given to parallel constructor. */ helper::Comm m_Comm; @@ -211,11 +208,8 @@ class ADIOS core::IO &io); adios2::UserOptions m_UserOptions; - adios2::HostOptions m_HostOptions; - adios2::HostConfig m_Test; void SetUserOptionDefaults(); void ProcessUserConfig(); - void ProcessHostConfig(); private: /* Global services that we want to initialize at most once and shutdown diff --git a/source/adios2/core/Engine.cpp b/source/adios2/core/Engine.cpp index 3f8c41b6f3..72ab54d4ac 100644 --- a/source/adios2/core/Engine.cpp +++ b/source/adios2/core/Engine.cpp @@ -24,7 +24,7 @@ namespace core Engine::Engine(const std::string engineType, IO &io, const std::string &name, const Mode openMode, helper::Comm comm) : m_EngineType(engineType), m_IO(io), m_Name(name), m_OpenMode(openMode), m_Comm(std::move(comm)), - m_UserOptions(io.m_ADIOS.GetUserOptions()), m_HostOptions(io.m_ADIOS.GetHostOptions()) + m_UserOptions(io.m_ADIOS.GetUserOptions()) { m_FailVerbose = (m_Comm.Rank() == 0); } diff --git a/source/adios2/core/Engine.h b/source/adios2/core/Engine.h index 358d551ccf..0ab6ed4a00 100644 --- a/source/adios2/core/Engine.h +++ b/source/adios2/core/Engine.h @@ -508,9 +508,6 @@ class Engine /** User options parsed by the ADIOS object, here just for easy reference */ const UserOptions &m_UserOptions; - /** Host options parsed by the ADIOS object, here just for easy reference */ - const HostOptions &m_HostOptions; - /** keeps track of current advance status */ StepStatus m_AdvanceStatus = StepStatus::OK; diff --git a/source/adios2/engine/campaign/CampaignReader.cpp b/source/adios2/engine/campaign/CampaignReader.cpp index 11bdc9ce90..dbf6ea3d43 100644 --- a/source/adios2/engine/campaign/CampaignReader.cpp +++ b/source/adios2/engine/campaign/CampaignReader.cpp @@ -229,75 +229,27 @@ void CampaignReader::InitTransports() std::string localPath; if (m_CampaignData.hosts[ds.hostIdx].hostname != m_Options.hostname) { - bool done = false; - auto it = m_HostOptions.find(m_CampaignData.hosts[ds.hostIdx].hostname); - if (it != m_HostOptions.end()) + const std::string remotePath = + m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + PathSeparator + ds.name; + const std::string remoteURL = + m_CampaignData.hosts[ds.hostIdx].hostname + ":" + remotePath; + localPath = m_Options.cachepath + PathSeparator + + m_CampaignData.hosts[ds.hostIdx].hostname + PathSeparator + m_Name + + PathSeparator + ds.name; + if (m_Options.verbose > 0) { - const HostConfig &ho = (it->second).front(); - if (ho.protocol == HostAccessProtocol::S3) - { - const std::string endpointURL = ho.endpoint; - const std::string objPath = - m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + "/" + ds.name; - Params p; - p.emplace("Library", "awssdk"); - p.emplace("endpoint", endpointURL); - p.emplace("cache", m_Options.cachepath + PathSeparator + - m_CampaignData.hosts[ds.hostIdx].hostname + - PathSeparator + m_Name); - p.emplace("verbose", std::to_string(ho.verbose)); - p.emplace("recheck_metadata", (ho.recheckMetadata ? "true" : "false")); - io.AddTransport("File", p); - io.SetEngine("BP5"); - localPath = m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + - PathSeparator + ds.name; - if (ho.isAWS_EC2) - { - adios2sys::SystemTools::PutEnv("AWS_EC2_METADATA_DISABLED=false"); - } - else - { - adios2sys::SystemTools::PutEnv("AWS_EC2_METADATA_DISABLED=true"); - } - - if (ho.awsProfile.empty()) - { - adios2sys::SystemTools::PutEnv("AWS_PROFILE=default"); - } - else - { - std::string es = "AWS_PROFILE=" + ho.awsProfile; - adios2sys::SystemTools::PutEnv(es); - } - - done = true; - } + std::cout << "Open remote file " << remoteURL + << "\n and use local cache for metadata at " << localPath << " \n"; } - - if (!done) + helper::CreateDirectory(localPath); + for (auto &bpf : ds.files) { - const std::string remotePath = - m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + PathSeparator + ds.name; - const std::string remoteURL = - m_CampaignData.hosts[ds.hostIdx].hostname + ":" + remotePath; - localPath = m_Options.cachepath + PathSeparator + - m_CampaignData.hosts[ds.hostIdx].hostname + PathSeparator + m_Name + - PathSeparator + ds.name; - if (m_Options.verbose > 0) - { - std::cout << "Open remote file " << remoteURL - << "\n and use local cache for metadata at " << localPath << " \n"; - } - helper::CreateDirectory(localPath); - for (auto &bpf : ds.files) - { - /*std::cout << " save file " << remoteURL << "/" << - bpf.name - << " to " << localPath << "/" << bpf.name << "\n";*/ - SaveToFile(m_DB, localPath + PathSeparator + bpf.name, bpf); - } - io.SetParameter("RemoteDataPath", remotePath); + /*std::cout << " save file " << remoteURL << "/" << + bpf.name + << " to " << localPath << "/" << bpf.name << "\n";*/ + SaveToFile(m_DB, localPath + PathSeparator + bpf.name, bpf); } + io.SetParameter("RemoteDataPath", remotePath); } else { diff --git a/source/adios2/helper/adiosYAML.cpp b/source/adios2/helper/adiosYAML.cpp index 68937b47b0..07f93697e9 100644 --- a/source/adios2/helper/adiosYAML.cpp +++ b/source/adios2/helper/adiosYAML.cpp @@ -94,9 +94,9 @@ inline void FixHomePath(std::string &path, std::string &homePath) template void SetOption(T &value, const std::string nodeName, const YAML::Node &upperNode, - const std::string &hint, bool mandatory = isNotMandatory) + const std::string &hint) { - auto node = YAMLNode(nodeName, upperNode, hint, mandatory, YAML::NodeType::Scalar); + auto node = YAMLNode(nodeName, upperNode, hint, isNotMandatory, YAML::NodeType::Scalar); if (node) { value = node.as(); @@ -272,7 +272,7 @@ void ParseUserOptionsFile(Comm &comm, const std::string &configFileYAML, UserOpt helper::Throw( "Helper", "adiosUserOptions", "ParseUserOptionsFile", "parser error in file " + configFileYAML + - ": invalid format. Check with any YAML editor if format is ill-formed, " + hint); + " invalid format. Check with any YAML editor if format is ill-formed, " + hint); } /* @@ -315,137 +315,5 @@ void ParseUserOptionsFile(Comm &comm, const std::string &configFileYAML, UserOpt } } -HostAccessProtocol GetHostAccessProtocol(std::string valueStr) -{ - std::transform(valueStr.begin(), valueStr.end(), valueStr.begin(), ::tolower); - if (valueStr == "ssh") - { - return HostAccessProtocol::SSH; - } - else if (valueStr == "s3") - { - return HostAccessProtocol::S3; - } - else if (valueStr == "xrootd") - { - return HostAccessProtocol::XRootD; - } - return HostAccessProtocol::Invalid; -} - -HostAuthProtocol GetHostAuthProtocol(std::string valueStr) -{ - std::transform(valueStr.begin(), valueStr.end(), valueStr.begin(), ::tolower); - if (valueStr == "password") - { - return HostAuthProtocol::Password; - } - else if (valueStr == "x509" || valueStr == "x.509") - { - return HostAuthProtocol::X509; - } - return HostAuthProtocol::Invalid; -} - -void ParseHostOptionsFile(Comm &comm, const std::string &configFileYAML, HostOptions &hosts, - std::string &homePath) -{ - const std::string hint = - "when parsing host config file " + configFileYAML + " in call to ADIOS constructor"; - - const std::string configFileContents = comm.BroadcastFile(configFileYAML, hint); - - const YAML::Node document = YAML::Load(configFileContents); - if (!document) - { - helper::Throw( - "Helper", "adiosHostOptions", "ParseHostOptionsFile", - "parser error in file " + configFileYAML + - ": invalid format. Check with any YAML editor if format is ill-formed, " + hint); - } - - if (!document.IsMap()) - { - helper::Throw("Helper", "adiosHostOptions", "ParseHostOptionsFile", - "parser error: not a YAML Map of hosts, " + hint); - } - - /* top level is a dictionary of */ - for (auto itDoc = document.begin(); itDoc != document.end(); ++itDoc) - { - std::string hostname = itDoc->first.as(); - - /* a dictionary of host options, with each entry a dictionary */ - YAML::Node hostentry = itDoc->second; - if (!hostentry.IsMap()) - { - helper::Throw( - "Helper", "adiosHostOptions", "ParseHostOptionsFile", - "parser error for host " + hostname + - ": each host must have a YAML Map of options, " + hint); - } - - std::vector hostConfigs; - for (auto itHost = hostentry.begin(); itHost != hostentry.end(); ++itHost) - { - /* one connection setup as a dictionary */ - HostConfig hc; - hc.name = itHost->first.as(); - const YAML::Node &hostmap = itHost->second; - if (!hostmap.IsMap()) - { - helper::Throw( - "Helper", "adiosHostOptions", "ParseHostOptionsFile", - "parser error for host " + hostname + - ": each entry in the list must be a YAML Map, " + hint); - } - - SetOption(hc.verbose, "verbose", hostmap, hint); - std::string protocolStr; - SetOption(protocolStr, "protocol", hostmap, hint, isMandatory); - hc.protocol = GetHostAccessProtocol(protocolStr); - switch (hc.protocol) - { - case HostAccessProtocol::SSH: { - std::string authStr; - SetOption(authStr, "authentication", hostmap, hint, isMandatory); - hc.authentication = GetHostAuthProtocol(authStr); - SetOption(hc.hostname, "host", hostmap, hint, isMandatory); - SetOption(hc.username, "user", hostmap, hint); - SetOption(hc.remoteServerPath, "serverpath", hostmap, hint, isMandatory); - SetOption(hc.port, "port", hostmap, hint); - SetOption(hc.localPort, "local_port", hostmap, hint); - break; - } - case HostAccessProtocol::XRootD: { - std::string authStr; - SetOption(authStr, "authentication", hostmap, hint, isMandatory); - hc.authentication = GetHostAuthProtocol(authStr); - SetOption(hc.hostname, "host", hostmap, hint, isMandatory); - SetOption(hc.username, "user", hostmap, hint); - SetOption(hc.remoteServerPath, "serverpath", hostmap, hint); - SetOption(hc.port, "port", hostmap, hint); - break; - } - case HostAccessProtocol::S3: { - SetOption(hc.awsProfile, "profile", hostmap, hint); - SetOption(hc.endpoint, "endpoint", hostmap, hint, isMandatory); - SetOption(hc.isAWS_EC2, "aws_ec2_metadata", hostmap, hint); - SetOption(hc.recheckMetadata, "recheck_metadata", hostmap, hint); - break; - } - default: - helper::Throw( - "Helper", "adiosHostOptions", "ParseHostOptionsFile", - "parser error: invalid access host protocol '" + protocolStr + " for " + - hc.name + "', " + hint); - } - - hostConfigs.push_back(hc); - } - hosts.emplace(hostname, hostConfigs); - } -} - } // end namespace helper } // end namespace adios2 diff --git a/source/adios2/helper/adiosYAML.h b/source/adios2/helper/adiosYAML.h index dd953e4d16..8bb9df19d5 100644 --- a/source/adios2/helper/adiosYAML.h +++ b/source/adios2/helper/adiosYAML.h @@ -35,9 +35,6 @@ std::string ParseConfigYAML(core::ADIOS &adios, const std::string &configFileYAM void ParseUserOptionsFile(Comm &comm, const std::string &configFileYAML, UserOptions &options, std::string &homePath); -void ParseHostOptionsFile(Comm &comm, const std::string &configFileYAML, HostOptions &hosts, - std::string &homePath); - } // end namespace helper } // end namespace adios2 diff --git a/source/adios2/toolkit/transport/file/FileAWSSDK.cpp b/source/adios2/toolkit/transport/file/FileAWSSDK.cpp index 7906a1a330..1e84738b83 100644 --- a/source/adios2/toolkit/transport/file/FileAWSSDK.cpp +++ b/source/adios2/toolkit/transport/file/FileAWSSDK.cpp @@ -69,12 +69,6 @@ void FileAWSSDK::SetParameters(const Params ¶ms) } } - helper::SetParameterValueInt("verbose", params, m_Verbose, ""); - - std::string recheckStr = "true"; - helper::SetParameterValue("recheck_metadata", params, recheckStr); - m_RecheckMetadata = helper::StringTo(recheckStr, ""); - core::ADIOS::Global_init_AWS_API(); s3ClientConfig = new Aws::S3::S3ClientConfiguration; @@ -83,12 +77,7 @@ void FileAWSSDK::SetParameters(const Params ¶ms) s3ClientConfig->enableEndpointDiscovery = false; s3Client = new Aws::S3::S3Client(*s3ClientConfig); - if (m_Verbose > 0) - { - std::cout << "FileAWSSDK::SetParameters: AWS Transport created with endpoint = '" - << m_Endpoint << "'" - << " recheck_metadata = " << m_RecheckMetadata << std::endl; - } + std::cout << "AWS Transport created with endpoint = '" << m_Endpoint << "'" << std::endl; } void FileAWSSDK::WaitForOpen() @@ -119,58 +108,28 @@ void FileAWSSDK::SetUpCache() if (m_CachingThisFile) { std::string const ep = std::regex_replace(m_Endpoint, std::regex("/|:"), "_"); + + m_CacheFileWrite = new FileFStream(m_Comm); const std::string path(m_CachePath + PathSeparator + ep + PathSeparator + m_BucketName + PathSeparator + m_ObjectName); m_CacheFilePath = path; - if (!m_RecheckMetadata) + const auto lastPathSeparator(path.find_last_of(PathSeparator)); + if (lastPathSeparator != std::string::npos) { - m_CacheFileRead = new FileFStream(m_Comm); - try - { - m_CacheFileRead->Open(path, Mode::Read); - m_Size = m_CacheFileRead->GetSize(); - m_IsCached = true; - m_CachingThisFile = false; - if (m_Verbose > 0) - { - std::cout << "FileAWSSDK::SetUpCache: Already cached " << path - << ", size = " << m_Size << std::endl; - } - } - catch (std::ios_base::failure &) - { - delete m_CacheFileRead; - m_IsCached = false; - } + const std::string dirpath(path.substr(0, lastPathSeparator)); + helper::CreateDirectory(dirpath); } - } -} -void FileAWSSDK::CheckCache(const size_t fileSize) -{ - if (m_CachingThisFile) - { - /* Check if cache file exists and size equals the cloud version*/ m_CacheFileRead = new FileFStream(m_Comm); try { - m_CacheFileRead->Open(m_CacheFilePath, Mode::Read); - size_t cacheSize = m_CacheFileRead->GetSize(); - if (cacheSize == fileSize) + m_CacheFileRead->Open(path, Mode::Read); + if (m_CacheFileRead->GetSize() == m_Size) { m_IsCached = true; m_CachingThisFile = false; - if (m_Verbose > 0) - { - std::cout << "FileAWSSDK::CheckCache: Already cached " << m_CacheFilePath - << ", full size = " << cacheSize << std::endl; - } - } - else - { - std::cout << "FileAWSSDK::CheckCache: Partially cached " << m_CacheFilePath - << ", cached size = " << cacheSize << " full size = " << fileSize - << std::endl; + delete m_CacheFileWrite; + std::cout << "Already cached " << path << std::endl; } } catch (std::ios_base::failure &) @@ -180,20 +139,8 @@ void FileAWSSDK::CheckCache(const size_t fileSize) if (m_CachingThisFile) { - /* Create output file for caching this data later */ - const auto lastPathSeparator(m_CacheFilePath.find_last_of(PathSeparator)); - if (lastPathSeparator != std::string::npos) - { - const std::string dirpath(m_CacheFilePath.substr(0, lastPathSeparator)); - helper::CreateDirectory(dirpath); - } - m_CacheFileWrite = new FileFStream(m_Comm); - m_CacheFileWrite->Open(m_CacheFilePath, Mode::Write); - if (m_Verbose > 0) - { - std::cout << "FileAWSSDK::CheckCache: Caching turn on for " << m_CacheFilePath - << std::endl; - } + m_CacheFileWrite->Open(path, Mode::Write); + std::cout << "Caching turn on for " << path << std::endl; } } } @@ -225,42 +172,28 @@ void FileAWSSDK::Open(const std::string &name, const Mode openMode, const bool a case Mode::Read: { ProfilerStart("open"); errno = 0; + Aws::S3::Model::HeadObjectRequest head_object_request; + head_object_request.SetBucket(m_BucketName); + head_object_request.SetKey(m_ObjectName); - SetUpCache(); - // m_IsCached=true if already found in cache and m_RecheckMetadata=false - // m_CachingThisFile=true if we want caching and m_IsCached=false - // m_CacheFilePath is the path to the local file in cache + std::cout << "S3 HeadObjectRequests bucket='" << head_object_request.GetBucket() + << "' object = '" << head_object_request.GetKey() << "'" << std::endl; - if (!m_IsCached) + head_object = s3Client->HeadObject(head_object_request); + if (!head_object.IsSuccess()) { - Aws::S3::Model::HeadObjectRequest head_object_request; - head_object_request.SetBucket(m_BucketName); - head_object_request.SetKey(m_ObjectName); - - if (m_Verbose > 0) - { - std::cout << "FileAWSSDK::Open: S3 HeadObjectRequests bucket='" - << head_object_request.GetBucket() << "' object = '" - << head_object_request.GetKey() << "'" << std::endl; - } - head_object = s3Client->HeadObject(head_object_request); - if (!head_object.IsSuccess()) - { - helper::Throw( - "Toolkit", "transport::file::FileAWSSDK", "Open", - "'bucket/object' " + m_Name + " does not exist "); - } - else - { - m_Size = head_object.GetResult().GetContentLength(); - /* Cache: check if we want to cache this file (metadata files) - and if we already have it fully in the cache - */ - CheckCache(m_Size); - } + helper::Throw("Toolkit", "transport::file::FileAWSSDK", "Open", + "'bucket/object' " + m_Name + " does not exist "); + } + else + { + m_Size = head_object.GetResult().GetContentLength(); - m_Errno = errno; + /* Cache: check if we want to cache this file (metadata files) */ + SetUpCache(); } + + m_Errno = errno; ProfilerStop("open"); break; } @@ -329,11 +262,8 @@ void FileAWSSDK::Read(char *buffer, size_t size, size_t start) if (m_IsCached) { m_CacheFileRead->Read(buffer, size, m_SeekPos); - if (m_Verbose > 0) - { - std::cout << "FileAWSSDK::Read: Read from cache " << m_CacheFileRead->m_Name - << " start = " << m_SeekPos << " size = " << size << std::endl; - } + std::cout << "Read from cache " << m_CacheFileRead->m_Name << " start = " << m_SeekPos + << " size = " << size << std::endl; return; } @@ -356,13 +286,10 @@ void FileAWSSDK::Read(char *buffer, size_t size, size_t start) } else { - if (m_Verbose > 0) - { - std::cout << "FileAWSSDK::Read: Successfully retrieved '" << m_ObjectName << "' from '" - << m_BucketName << "'." - << "\nObject length = " << outcome.GetResult().GetContentLength() - << "\nRange requested = " << range.str() << std::endl; - } + std::cout << "Successfully retrieved '" << m_ObjectName << "' from '" << m_BucketName + << "'." + << "\nObject length = " << outcome.GetResult().GetContentLength() + << "\nRange requested = " << range.str() << std::endl; auto body = outcome.GetResult().GetBody().rdbuf(); body->sgetn(buffer, size); @@ -370,12 +297,8 @@ void FileAWSSDK::Read(char *buffer, size_t size, size_t start) if (m_CachingThisFile) { m_CacheFileWrite->Write(buffer, size, m_SeekPos); - m_CacheFileWrite->Flush(); - if (m_Verbose > 0) - { - std::cout << "FileAWSSDK::Read: Written to cache " << m_CacheFileWrite->m_Name - << " start = " << m_SeekPos << " size = " << size << std::endl; - } + std::cout << "Written to cache " << m_CacheFileWrite->m_Name << " start = " << m_SeekPos + << " size = " << size << std::endl; } } } @@ -402,10 +325,7 @@ void FileAWSSDK::Flush() {} void FileAWSSDK::Close() { WaitForOpen(); - if (m_Verbose > 0) - { - std::cout << "FileAWSSDK::Close(" << m_Name << ") Enter" << std::endl; - } + std::cout << "FileAWSSDK::Close(" << m_Name << ") Enter" << std::endl; ProfilerStart("close"); errno = 0; m_Errno = errno; @@ -422,12 +342,10 @@ void FileAWSSDK::Close() if (m_CachingThisFile) { m_CacheFileWrite->Close(); - delete m_CacheFileWrite; } if (m_IsCached) { m_CacheFileRead->Close(); - delete m_CacheFileRead; } m_IsOpen = false; @@ -446,7 +364,7 @@ void FileAWSSDK::Delete() void FileAWSSDK::CheckFile(const std::string hint) const { - if (!m_IsCached && !head_object.IsSuccess()) + if (!head_object.IsSuccess()) { helper::Throw("Toolkit", "transport::file::FileAWSSDK", "CheckFile", hint); diff --git a/source/adios2/toolkit/transport/file/FileAWSSDK.h b/source/adios2/toolkit/transport/file/FileAWSSDK.h index 6ae0e43df4..e55e04d2f1 100644 --- a/source/adios2/toolkit/transport/file/FileAWSSDK.h +++ b/source/adios2/toolkit/transport/file/FileAWSSDK.h @@ -95,11 +95,8 @@ class FileAWSSDK : public Transport std::future m_OpenFuture; size_t m_SeekPos = 0; size_t m_Size = 0; - int m_Verbose = 0; - bool m_RecheckMetadata = true; // always check if cached metadata is complete void SetUpCache(); - void CheckCache(const size_t fileSize); std::string m_CachePath; // local cache directory bool m_CachingThisFile = false; // save content to local cache FileFStream *m_CacheFileWrite; diff --git a/source/utils/adios_campaign_manager/adios2_campaign_manager.py b/source/utils/adios_campaign_manager/adios2_campaign_manager.py index 18f14583d4..818cd2ca3e 100755 --- a/source/utils/adios_campaign_manager/adios2_campaign_manager.py +++ b/source/utils/adios_campaign_manager/adios2_campaign_manager.py @@ -7,7 +7,6 @@ import yaml from dataclasses import dataclass from datetime import datetime -from dateutil.parser import parse from os import chdir, getcwd, remove, stat from os.path import exists, isdir, expanduser from re import sub @@ -18,7 +17,6 @@ ADIOS_ACA_VERSION = "0.1" - @dataclass class UserOption: adios_campaign_store: str = None @@ -47,17 +45,6 @@ def ReadUserConfig(): return opts -def ReadHostConfig() -> dict: - path = expanduser("~/.config/adios2/hosts.yaml") - doc = {} - try: - with open(path) as f: - doc = yaml.safe_load(f) - except FileNotFoundError: - None - return doc - - def SetupArgs(): parser = argparse.ArgumentParser() parser.add_argument( @@ -73,19 +60,11 @@ def SetupArgs(): "--campaign_store", "-s", help="Path to local campaign store", default=None ) parser.add_argument("--hostname", "-n", help="Host name unique for hosts in a campaign") - parser.add_argument("--s3_bucket", "-b", help="Bucket on S3 server", default=None) - parser.add_argument( - "--s3_datetime", - "-t", - help="Datetime of data on S3 server in " "'2024-04-19 10:20:15 -0400' format", - default=None, - ) parser.add_argument("-f", "--files", nargs="+", help="Add ADIOS files manually") args = parser.parse_args() # default values args.user_options = ReadUserConfig() - args.host_options = ReadHostConfig() if args.verbose == 0: args.verbose = args.user_options.verbose @@ -97,33 +76,16 @@ def SetupArgs(): while args.campaign_store[-1] == "/": args.campaign_store = args.campaign_store[:-1] - args.remote_data = False - args.s3_endpoint = None if args.hostname is None: args.hostname = args.user_options.hostname - elif args.hostname in args.host_options and args.hostname != args.user_options.hostname: - args.remote_data = True - hostopt = args.host_options.get(args.hostname) - if hostopt is not None: - optID = next(iter(hostopt)) - if hostopt[optID]["protocol"].casefold() == "s3": - args.s3_endpoint = hostopt[optID]["endpoint"] - if args.s3_bucket is None: - print("ERROR: Remote option for an S3 server requires --s3_bucket") - exit(1) - if args.s3_datetime is None: - print("ERROR: Remote option for an S3 server requires --s3_datetime") - exit(1) args.CampaignFileName = args.campaign if args.campaign is not None: if not args.campaign.endswith(".aca"): args.CampaignFileName += ".aca" - if ( - not exists(args.CampaignFileName) and - not args.CampaignFileName.startswith("/") and - args.campaign_store is not None - ): + if (not exists(args.CampaignFileName) and + not args.CampaignFileName.startswith("/") and + args.campaign_store is not None): args.CampaignFileName = args.campaign_store + "/" + args.CampaignFileName if args.files is None: @@ -154,13 +116,6 @@ def CheckLocalCampaignDir(args): exit(1) -def parse_date_to_utc(date, fmt=None): - if fmt is None: - fmt = "%Y-%m-%d %H:%M:%S %z" # Defaults to : 2022-08-31 07:47:30 -0000 - get_date_obj = parse(str(date)) - return get_date_obj.timestamp() - - def IsADIOSDataset(dataset): if not isdir(dataset): return False @@ -232,9 +187,9 @@ def AddFileToArchive(args: dict, filename: str, cur: sqlite3.Cursor, dsID: int): ) -def AddDatasetToArchive( - args: dict, hostID: int, dirID: int, dataset: str, cur: sqlite3.Cursor -) -> int: +def AddDatasetToArchive(hostID: int, dirID: int, dataset: str, cur: sqlite3.Cursor) -> int: + statres = stat(dataset) + ct = statres.st_ctime_ns select_cmd = ( "select rowid from bpdataset " f"where hostid = {hostID} and dirid = {dirID} and name = '{dataset}'" @@ -249,15 +204,6 @@ def AddDatasetToArchive( ) else: print(f"Add dataset {dataset} to archive") - if args.remote_data: - if args.s3_datetime: - ct = parse_date_to_utc(args.s3_datetime) - else: - ct = 0 - else: - statres = stat(dataset) - ct = statres.st_ctime_ns - curDS = cur.execute( "insert into bpdataset (hostid, dirid, name, ctime) values (?, ?, ?, ?)", (hostID, dirID, dataset, ct), @@ -275,10 +221,8 @@ def ProcessFiles(args: dict, cur: sqlite3.Cursor, hostID: int, dirID: int): print(f"Process entry {entry}:") dsID = 0 dataset = entry - if args.remote_data: - dsID = AddDatasetToArchive(args, hostID, dirID, dataset, cur) - elif IsADIOSDataset(dataset): - dsID = AddDatasetToArchive(args, hostID, dirID, dataset, cur) + if IsADIOSDataset(dataset): + dsID = AddDatasetToArchive(hostID, dirID, dataset, cur) cwd = getcwd() chdir(dataset) mdFileList = glob.glob("*md.*") @@ -292,20 +236,16 @@ def ProcessFiles(args: dict, cur: sqlite3.Cursor, hostID: int, dirID: int): def GetHostName(): - if args.s3_endpoint: - longhost = args.s3_endpoint - else: - longhost = getfqdn() - if longhost.startswith("login"): - longhost = sub("^login[0-9]*\\.", "", longhost) - if longhost.startswith("batch"): - longhost = sub("^batch[0-9]*\\.", "", longhost) - + host = getfqdn() + if host.startswith("login"): + host = sub("^login[0-9]*\\.", "", host) + if host.startswith("batch"): + host = sub("^batch[0-9]*\\.", "", host) if args.hostname is None: - shorthost = longhost.split(".")[0] + shorthost = host.split(".")[0] else: - shorthost = args.hostname - return longhost, shorthost + shorthost = args.user_options.hostname + return host, shorthost def AddHostName(longHostName, shortHostName): @@ -362,10 +302,7 @@ def Update(args: dict, cur: sqlite3.Cursor): hostID = AddHostName(longHostName, shortHostName) - if args.remote_data and args.s3_bucket is not None: - rootdir = args.s3_bucket - else: - rootdir = getcwd() + rootdir = getcwd() dirID = AddDirectory(hostID, rootdir) con.commit()