Skip to content

Commit

Permalink
Revert "Merge pull request ornladios#4189 from pnorbert/campaign-s3-s…
Browse files Browse the repository at this point in the history 
…upport"

This reverts commit 04a68d0, reversing
changes made to f626281.
  • Loading branch information
@vicentebolea
vicentebolea committed Oct 24, 2024
1 parent f20747e commit b940238
Show file tree
Hide file tree
Showing 11 changed files with 78 additions and 481 deletions.
43 changes: 0 additions & 43 deletions source/adios2/common/ADIOSTypes.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
#include <complex>
#include <limits>
#include <map>
#include <memory>
#include <string>
#include <type_traits>
#include <utility> //std::pair
Expand Down Expand Up @@ -385,48 +384,6 @@ struct UserOptions
SST sst;
};

/** Host access protocols */
enum class HostAccessProtocol
{
Invalid,
SSH,
XRootD,
S3
};

/** Host authentication protocols */
enum class HostAuthProtocol
{
Invalid,
Password,
X509
};

struct HostConfig
{
std::string name; // Connection Option name
HostAccessProtocol protocol = HostAccessProtocol::Invalid;

/* ssh and xrootd parameters */
uint16_t port = 0;
uint16_t localPort = 0;
std::string hostname = "";
std::string username = "";
std::string remoteServerPath = "";
HostAuthProtocol authentication = HostAuthProtocol::Invalid;

/* s3 parameters */
std::string endpoint = "";
std::string awsProfile = "default"; // profile name in ~/.aws/credentials
bool isAWS_EC2 = false;
bool recheckMetadata = true;

int verbose = 0;
};

/** HostOptions holds all user options from ~/.config/adios2/hosts.yaml */
using HostOptions = std::map<std::string, std::vector<HostConfig>>;

/**
* os << [adios2_type] enables output of adios2 enums/classes directly
* to output streams (e.g. std::cout), if ToString() can handle [adios2_type].
Expand Down
20 changes: 0 additions & 20 deletions source/adios2/core/ADIOS.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,9 +110,6 @@ static std::atomic_uint adios_count(0); // total adios objects during runtime
/** User defined options from ~/.config/adios2/adios2.yaml if it exists */
const adios2::UserOptions &ADIOS::GetUserOptions() { return m_UserOptions; };

/** A constant reference to the host options from ~/.config/adios2/hosts.yaml */
const adios2::HostOptions &ADIOS::GetHostOptions() { return m_HostOptions; };

ADIOS::ADIOS(const std::string configFile, helper::Comm comm, const std::string hostLanguage)
: m_HostLanguage(hostLanguage), m_Comm(std::move(comm)), m_ConfigFile(configFile),
m_CampaignManager(m_Comm)
Expand All @@ -132,7 +129,6 @@ ADIOS::ADIOS(const std::string configFile, helper::Comm comm, const std::string
}
#endif
ProcessUserConfig();
ProcessHostConfig();
if (!configFile.empty())
{
if (!adios2sys::SystemTools::FileExists(configFile))
Expand Down Expand Up @@ -215,22 +211,6 @@ void ADIOS::ProcessUserConfig()
}
}

void ADIOS::ProcessHostConfig()
{
// read config parameters from config file
std::string homePath;
#ifdef _WIN32
homePath = getenv("HOMEPATH");
#else
homePath = getenv("HOME");
#endif
const std::string cfgFile = homePath + "/.config/adios2/hosts.yaml";
if (adios2sys::SystemTools::FileExists(cfgFile))
{
helper::ParseHostOptionsFile(m_Comm, cfgFile, m_HostOptions, homePath);
}
}

IO &ADIOS::DeclareIO(const std::string name, const ArrayOrdering ArrayOrder)
{
auto itIO = m_IOs.find(name);
Expand Down
6 changes: 0 additions & 6 deletions source/adios2/core/ADIOS.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,9 +167,6 @@ class ADIOS
/** A constant reference to the user options from ~/.config/adios2/adios2.yaml */
const adios2::UserOptions &GetUserOptions();

/** A constant reference to the host options from ~/.config/adios2/hosts.yaml */
const adios2::HostOptions &GetHostOptions();

private:
/** Communicator given to parallel constructor. */
helper::Comm m_Comm;
Expand Down Expand Up @@ -211,11 +208,8 @@ class ADIOS
core::IO &io);

adios2::UserOptions m_UserOptions;
adios2::HostOptions m_HostOptions;
adios2::HostConfig m_Test;
void SetUserOptionDefaults();
void ProcessUserConfig();
void ProcessHostConfig();

private:
/* Global services that we want to initialize at most once and shutdown
Expand Down
2 changes: 1 addition & 1 deletion source/adios2/core/Engine.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ namespace core
Engine::Engine(const std::string engineType, IO &io, const std::string &name, const Mode openMode,
helper::Comm comm)
: m_EngineType(engineType), m_IO(io), m_Name(name), m_OpenMode(openMode), m_Comm(std::move(comm)),
m_UserOptions(io.m_ADIOS.GetUserOptions()), m_HostOptions(io.m_ADIOS.GetHostOptions())
m_UserOptions(io.m_ADIOS.GetUserOptions())
{
m_FailVerbose = (m_Comm.Rank() == 0);
}
Expand Down
3 changes: 0 additions & 3 deletions source/adios2/core/Engine.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -508,9 +508,6 @@ class Engine
/** User options parsed by the ADIOS object, here just for easy reference */
const UserOptions &m_UserOptions;

/** Host options parsed by the ADIOS object, here just for easy reference */
const HostOptions &m_HostOptions;

/** keeps track of current advance status */
StepStatus m_AdvanceStatus = StepStatus::OK;

Expand Down
82 changes: 17 additions & 65 deletions source/adios2/engine/campaign/CampaignReader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,75 +229,27 @@ void CampaignReader::InitTransports()
std::string localPath;
if (m_CampaignData.hosts[ds.hostIdx].hostname != m_Options.hostname)
{
bool done = false;
auto it = m_HostOptions.find(m_CampaignData.hosts[ds.hostIdx].hostname);
if (it != m_HostOptions.end())
const std::string remotePath =
m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + PathSeparator + ds.name;
const std::string remoteURL =
m_CampaignData.hosts[ds.hostIdx].hostname + ":" + remotePath;
localPath = m_Options.cachepath + PathSeparator +
m_CampaignData.hosts[ds.hostIdx].hostname + PathSeparator + m_Name +
PathSeparator + ds.name;
if (m_Options.verbose > 0)
{
const HostConfig &ho = (it->second).front();
if (ho.protocol == HostAccessProtocol::S3)
{
const std::string endpointURL = ho.endpoint;
const std::string objPath =
m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + "/" + ds.name;
Params p;
p.emplace("Library", "awssdk");
p.emplace("endpoint", endpointURL);
p.emplace("cache", m_Options.cachepath + PathSeparator +
m_CampaignData.hosts[ds.hostIdx].hostname +
PathSeparator + m_Name);
p.emplace("verbose", std::to_string(ho.verbose));
p.emplace("recheck_metadata", (ho.recheckMetadata ? "true" : "false"));
io.AddTransport("File", p);
io.SetEngine("BP5");
localPath = m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] +
PathSeparator + ds.name;
if (ho.isAWS_EC2)
{
adios2sys::SystemTools::PutEnv("AWS_EC2_METADATA_DISABLED=false");
}
else
{
adios2sys::SystemTools::PutEnv("AWS_EC2_METADATA_DISABLED=true");
}

if (ho.awsProfile.empty())
{
adios2sys::SystemTools::PutEnv("AWS_PROFILE=default");
}
else
{
std::string es = "AWS_PROFILE=" + ho.awsProfile;
adios2sys::SystemTools::PutEnv(es);
}

done = true;
}
std::cout << "Open remote file " << remoteURL
<< "\n and use local cache for metadata at " << localPath << " \n";
}

if (!done)
helper::CreateDirectory(localPath);
for (auto &bpf : ds.files)
{
const std::string remotePath =
m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + PathSeparator + ds.name;
const std::string remoteURL =
m_CampaignData.hosts[ds.hostIdx].hostname + ":" + remotePath;
localPath = m_Options.cachepath + PathSeparator +
m_CampaignData.hosts[ds.hostIdx].hostname + PathSeparator + m_Name +
PathSeparator + ds.name;
if (m_Options.verbose > 0)
{
std::cout << "Open remote file " << remoteURL
<< "\n and use local cache for metadata at " << localPath << " \n";
}
helper::CreateDirectory(localPath);
for (auto &bpf : ds.files)
{
/*std::cout << " save file " << remoteURL << "/" <<
bpf.name
<< " to " << localPath << "/" << bpf.name << "\n";*/
SaveToFile(m_DB, localPath + PathSeparator + bpf.name, bpf);
}
io.SetParameter("RemoteDataPath", remotePath);
/*std::cout << " save file " << remoteURL << "/" <<
bpf.name
<< " to " << localPath << "/" << bpf.name << "\n";*/
SaveToFile(m_DB, localPath + PathSeparator + bpf.name, bpf);
}
io.SetParameter("RemoteDataPath", remotePath);
}
else
{
Expand Down
138 changes: 3 additions & 135 deletions source/adios2/helper/adiosYAML.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,9 +94,9 @@ inline void FixHomePath(std::string &path, std::string &homePath)

template <class T>
void SetOption(T &value, const std::string nodeName, const YAML::Node &upperNode,
const std::string &hint, bool mandatory = isNotMandatory)
const std::string &hint)
{
auto node = YAMLNode(nodeName, upperNode, hint, mandatory, YAML::NodeType::Scalar);
auto node = YAMLNode(nodeName, upperNode, hint, isNotMandatory, YAML::NodeType::Scalar);
if (node)
{
value = node.as<T>();
Expand Down Expand Up @@ -272,7 +272,7 @@ void ParseUserOptionsFile(Comm &comm, const std::string &configFileYAML, UserOpt
helper::Throw<std::invalid_argument>(
"Helper", "adiosUserOptions", "ParseUserOptionsFile",
"parser error in file " + configFileYAML +
": invalid format. Check with any YAML editor if format is ill-formed, " + hint);
" invalid format. Check with any YAML editor if format is ill-formed, " + hint);
}

/*
Expand Down Expand Up @@ -315,137 +315,5 @@ void ParseUserOptionsFile(Comm &comm, const std::string &configFileYAML, UserOpt
}
}

HostAccessProtocol GetHostAccessProtocol(std::string valueStr)
{
std::transform(valueStr.begin(), valueStr.end(), valueStr.begin(), ::tolower);
if (valueStr == "ssh")
{
return HostAccessProtocol::SSH;
}
else if (valueStr == "s3")
{
return HostAccessProtocol::S3;
}
else if (valueStr == "xrootd")
{
return HostAccessProtocol::XRootD;
}
return HostAccessProtocol::Invalid;
}

HostAuthProtocol GetHostAuthProtocol(std::string valueStr)
{
std::transform(valueStr.begin(), valueStr.end(), valueStr.begin(), ::tolower);
if (valueStr == "password")
{
return HostAuthProtocol::Password;
}
else if (valueStr == "x509" || valueStr == "x.509")
{
return HostAuthProtocol::X509;
}
return HostAuthProtocol::Invalid;
}

void ParseHostOptionsFile(Comm &comm, const std::string &configFileYAML, HostOptions &hosts,
std::string &homePath)
{
const std::string hint =
"when parsing host config file " + configFileYAML + " in call to ADIOS constructor";

const std::string configFileContents = comm.BroadcastFile(configFileYAML, hint);

const YAML::Node document = YAML::Load(configFileContents);
if (!document)
{
helper::Throw<std::invalid_argument>(
"Helper", "adiosHostOptions", "ParseHostOptionsFile",
"parser error in file " + configFileYAML +
": invalid format. Check with any YAML editor if format is ill-formed, " + hint);
}

if (!document.IsMap())
{
helper::Throw<std::invalid_argument>("Helper", "adiosHostOptions", "ParseHostOptionsFile",
"parser error: not a YAML Map of hosts, " + hint);
}

/* top level is a dictionary of <hostname, dictionary of options> */
for (auto itDoc = document.begin(); itDoc != document.end(); ++itDoc)
{
std::string hostname = itDoc->first.as<std::string>();

/* a dictionary of host options, with each entry a dictionary */
YAML::Node hostentry = itDoc->second;
if (!hostentry.IsMap())
{
helper::Throw<std::invalid_argument>(
"Helper", "adiosHostOptions", "ParseHostOptionsFile",
"parser error for host " + hostname +
": each host must have a YAML Map of options, " + hint);
}

std::vector<HostConfig> hostConfigs;
for (auto itHost = hostentry.begin(); itHost != hostentry.end(); ++itHost)
{
/* one connection setup as a dictionary */
HostConfig hc;
hc.name = itHost->first.as<std::string>();
const YAML::Node &hostmap = itHost->second;
if (!hostmap.IsMap())
{
helper::Throw<std::invalid_argument>(
"Helper", "adiosHostOptions", "ParseHostOptionsFile",
"parser error for host " + hostname +
": each entry in the list must be a YAML Map, " + hint);
}

SetOption(hc.verbose, "verbose", hostmap, hint);
std::string protocolStr;
SetOption(protocolStr, "protocol", hostmap, hint, isMandatory);
hc.protocol = GetHostAccessProtocol(protocolStr);
switch (hc.protocol)
{
case HostAccessProtocol::SSH: {
std::string authStr;
SetOption(authStr, "authentication", hostmap, hint, isMandatory);
hc.authentication = GetHostAuthProtocol(authStr);
SetOption(hc.hostname, "host", hostmap, hint, isMandatory);
SetOption(hc.username, "user", hostmap, hint);
SetOption(hc.remoteServerPath, "serverpath", hostmap, hint, isMandatory);
SetOption(hc.port, "port", hostmap, hint);
SetOption(hc.localPort, "local_port", hostmap, hint);
break;
}
case HostAccessProtocol::XRootD: {
std::string authStr;
SetOption(authStr, "authentication", hostmap, hint, isMandatory);
hc.authentication = GetHostAuthProtocol(authStr);
SetOption(hc.hostname, "host", hostmap, hint, isMandatory);
SetOption(hc.username, "user", hostmap, hint);
SetOption(hc.remoteServerPath, "serverpath", hostmap, hint);
SetOption(hc.port, "port", hostmap, hint);
break;
}
case HostAccessProtocol::S3: {
SetOption(hc.awsProfile, "profile", hostmap, hint);
SetOption(hc.endpoint, "endpoint", hostmap, hint, isMandatory);
SetOption(hc.isAWS_EC2, "aws_ec2_metadata", hostmap, hint);
SetOption(hc.recheckMetadata, "recheck_metadata", hostmap, hint);
break;
}
default:
helper::Throw<std::invalid_argument>(
"Helper", "adiosHostOptions", "ParseHostOptionsFile",
"parser error: invalid access host protocol '" + protocolStr + " for " +
hc.name + "', " + hint);
}

hostConfigs.push_back(hc);
}
hosts.emplace(hostname, hostConfigs);
}
}

} // end namespace helper
} // end namespace adios2
Loading

0 comments on commit b940238

Please sign in to comment.