Merge pull request #4217 from pnorbert/backport-PR4189-to-master

Backport PR 4189 and 4174 to master

source/adios2/common/ADIOSTypes.h

@@ -21,6 +21,7 @@
 #include <complex>
 #include <limits>
 #include <map>
+#include <memory>
 #include <string>
 #include <type_traits>
 #include <utility> //std::pair
@@ -384,6 +385,48 @@ struct UserOptions
     SST sst;
 };
 
+/** Host access protocols */
+enum class HostAccessProtocol
+{
+    Invalid,
+    SSH,
+    XRootD,
+    S3
+};
+
+/** Host authentication protocols */
+enum class HostAuthProtocol
+{
+    Invalid,
+    Password,
+    X509
+};
+
+struct HostConfig
+{
+    std::string name; // Connection Option name
+    HostAccessProtocol protocol = HostAccessProtocol::Invalid;
+
+    /* ssh and xrootd parameters */
+    uint16_t port = 0;
+    uint16_t localPort = 0;
+    std::string hostname = "";
+    std::string username = "";
+    std::string remoteServerPath = "";
+    HostAuthProtocol authentication = HostAuthProtocol::Invalid;
+
+    /* s3 parameters */
+    std::string endpoint = "";
+    std::string awsProfile = "default"; // profile name in ~/.aws/credentials
+    bool isAWS_EC2 = false;
+    bool recheckMetadata = true;
+
+    int verbose = 0;
+};
+
+/** HostOptions holds all user options from ~/.config/adios2/hosts.yaml */
+using HostOptions = std::map<std::string, std::vector<HostConfig>>;
+
 /**
  * os << [adios2_type] enables output of adios2 enums/classes directly
  * to output streams (e.g. std::cout), if ToString() can handle [adios2_type].

source/adios2/core/ADIOS.cpp

@@ -110,6 +110,9 @@ static std::atomic_uint adios_count(0);    // total adios objects during runtime
 /** User defined options from ~/.config/adios2/adios2.yaml if it exists */
 const adios2::UserOptions &ADIOS::GetUserOptions() { return m_UserOptions; };
 
+/** A constant reference to the host options from ~/.config/adios2/hosts.yaml */
+const adios2::HostOptions &ADIOS::GetHostOptions() { return m_HostOptions; };
+
 ADIOS::ADIOS(const std::string configFile, helper::Comm comm, const std::string hostLanguage)
 : m_HostLanguage(hostLanguage), m_Comm(std::move(comm)), m_ConfigFile(configFile),
   m_CampaignManager(m_Comm)
@@ -129,6 +132,7 @@ ADIOS::ADIOS(const std::string configFile, helper::Comm comm, const std::string
     }
 #endif
     ProcessUserConfig();
+    ProcessHostConfig();
     if (!configFile.empty())
     {
         if (!adios2sys::SystemTools::FileExists(configFile))
@@ -185,7 +189,7 @@ void ADIOS::SetUserOptionDefaults()
 {
     m_UserOptions.general.verbose = 0;
 
-    m_UserOptions.campaign.active = true;
+    m_UserOptions.campaign.active = false;
     m_UserOptions.campaign.verbose = 0;
     m_UserOptions.campaign.hostname = "";
     m_UserOptions.campaign.campaignstorepath = "";
@@ -211,6 +215,22 @@ void ADIOS::ProcessUserConfig()
     }
 }
 
+void ADIOS::ProcessHostConfig()
+{
+    // read config parameters from config file
+    std::string homePath;
+#ifdef _WIN32
+    homePath = getenv("HOMEPATH");
+#else
+    homePath = getenv("HOME");
+#endif
+    const std::string cfgFile = homePath + "/.config/adios2/hosts.yaml";
+    if (adios2sys::SystemTools::FileExists(cfgFile))
+    {
+        helper::ParseHostOptionsFile(m_Comm, cfgFile, m_HostOptions, homePath);
+    }
+}
+
 IO &ADIOS::DeclareIO(const std::string name, const ArrayOrdering ArrayOrder)
 {
     auto itIO = m_IOs.find(name);

source/adios2/core/ADIOS.h

@@ -167,6 +167,9 @@ class ADIOS
     /** A constant reference to the user options from ~/.config/adios2/adios2.yaml */
     const adios2::UserOptions &GetUserOptions();
 
+    /** A constant reference to the host options from ~/.config/adios2/hosts.yaml */
+    const adios2::HostOptions &GetHostOptions();
+
 private:
     /** Communicator given to parallel constructor. */
     helper::Comm m_Comm;
@@ -208,8 +211,11 @@ class ADIOS
                     core::IO &io);
 
     adios2::UserOptions m_UserOptions;
+    adios2::HostOptions m_HostOptions;
+    adios2::HostConfig m_Test;
     void SetUserOptionDefaults();
     void ProcessUserConfig();
+    void ProcessHostConfig();
 
 private:
     /* Global services that we want to initialize at most once and shutdown

source/adios2/core/Engine.cpp

@@ -24,7 +24,7 @@ namespace core
 Engine::Engine(const std::string engineType, IO &io, const std::string &name, const Mode openMode,
                helper::Comm comm)
 : m_EngineType(engineType), m_IO(io), m_Name(name), m_OpenMode(openMode), m_Comm(std::move(comm)),
-  m_UserOptions(io.m_ADIOS.GetUserOptions())
+  m_UserOptions(io.m_ADIOS.GetUserOptions()), m_HostOptions(io.m_ADIOS.GetHostOptions())
 {
     m_FailVerbose = (m_Comm.Rank() == 0);
 }

source/adios2/core/Engine.h

@@ -510,6 +510,9 @@ class Engine
     /** User options parsed by the ADIOS object, here just for easy reference */
     const UserOptions &m_UserOptions;
 
+    /** Host options parsed by the ADIOS object, here just for easy reference */
+    const HostOptions &m_HostOptions;
+
     /** keeps track of current advance status */
     StepStatus m_AdvanceStatus = StepStatus::OK;
 

source/adios2/engine/campaign/CampaignManager.h

@@ -51,7 +51,7 @@ class CampaignManager
     int m_WriterRank;
     CampaignRecordMap cmap;
     std::ofstream m_Output;
-    const std::string m_CampaignDir = "adios-campaign";
+    const std::string m_CampaignDir = ".adios-campaign";
 
 #else
 public:

source/adios2/engine/campaign/CampaignReader.cpp

@@ -229,27 +229,75 @@ void CampaignReader::InitTransports()
         std::string localPath;
         if (m_CampaignData.hosts[ds.hostIdx].hostname != m_Options.hostname)
         {
-            const std::string remotePath =
-                m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + PathSeparator + ds.name;
-            const std::string remoteURL =
-                m_CampaignData.hosts[ds.hostIdx].hostname + ":" + remotePath;
-            localPath = m_Options.cachepath + PathSeparator +
-                        m_CampaignData.hosts[ds.hostIdx].hostname + PathSeparator + m_Name +
-                        PathSeparator + ds.name;
-            if (m_Options.verbose > 0)
+            bool done = false;
+            auto it = m_HostOptions.find(m_CampaignData.hosts[ds.hostIdx].hostname);
+            if (it != m_HostOptions.end())
             {
-                std::cout << "Open remote file " << remoteURL
-                          << "\n    and use local cache for metadata at " << localPath << " \n";
+                const HostConfig &ho = (it->second).front();
+                if (ho.protocol == HostAccessProtocol::S3)
+                {
+                    const std::string endpointURL = ho.endpoint;
+                    const std::string objPath =
+                        m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + "/" + ds.name;
+                    Params p;
+                    p.emplace("Library", "awssdk");
+                    p.emplace("endpoint", endpointURL);
+                    p.emplace("cache", m_Options.cachepath + PathSeparator +
+                                           m_CampaignData.hosts[ds.hostIdx].hostname +
+                                           PathSeparator + m_Name);
+                    p.emplace("verbose", std::to_string(ho.verbose));
+                    p.emplace("recheck_metadata", (ho.recheckMetadata ? "true" : "false"));
+                    io.AddTransport("File", p);
+                    io.SetEngine("BP5");
+                    localPath = m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] +
+                                PathSeparator + ds.name;
+                    if (ho.isAWS_EC2)
+                    {
+                        adios2sys::SystemTools::PutEnv("AWS_EC2_METADATA_DISABLED=false");
+                    }
+                    else
+                    {
+                        adios2sys::SystemTools::PutEnv("AWS_EC2_METADATA_DISABLED=true");
+                    }
+
+                    if (ho.awsProfile.empty())
+                    {
+                        adios2sys::SystemTools::PutEnv("AWS_PROFILE=default");
+                    }
+                    else
+                    {
+                        std::string es = "AWS_PROFILE=" + ho.awsProfile;
+                        adios2sys::SystemTools::PutEnv(es);
+                    }
+
+                    done = true;
+                }
             }
-            helper::CreateDirectory(localPath);
-            for (auto &bpf : ds.files)
+
+            if (!done)
             {
-                /*std::cout << "     save file " << remoteURL << "/" <<
-                   bpf.name
-                          << " to " << localPath << "/" << bpf.name << "\n";*/
-                SaveToFile(m_DB, localPath + PathSeparator + bpf.name, bpf);
+                const std::string remotePath =
+                    m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + PathSeparator + ds.name;
+                const std::string remoteURL =
+                    m_CampaignData.hosts[ds.hostIdx].hostname + ":" + remotePath;
+                localPath = m_Options.cachepath + PathSeparator +
+                            m_CampaignData.hosts[ds.hostIdx].hostname + PathSeparator + m_Name +
+                            PathSeparator + ds.name;
+                if (m_Options.verbose > 0)
+                {
+                    std::cout << "Open remote file " << remoteURL
+                              << "\n    and use local cache for metadata at " << localPath << " \n";
+                }
+                helper::CreateDirectory(localPath);
+                for (auto &bpf : ds.files)
+                {
+                    /*std::cout << "     save file " << remoteURL << "/" <<
+                       bpf.name
+                              << " to " << localPath << "/" << bpf.name << "\n";*/
+                    SaveToFile(m_DB, localPath + PathSeparator + bpf.name, bpf);
+                }
+                io.SetParameter("RemoteDataPath", remotePath);
             }
-            io.SetParameter("RemoteDataPath", remotePath);
         }
         else
         {