From 60976c33f853630984d2123f55c65ab332ea5349 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Mon, 15 May 2023 08:00:23 -0400 Subject: [PATCH 01/58] Added CampaignManager to be created by ADIOS object. It does nothing at this point. Added CampaignReader engine as a skeleton reader. --- source/adios2/CMakeLists.txt | 3 + source/adios2/core/ADIOS.cpp | 14 +- source/adios2/core/ADIOS.h | 6 + .../engine/campaign/CampaignManager.cpp | 79 ++++++++ .../adios2/engine/campaign/CampaignManager.h | 50 +++++ .../adios2/engine/campaign/CampaignReader.cpp | 173 ++++++++++++++++++ .../adios2/engine/campaign/CampaignReader.h | 91 +++++++++ .../adios2/engine/campaign/CampaignReader.tcc | 64 +++++++ 8 files changed, 478 insertions(+), 2 deletions(-) create mode 100644 source/adios2/engine/campaign/CampaignManager.cpp create mode 100644 source/adios2/engine/campaign/CampaignManager.h create mode 100644 source/adios2/engine/campaign/CampaignReader.cpp create mode 100644 source/adios2/engine/campaign/CampaignReader.h create mode 100644 source/adios2/engine/campaign/CampaignReader.tcc diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index 659ca94e70..9e0fb0b216 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -56,6 +56,9 @@ add_library(adios2_core engine/inline/InlineReader.cpp engine/inline/InlineReader.tcc engine/inline/InlineWriter.cpp engine/inline/InlineWriter.tcc + engine/campaign/CampaignManager.cpp + engine/campaign/CampaignReader.cpp engine/campaign/CampaignReader.tcc + engine/null/NullWriter.cpp engine/null/NullWriter.tcc engine/null/NullReader.cpp engine/null/NullReader.tcc diff --git a/source/adios2/core/ADIOS.cpp b/source/adios2/core/ADIOS.cpp index 3e058c9f20..7380d6f850 100644 --- a/source/adios2/core/ADIOS.cpp +++ b/source/adios2/core/ADIOS.cpp @@ -103,12 +103,15 @@ class ADIOS::GlobalServices ADIOS::GlobalServices ADIOS::m_GlobalServices; std::mutex PerfStubsMutex; -static std::atomic_uint adios_refcount(0); +static std::atomic_uint adios_refcount(0); // adios objects at the same time +static std::atomic_uint adios_count(0); // total adios objects during runtime ADIOS::ADIOS(const std::string configFile, helper::Comm comm, const std::string hostLanguage) -: m_HostLanguage(hostLanguage), m_Comm(std::move(comm)), m_ConfigFile(configFile) +: m_HostLanguage(hostLanguage), m_Comm(std::move(comm)), m_ConfigFile(configFile), + campaignManager(m_Comm) { ++adios_refcount; + ++adios_count; #ifdef PERFSTUBS_USE_TIMERS { std::lock_guard lck(PerfStubsMutex); @@ -140,6 +143,12 @@ ADIOS::ADIOS(const std::string configFile, helper::Comm comm, const std::string #ifdef ADIOS2_HAVE_KOKKOS m_GlobalServices.Init_Kokkos_API(); #endif + std::string campaignName = "campaign"; + if (adios_count > 1) + { + campaignName = "campaign" + std::to_string(adios_count); + } + campaignManager.Open(campaignName); } ADIOS::ADIOS(const std::string configFile, const std::string hostLanguage) @@ -161,6 +170,7 @@ ADIOS::~ADIOS() { m_GlobalServices.Finalize(); } + campaignManager.Close(); } IO &ADIOS::DeclareIO(const std::string name, const ArrayOrdering ArrayOrder) diff --git a/source/adios2/core/ADIOS.h b/source/adios2/core/ADIOS.h index 1e514ca051..621cf9c192 100644 --- a/source/adios2/core/ADIOS.h +++ b/source/adios2/core/ADIOS.h @@ -25,6 +25,9 @@ #include "adios2/core/VariableStruct.h" #include "adios2/helper/adiosComm.h" +// Campaign Manager as Global Service +#include + namespace adios2 { namespace core @@ -179,6 +182,9 @@ class ADIOS /** operators created with DefineOperator */ std::unordered_map> m_Operators; + /** campaign manager */ + engine::CampaignManager campaignManager; + /** Flag for Enter/ExitComputationBlock */ bool enteredComputationBlock = false; diff --git a/source/adios2/engine/campaign/CampaignManager.cpp b/source/adios2/engine/campaign/CampaignManager.cpp new file mode 100644 index 0000000000..6ea2c5451f --- /dev/null +++ b/source/adios2/engine/campaign/CampaignManager.cpp @@ -0,0 +1,79 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * CampaignManager.cpp + * + * Created on: May 15, 2023 + * Author: Norbert Podhorszki pnorbert@ornl.gov + */ + +#include "CampaignManager.h" + +#include "adios2/helper/adiosFunctions.h" + +#include + +namespace adios2 +{ +namespace core +{ +namespace engine +{ + +CampaignManager::CampaignManager(adios2::helper::Comm &comm) +{ + m_WriterRank = comm.Rank(); + if (m_Verbosity == 5) + { + std::cout << "Campaign Manager " << m_WriterRank + << " constructor called" << std::endl; + } +} + +CampaignManager::~CampaignManager() +{ + if (m_Verbosity == 5) + { + std::cout << "Campaign Manager " << m_WriterRank + << " desctructor called\n"; + } + if (m_Opened) + { + Close(); + } +} + +void CampaignManager::Open(const std::string &name) +{ + if (m_Verbosity == 5) + { + std::cout << "Campaign Manager " << m_WriterRank << " Open(" << name + << ")\n"; + } + m_Opened = true; +} + +void CampaignManager::Record(const std::string &name, const size_t step, + const double time) +{ + if (m_Verbosity == 5) + { + std::cout << "Campaign Manager " << m_WriterRank + << " Record name = " << name << " step = " << step + << " time = " << time << "\n"; + } +} + +void CampaignManager::Close() +{ + if (m_Verbosity == 5) + { + std::cout << "Campaign Manager " << m_WriterRank << " Close()\n"; + } + m_Opened = false; +} + +} // end namespace engine +} // end namespace core +} // end namespace adios2 diff --git a/source/adios2/engine/campaign/CampaignManager.h b/source/adios2/engine/campaign/CampaignManager.h new file mode 100644 index 0000000000..164e0e2d94 --- /dev/null +++ b/source/adios2/engine/campaign/CampaignManager.h @@ -0,0 +1,50 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * CampaignManager.h + * Campaign manager object created by the ADIOS object as GlobalService. + * Record all output under the ADIOS object so that CampaignReader can + * see all data for as a single reader. + * This is NOT a writer Engine but the CampaignReader is a reader Engine. + * + * Created on: May 15, 2023 + * Author: Norbert Podhorszki pnorbert@ornl.gov + */ + +#ifndef ADIOS2_ENGINE_CAMPAIGNMANAGER_H_ +#define ADIOS2_ENGINE_CAMPAIGNMANAGER_H_ + +#include "adios2/common/ADIOSConfig.h" +#include "adios2/helper/adiosComm.h" + +namespace adios2 +{ +namespace core +{ + +namespace engine +{ + +class CampaignManager +{ + +public: + CampaignManager(helper::Comm &comm); + ~CampaignManager(); + + void Open(const std::string &name); + void Record(const std::string &name, const size_t step, const double time); + void Close(); + +private: + bool m_Opened = false; + int m_WriterRank; + int m_Verbosity = 5; +}; + +} // end namespace engine +} // end namespace core +} // end namespace adios2 + +#endif /* ADIOS2_ENGINE_CAMPAIGNMANAGER_H_ */ diff --git a/source/adios2/engine/campaign/CampaignReader.cpp b/source/adios2/engine/campaign/CampaignReader.cpp new file mode 100644 index 0000000000..c8dae82793 --- /dev/null +++ b/source/adios2/engine/campaign/CampaignReader.cpp @@ -0,0 +1,173 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * CampaignReader.cpp + * + * Created on: May 15, 2023 + * Author: Norbert Podhorszki pnorbert@ornl.gov + */ + +#include "CampaignReader.h" +#include "CampaignReader.tcc" + +#include "adios2/helper/adiosFunctions.h" // CSVToVector + +#include + +namespace adios2 +{ +namespace core +{ +namespace engine +{ + +CampaignReader::CampaignReader(IO &io, const std::string &name, const Mode mode, + helper::Comm comm) +: Engine("CampaignReader", io, name, mode, std::move(comm)) +{ + m_ReaderRank = m_Comm.Rank(); + Init(); + if (m_Verbosity == 5) + { + std::cout << "Skeleton Reader " << m_ReaderRank << " Open(" << m_Name + << ") in constructor." << std::endl; + } + m_IsOpen = true; +} + +CampaignReader::~CampaignReader() +{ + /* m_Skeleton deconstructor does close and finalize */ + if (m_Verbosity == 5) + { + std::cout << "Skeleton Reader " << m_ReaderRank << " deconstructor on " + << m_Name << "\n"; + } + if (m_IsOpen) + { + DestructorClose(m_FailVerbose); + } + m_IsOpen = false; +} + +StepStatus CampaignReader::BeginStep(const StepMode mode, + const float timeoutSeconds) +{ + // step info should be received from the writer side in BeginStep() + // so this forced increase should not be here + ++m_CurrentStep; + + if (m_Verbosity == 5) + { + std::cout << "Skeleton Reader " << m_ReaderRank + << " BeginStep() new step " << m_CurrentStep << "\n"; + } + + // If we reach the end of stream (writer is gone or explicitly tells the + // reader) + // we return EndOfStream to the reader application + if (m_CurrentStep == 2) + { + std::cout << "Skeleton Reader " << m_ReaderRank + << " forcefully returns End of Stream at this step\n"; + + return StepStatus::EndOfStream; + } + + // We should block until a new step arrives or reach the timeout + + // m_IO Variables and Attributes should be defined at this point + // so that the application can inquire them and start getting data + + return StepStatus::OK; +} + +void CampaignReader::PerformGets() +{ + if (m_Verbosity == 5) + { + std::cout << "Skeleton Reader " << m_ReaderRank + << " PerformGets()\n"; + } + m_NeedPerformGets = false; +} + +size_t CampaignReader::CurrentStep() const { return m_CurrentStep; } + +void CampaignReader::EndStep() +{ + // EndStep should call PerformGets() if there are unserved GetDeferred() + // requests + if (m_NeedPerformGets) + { + PerformGets(); + } + + if (m_Verbosity == 5) + { + std::cout << "Skeleton Reader " << m_ReaderRank << " EndStep()\n"; + } +} + +// PRIVATE + +#define declare_type(T) \ + void CampaignReader::DoGetSync(Variable &variable, T *data) \ + { \ + GetSyncCommon(variable, data); \ + } \ + void CampaignReader::DoGetDeferred(Variable &variable, T *data) \ + { \ + GetDeferredCommon(variable, data); \ + } + +ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + +void CampaignReader::Init() +{ + InitParameters(); + InitTransports(); +} + +void CampaignReader::InitParameters() +{ + for (const auto &pair : m_IO.m_Parameters) + { + std::string key(pair.first); + std::transform(key.begin(), key.end(), key.begin(), ::tolower); + + std::string value(pair.second); + std::transform(value.begin(), value.end(), value.begin(), ::tolower); + + if (key == "verbose") + { + m_Verbosity = std::stoi(value); + if (m_Verbosity < 0 || m_Verbosity > 5) + helper::Throw( + "Engine", "CampaignReader", "InitParameters", + "Method verbose argument must be an " + "integer in the range [0,5], in call to " + "Open or Engine constructor"); + } + } +} + +void CampaignReader::InitTransports() +{ + // Nothing to process from m_IO.m_TransportsParameters +} + +void CampaignReader::DoClose(const int transportIndex) +{ + if (m_Verbosity == 5) + { + std::cout << "Skeleton Reader " << m_ReaderRank << " Close(" << m_Name + << ")\n"; + } +} + +} // end namespace engine +} // end namespace core +} // end namespace adios2 diff --git a/source/adios2/engine/campaign/CampaignReader.h b/source/adios2/engine/campaign/CampaignReader.h new file mode 100644 index 0000000000..7f0078d392 --- /dev/null +++ b/source/adios2/engine/campaign/CampaignReader.h @@ -0,0 +1,91 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * CampaignReader.h + * An empty skeleton engine from which any engine can be built + * + * Created on: May 15, 2023 + * Author: Norbert Podhorszki pnorbert@ornl.gov + */ + +#ifndef ADIOS2_ENGINE_CAMPAIGNREADER_H_ +#define ADIOS2_ENGINE_CAMPAIGNREADER_H_ + +#include "adios2/common/ADIOSConfig.h" +#include "adios2/core/ADIOS.h" +#include "adios2/core/Engine.h" +#include "adios2/helper/adiosComm.h" +#include "adios2/helper/adiosFunctions.h" + +namespace adios2 +{ +namespace core +{ +namespace engine +{ + +class CampaignReader : public Engine +{ +public: + /** + * Constructor for single BP capsule engine, writes in BP format into a + * single + * heap capsule + * @param name unique name given to the engine + * @param accessMode + * @param comm + * @param method + * @param hostLanguage + */ + CampaignReader(IO &adios, const std::string &name, const Mode mode, + helper::Comm comm); + + ~CampaignReader(); + StepStatus BeginStep(StepMode mode = StepMode::Read, + const float timeoutSeconds = -1.0) final; + void PerformGets() final; + size_t CurrentStep() const final; + void EndStep() final; + +private: + int m_Verbosity = 0; + int m_ReaderRank; // my rank in the readers' comm + + // step info should be received from the writer side in BeginStep() + int m_CurrentStep = -1; + + // EndStep must call PerformGets if necessary + bool m_NeedPerformGets = false; + + void Init() final; ///< called from constructor, gets the selected Skeleton + /// transport method from settings + void InitParameters() final; + void InitTransports() final; + +#define declare_type(T) \ + void DoGetSync(Variable &, T *) final; \ + void DoGetDeferred(Variable &, T *) final; + ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + + void DoClose(const int transportIndex = -1); + + /** + * Called if destructor is called on an open engine. Should warn or take + * any non-complex measure that might help recover. + */ + void DestructorClose(bool Verbose) noexcept final{}; + + template + void GetSyncCommon(Variable &variable, T *data); + + template + void GetDeferredCommon(Variable &variable, T *data); +}; + +} // end namespace engine +} // end namespace core +} // end namespace adios2 + +#endif /* ADIOS2_ENGINE_CAMPAIGNREADER_H_ */ diff --git a/source/adios2/engine/campaign/CampaignReader.tcc b/source/adios2/engine/campaign/CampaignReader.tcc new file mode 100644 index 0000000000..d4d3252281 --- /dev/null +++ b/source/adios2/engine/campaign/CampaignReader.tcc @@ -0,0 +1,64 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * CampaignReader.tcc + * + * Created on: May 15, 2023 + * Author: Norbert Podhorszki pnorbert@ornl.gov + */ + +#ifndef ADIOS2_ENGINE_CAMPAIGNREADER_TCC_ +#define ADIOS2_ENGINE_CAMPAIGNREADER_TCC_ + +#include "CampaignReader.h" + +#include + +namespace adios2 +{ +namespace core +{ +namespace engine +{ + +template <> +inline void CampaignReader::GetSyncCommon(Variable &variable, + std::string *data) +{ + variable.m_Data = data; + if (m_Verbosity == 5) + { + std::cout << "Skeleton Reader " << m_ReaderRank << " GetSync(" + << variable.m_Name << ")\n"; + } +} + +template +inline void CampaignReader::GetSyncCommon(Variable &variable, T *data) +{ + variable.m_Data = data; + if (m_Verbosity == 5) + { + std::cout << "Skeleton Reader " << m_ReaderRank << " GetSync(" + << variable.m_Name << ")\n"; + } +} + +template +void CampaignReader::GetDeferredCommon(Variable &variable, T *data) +{ + // returns immediately + if (m_Verbosity == 5) + { + std::cout << "Skeleton Reader " << m_ReaderRank << " GetDeferred(" + << variable.m_Name << ")\n"; + } + m_NeedPerformGets = true; +} + +} // end namespace engine +} // end namespace core +} // end namespace adios2 + +#endif // ADIOS2_ENGINE_CAMPAIGNREADER_TCC_ From 45b08920f81d408b2e54bb77a09ed46bb3073c6a Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 16 May 2023 08:12:54 -0400 Subject: [PATCH 02/58] store campaign records in memory so far --- source/adios2/common/ADIOSTypes.h | 3 + source/adios2/core/ADIOS.cpp | 11 +++- source/adios2/core/ADIOS.h | 6 +- source/adios2/engine/bp5/BP5Writer.cpp | 4 ++ .../engine/campaign/CampaignManager.cpp | 59 ++++++++++++++++++- .../adios2/engine/campaign/CampaignManager.h | 2 + 6 files changed, 80 insertions(+), 5 deletions(-) diff --git a/source/adios2/common/ADIOSTypes.h b/source/adios2/common/ADIOSTypes.h index acc64ee917..3b4a52b8d7 100644 --- a/source/adios2/common/ADIOSTypes.h +++ b/source/adios2/common/ADIOSTypes.h @@ -284,6 +284,9 @@ constexpr bool end_step = true; constexpr bool LocalValue = true; constexpr bool GlobalValue = false; +constexpr size_t UnknownStep = MaxU64; +constexpr double UnknownTime = std::numeric_limits::infinity(); + using Dims = std::vector; using Params = std::map; using vParams = std::vector>; diff --git a/source/adios2/core/ADIOS.cpp b/source/adios2/core/ADIOS.cpp index 7380d6f850..9e075dda38 100644 --- a/source/adios2/core/ADIOS.cpp +++ b/source/adios2/core/ADIOS.cpp @@ -108,7 +108,7 @@ static std::atomic_uint adios_count(0); // total adios objects during runtime ADIOS::ADIOS(const std::string configFile, helper::Comm comm, const std::string hostLanguage) : m_HostLanguage(hostLanguage), m_Comm(std::move(comm)), m_ConfigFile(configFile), - campaignManager(m_Comm) + m_CampaignManager(m_Comm) { ++adios_refcount; ++adios_count; @@ -148,7 +148,7 @@ ADIOS::ADIOS(const std::string configFile, helper::Comm comm, const std::string { campaignName = "campaign" + std::to_string(adios_count); } - campaignManager.Open(campaignName); + m_CampaignManager.Open(campaignName); } ADIOS::ADIOS(const std::string configFile, const std::string hostLanguage) @@ -170,7 +170,7 @@ ADIOS::~ADIOS() { m_GlobalServices.Finalize(); } - campaignManager.Close(); + m_CampaignManager.Close(); } IO &ADIOS::DeclareIO(const std::string name, const ArrayOrdering ArrayOrder) @@ -306,6 +306,11 @@ void ADIOS::YAMLInit(const std::string &configFileYAML) helper::ParseConfigYAML(*this, configFileYAML, m_IOs); } +void ADIOS::RecordOutputStep(const std::string &name, const size_t step, const double time) +{ + m_CampaignManager.Record(name, step, time); +} + void ADIOS::Global_init_AWS_API() { m_GlobalServices.CheckStatus(); diff --git a/source/adios2/core/ADIOS.h b/source/adios2/core/ADIOS.h index 621cf9c192..e6d7940b48 100644 --- a/source/adios2/core/ADIOS.h +++ b/source/adios2/core/ADIOS.h @@ -161,6 +161,10 @@ class ADIOS * in main thread. Useful when using Async IO */ void ExitComputationBlock() noexcept; + void RecordOutputStep(const std::string &name, + const size_t step = UnknownStep, + const double time = UnknownTime); + private: /** Communicator given to parallel constructor. */ helper::Comm m_Comm; @@ -183,7 +187,7 @@ class ADIOS std::unordered_map> m_Operators; /** campaign manager */ - engine::CampaignManager campaignManager; + engine::CampaignManager m_CampaignManager; /** Flag for Enter/ExitComputationBlock */ bool enteredComputationBlock = false; diff --git a/source/adios2/engine/bp5/BP5Writer.cpp b/source/adios2/engine/bp5/BP5Writer.cpp index c0b6462f7a..b5a4a3260c 100644 --- a/source/adios2/engine/bp5/BP5Writer.cpp +++ b/source/adios2/engine/bp5/BP5Writer.cpp @@ -653,6 +653,10 @@ void BP5Writer::EndStep() m_Profiler.Stop("ES"); m_WriterStep++; m_EndStepEnd = Now(); + if (!m_RankMPI) + { + m_IO.m_ADIOS.RecordOutputStep(m_Name, UnknownStep, UnknownTime); + } /* Seconds ts2 = Now() - m_EngineStart; std::cout << "END STEP ended at: " << ts2.count() << std::endl;*/ } diff --git a/source/adios2/engine/campaign/CampaignManager.cpp b/source/adios2/engine/campaign/CampaignManager.cpp index 6ea2c5451f..cd8b2f38c4 100644 --- a/source/adios2/engine/campaign/CampaignManager.cpp +++ b/source/adios2/engine/campaign/CampaignManager.cpp @@ -4,6 +4,8 @@ * * CampaignManager.cpp * + * This is NOT a writer Engine but the CampaignReader is a reader Engine. + * * Created on: May 15, 2023 * Author: Norbert Podhorszki pnorbert@ornl.gov */ @@ -63,13 +65,68 @@ void CampaignManager::Record(const std::string &name, const size_t step, << " Record name = " << name << " step = " << step << " time = " << time << "\n"; } + auto r = cmap.find(name); + if (r != cmap.end()) + { + // update record + size_t last_step = r->second.steps.back(); + size_t delta_step = step - last_step; + double last_time = r->second.times.back(); + double delta_time = time - last_time; + int nsteps = r->second.steps.size(); + if (nsteps == 1) + { + r->second.delta_step = delta_step; + r->second.delta_time = delta_time; + } + else + { + size_t old_delta_step = + r->second.steps.back() - r->second.steps.rbegin()[1]; + if (old_delta_step != delta_step) + { + r->second.delta_step = 0; + r->second.delta_time = 0.0; + r->second.varying_deltas = true; + } + } + r->second.steps.push_back(step); + r->second.times.push_back(time); + } + else + { + // new entry + CampaignRecord r(step, time); + cmap.emplace(name, r); + } } void CampaignManager::Close() { if (m_Verbosity == 5) { - std::cout << "Campaign Manager " << m_WriterRank << " Close()\n"; + std::cout << "Campaign Manager " << m_WriterRank + << " Close()\nCampaign"; + for (const auto &r : cmap) + { + std::cout << "name = " << r.first << "\n"; + std::cout << " varying_deltas = " << r.second.varying_deltas + << "\n"; + std::cout << " delta_step = " << r.second.delta_step << "\n"; + std::cout << " delta_time = " << r.second.delta_time << "\n"; + std::cout << " steps = {"; + for (const auto s : r.second.steps) + { + std::cout << s << " "; + } + std::cout << "}\n"; + std::cout << " times = {"; + for (const auto t : r.second.times) + { + std::cout << t << " "; + } + std::cout << "}\n"; + } } m_Opened = false; } diff --git a/source/adios2/engine/campaign/CampaignManager.h b/source/adios2/engine/campaign/CampaignManager.h index 164e0e2d94..3b05eb6f6a 100644 --- a/source/adios2/engine/campaign/CampaignManager.h +++ b/source/adios2/engine/campaign/CampaignManager.h @@ -15,6 +15,7 @@ #ifndef ADIOS2_ENGINE_CAMPAIGNMANAGER_H_ #define ADIOS2_ENGINE_CAMPAIGNMANAGER_H_ +#include "CampaignData.h" #include "adios2/common/ADIOSConfig.h" #include "adios2/helper/adiosComm.h" @@ -41,6 +42,7 @@ class CampaignManager bool m_Opened = false; int m_WriterRank; int m_Verbosity = 5; + CampaignMap cmap; }; } // end namespace engine From 6825d946466220645b693791bda73ab57bbc683f Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Wed, 17 May 2023 09:50:31 -0400 Subject: [PATCH 03/58] Added example to write many different files from different communicators and different ADIOS objects. --- examples/CMakeLists.txt | 1 + examples/campaign/CMakeLists.txt | 9 + examples/campaign/campaign_write.cpp | 406 +++++++++++++++++++++++++++ 3 files changed, 416 insertions(+) create mode 100644 examples/campaign/CMakeLists.txt create mode 100644 examples/campaign/campaign_write.cpp diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index e14d76bb05..54a32a8c9b 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -10,6 +10,7 @@ add_subdirectory(useCases) add_subdirectory(inlineMWE) add_subdirectory(plugins) add_subdirectory(fides) +add_subdirectory(campaign) if(ADIOS2_HAVE_MPI) add_subdirectory(heatTransfer) diff --git a/examples/campaign/CMakeLists.txt b/examples/campaign/CMakeLists.txt new file mode 100644 index 0000000000..ad7b3aed0d --- /dev/null +++ b/examples/campaign/CMakeLists.txt @@ -0,0 +1,9 @@ +#------------------------------------------------------------------------------# +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +#------------------------------------------------------------------------------# + +if(ADIOS2_HAVE_MPI) + add_executable(campaign_write campaign_write.cpp) + target_link_libraries(campaign_write adios2::cxx11_mpi adios2_core MPI::MPI_C) +endif() diff --git a/examples/campaign/campaign_write.cpp b/examples/campaign/campaign_write.cpp new file mode 100644 index 0000000000..6d919f09e6 --- /dev/null +++ b/examples/campaign/campaign_write.cpp @@ -0,0 +1,406 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * Write various outputs, that the ADIOS built-in campaign manager records + * and one can read all data using a single reader engine on the campaign + * output instead of the individual files + * + * Campaign managment handles multiple restarts of the writer, where + * some existing outputs get appended to (at some specific step) and some + * other new outputs are created. + * + * Campaign management handles BP3/BP4/BP5 and HDF5 outputs + * + * Outputs: + * dataAll.bp: written by every process + * // dataFirstRank.bp: written by rank 0 only + * // dataLastRank.bp: written by last rank only + * dataStep{N}.bp a file series every step (written by all) + * dataAnother.h5: written by rank 1..N from another ADIOS object + * dataNew{N}.bp : a new file when restarting from step N + * dataFinal.bp: written by rank 1 from a third ADIOS object at the end + + * Created on: May 17, 2023 + * Author: Norbert Podhorszki + */ + +#include +#include +#include +#include + +#include +#include // ADIOS2_USE_HDF5 macro +#include // StringToSizeT() +#include + +/* Arguments */ +size_t nSteps = 5; +size_t startStep = 0; + +/* MPI variables */ +int rank, nproc; +MPI_Comm comm; +int wrank, wnproc; // MPI_COMM_WORLD rank and nproc + +MPI_Comm commAnother; +MPI_Comm commFirstRank; +MPI_Comm commLastRank; + +constexpr size_t Nx = 6; +constexpr size_t Ny = 4; +constexpr size_t Nz = 5; + +// imitating having a physical iteration counter and time value +uint64_t physicalStep = 0; +double physicalTime = 0.0; +constexpr uint64_t physicalStep_dt = 100; +constexpr double physicalTime_dt = 0.01; + +void printUsage() +{ + std::cout << "Usage: campaign_writer steps [start_step] \n" + << " steps: the total number of steps to output\n" + << " start_step: restart from this step\n\n"; +} + +unsigned int convertToUint(std::string varName, char *arg) +{ + char *end; + unsigned int retval = std::strtoul(arg, &end, 10); + if (end[0] || errno == ERANGE) + { + throw std::invalid_argument("Invalid value given for " + varName + + ": " + std::string(arg)); + } + return retval; +} + +int main(int argc, char *argv[]) +{ + if (argc < 2) + { + printUsage(); + return -1; + } + nSteps = + adios2::helper::StringToSizeT(argv[1], "when parsing argument steps"); + if (argc > 2) + { + startStep = adios2::helper::StringToSizeT( + argv[2], "when parsing argument start_step"); + } + /*nSteps = convertToUint("steps", argv[1]); + if (argc > 2) + { + startStep = convertToUint("start_step", argv[2]); + }*/ + + int provided; + // MPI_THREAD_MULTIPLE is only required if you use SST with MPI DataPlane + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); + MPI_Comm_rank(MPI_COMM_WORLD, &wrank); + MPI_Comm_size(MPI_COMM_WORLD, &wnproc); + if (wnproc < 2) + { + std::cout << "This MPI example needs at least 2 processes" << std::endl; + return -2; + } + + const unsigned int color = 1; + MPI_Comm_split(MPI_COMM_WORLD, color, wrank, &comm); + MPI_Comm_rank(comm, &rank); + MPI_Comm_size(comm, &nproc); + MPI_Group group_world; + MPI_Comm_group(comm, &group_world); + + // communicator for ranks 1..N + const unsigned int colorAnother = (rank ? 1 : 0); + MPI_Comm_split(comm, colorAnother, rank, &commAnother); + + // communicator for first rank + MPI_Group group_first; + int first_rank = 0; + MPI_Group_incl(group_world, 1, &first_rank, &group_first); + MPI_Comm_create(comm, group_first, &commFirstRank); + + // communicator for last rank + MPI_Group group_last; + int last_rank = nproc - 1; + MPI_Group_incl(group_world, 1, &last_rank, &group_last); + MPI_Comm_create(comm, group_last, &commLastRank); + + // some data + std::vector x(Nx); + std::vector y(Ny); + std::vector z(Nz); + + { + adios2::ADIOS adiosAll(comm); + adios2::ADIOS adiosAnother(commAnother); + + adios2::IO ioAll; + adios2::IO ioFirstRank; + adios2::IO ioLastRank; + adios2::IO ioStep; + adios2::IO ioNew; + adios2::IO ioAnother; + + adios2::Engine writerAll; + adios2::Engine writerFirstRank; + adios2::Engine writerLastRank; + adios2::Engine writerAnother; + + ioAll = adiosAll.DeclareIO("OutputAll"); + ioAll.SetEngine("BP5"); + + if (rank == 0) + { + ioFirstRank = adiosAll.DeclareIO("OutputFirst"); + ioFirstRank.SetEngine("BP5"); + } + if (rank == nproc - 1) + { + ioLastRank = adiosAll.DeclareIO("OutputLast"); + ioLastRank.SetEngine("BP5"); + } + ioStep = adiosAll.DeclareIO("OutputStep"); + ioStep.SetEngine("BP4"); + + ioNew = adiosAll.DeclareIO("OutputNew"); +#ifdef ADIOS2_USE_HDF5 + ioNew.SetEngine("HDF5"); +#else + ioNew.SetEngine("BP5"); +#endif + + if (rank > 0) + { + ioAnother = adiosAnother.DeclareIO("OutputAnother"); +#ifdef ADIOS2_USE_HDF5 + ioAnother.SetEngine("HDF5"); +#else + ioAnother.SetEngine("BP5"); +#endif + } + + adios2::Mode mode = adios2::Mode::Write; + if (startStep > 0) + { + mode = adios2::Mode::Append; + auto v = std::to_string(startStep); + ioAll.SetParameter("AppendAfterSteps", v); + if (rank == 0) + { + ioFirstRank.SetParameter("AppendAfterSteps", v); + } + if (rank == nproc - 1) + { + ioLastRank.SetParameter("AppendAfterSteps", v); + } + if (rank > 0) + { + ioAnother.SetParameter("AppendAfterSteps", v); + } + } + + adios2::Variable varXAll = ioAll.DefineVariable( + "x", {static_cast(nproc), Nx}, + {static_cast(rank), 0}, {1, Nx}); + adios2::Variable varYAll = ioAll.DefineVariable( + "y", {static_cast(nproc), Ny}, + {static_cast(rank), 0}, {1, Ny}); + adios2::Variable varZAll = ioAll.DefineVariable( + "z", {static_cast(nproc), Nz}, + {static_cast(rank), 0}, {1, Nz}); + adios2::Variable varStepAll = + ioAll.DefineVariable("AdiosStep"); + adios2::Variable varPhysStepAll = + ioAll.DefineVariable("iteration"); + adios2::Variable varPhysTimeAll = + ioAll.DefineVariable("time"); + ioAll.DefineAttribute("comment", + "Written by all processes"); + + adios2::Variable varZStep = ioStep.DefineVariable( + "z", {(unsigned int)nproc, Nz}, {static_cast(rank), 0}, + {1, Nz}); + adios2::Variable varStepStep = + ioStep.DefineVariable("AdiosStep"); + adios2::Variable varPhysStepStep = + ioStep.DefineVariable("iteration"); + adios2::Variable varPhysTimeStep = + ioStep.DefineVariable("time"); + + adios2::Variable varYNew = ioStep.DefineVariable( + "y", {(unsigned int)nproc, Ny}, {static_cast(rank), 0}, + {1, Ny}); + adios2::Variable varStepNew = + ioNew.DefineVariable("AdiosStep"); + adios2::Variable varPhysStepNew = + ioNew.DefineVariable("iteration"); + adios2::Variable varPhysTimeNew = + ioNew.DefineVariable("time"); + + writerAll = ioAll.Open("dataAll.bp", mode); + + adios2::Variable varXFirstRank; + if (rank == 0) + { + varXFirstRank = + ioFirstRank.DefineVariable("x", {Nx}, {0}, {Nx}); + ioFirstRank.DefineAttribute("comment", + "Written by rank 0"); + writerFirstRank = + ioFirstRank.Open("dataFirstRank.bp", mode, commFirstRank); + } + + adios2::Variable varXLastRank; + if (rank == nproc - 1) + { + varXLastRank = + ioLastRank.DefineVariable("x", {Nx}, {0}, {Nx}); + ioLastRank.DefineAttribute( + "comment", "Written by rank " + std::to_string(nproc - 1)); + writerLastRank = + ioLastRank.Open("dataLastRank.bp", mode, commLastRank); + } + + // the other ADIOS object (valid on rank 1..N, not on rank 0) + adios2::Variable varXAnother; + if (rank > 0) + { + varXAnother = ioAnother.DefineVariable( + "x", {static_cast(nproc - 1), Nx}); + ioAnother.DefineAttribute( + "comment", "Written by ranks 1.." + std::to_string(nproc - 1)); + writerAnother = ioAnother.Open("dataAnother.bp", mode); + } + + physicalStep = physicalStep_dt * startStep; + physicalTime = physicalTime_dt * startStep; + for (size_t step = startStep; step < startStep + nSteps; step++) + { + + for (size_t i = 0; i < Nx; i++) + { + x[i] = step * Nx * nproc * 1.0 + rank * Nx * 1.0 + (double)i; + } + for (size_t i = 0; i < Ny; i++) + { + y[i] = step * Ny * nproc * 1.0 + rank * Ny * 1.0 + (double)i; + } + for (size_t i = 0; i < Nz; i++) + { + z[i] = step * Nz * nproc * 1.0 + rank * Nz * 1.0 + (double)i; + } + + writerAll.BeginStep(); + writerAll.Put(varXAll, x.data()); + writerAll.Put(varYAll, y.data()); + writerAll.Put(varZAll, z.data()); + writerAll.Put(varStepAll, step); + writerAll.Put(varPhysStepAll, physicalStep); + writerAll.Put(varPhysTimeAll, physicalTime); + + writerAll.EndStep(); + + adios2::Engine writerStep = ioStep.Open( + "dataStep" + std::to_string(step) + ".bp", adios2::Mode::Write); + writerStep.Put(varZStep, z.data()); + writerStep.Put(varStepStep, step); + writerStep.Put(varPhysStepStep, physicalStep); + writerStep.Put(varPhysTimeStep, physicalTime); + writerStep.Close(); + + if (rank > 0) + { + writerAnother.BeginStep(); + varXAnother.SetSelection(adios2::Box( + {static_cast(rank - 1), 0}, + {1, static_cast(Nx)})); + writerAnother.Put(varXAnother, x.data()); + writerAnother.EndStep(); + } + + if (rank == 0) + { + writerFirstRank.BeginStep(); + writerFirstRank.Put(varXFirstRank, x.data()); + writerFirstRank.EndStep(); + } + + if (rank == nproc - 1) + { + writerLastRank.BeginStep(); + writerLastRank.Put(varXLastRank, x.data()); + writerLastRank.EndStep(); + } + + if (startStep > 0 && step == startStep) + { + adios2::Engine writerNew = + ioNew.Open("dataNew" + std::to_string(step) + ".bp", + adios2::Mode::Write); + writerNew.Put(varYNew, y.data()); + writerNew.Put(varStepNew, step); + writerNew.Put(varPhysStepNew, physicalStep); + writerNew.Put(varPhysTimeNew, physicalTime); + writerNew.Close(); + } + + std::this_thread::sleep_for(std::chrono::duration(1.0)); + + physicalStep += physicalStep_dt; + physicalTime += physicalTime_dt; + } + + writerAll.Close(); + if (rank == 0) + { + writerFirstRank.Close(); + } + if (rank == nproc - 1) + { + writerLastRank.Close(); + } + if (rank > 0) + { + writerAnother.Close(); + } + } + + // Final: to test creating yet another ADIOS object later + { + if (rank == 1) + { + adios2::ADIOS adiosFinal; + adios2::IO ioFinal = adiosFinal.DeclareIO("OutputFinal"); + ioFinal.SetEngine("BP5"); + adios2::Mode mode = adios2::Mode::Write; + + adios2::Variable varXFinal = + ioFinal.DefineVariable("x_on_rank_1", {Nx}, {0}, {Nx}); + adios2::Variable varStepFinal = + ioFinal.DefineVariable("AdiosStep"); + adios2::Variable varPhysStepFinal = + ioFinal.DefineVariable("iteration"); + adios2::Variable varPhysTimeFinal = + ioFinal.DefineVariable("time"); + ioFinal.DefineAttribute( + "comment", "Written by rank 1 at end of run"); + + adios2::Engine writerFinal = ioFinal.Open("dataFinal.bp", mode); + writerFinal.Put(varXFinal, x.data()); + writerFinal.Put(varStepFinal, startStep + nSteps - 1); + writerFinal.Put(varPhysStepFinal, physicalStep); + writerFinal.Put(varPhysTimeFinal, physicalTime); + writerFinal.Close(); + } + } + + MPI_Finalize(); + + return 0; +} From b085e37c5b1d89e35687a7aeb358603cc96c9512 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Fri, 19 May 2023 09:11:58 -0400 Subject: [PATCH 04/58] convert info to json at Close() and write out. --- .../engine/campaign/CampaignManager.cpp | 52 +++++++++++++++++-- .../adios2/engine/campaign/CampaignManager.h | 4 ++ 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/source/adios2/engine/campaign/CampaignManager.cpp b/source/adios2/engine/campaign/CampaignManager.cpp index cd8b2f38c4..9de8028b5f 100644 --- a/source/adios2/engine/campaign/CampaignManager.cpp +++ b/source/adios2/engine/campaign/CampaignManager.cpp @@ -16,6 +16,8 @@ #include +#include + namespace adios2 { namespace core @@ -23,6 +25,42 @@ namespace core namespace engine { +static std::string CMapToJson(const CampaignMap &cmap, const int rank, + const std::string name) +{ + nlohmann::json j = nlohmann::json::array(); + std::cout << "Campaign Manager " << rank << " Close(" << name + << ")\nCampaign"; + for (auto &r : cmap) + { + nlohmann::json c = + nlohmann::json{{"name", r.first}, + {"varying_deltas", r.second.varying_deltas}, + {"delta_step", r.second.delta_step}, + {"delta_time", r.second.delta_time}, + {"steps", r.second.steps}, + {"times", r.second.times}}; + std::cout << "name = " << r.first << "\n"; + std::cout << " varying_deltas = " << r.second.varying_deltas << "\n"; + std::cout << " delta_step = " << r.second.delta_step << "\n"; + std::cout << " delta_time = " << r.second.delta_time << "\n"; + std::cout << " steps = {"; + for (const auto s : r.second.steps) + { + std::cout << s << " "; + } + std::cout << "}\n"; + std::cout << " times = {"; + for (const auto t : r.second.times) + { + std::cout << t << " "; + } + std::cout << "}\n"; + j.push_back(c); + } + return nlohmann::to_string(j); +} + CampaignManager::CampaignManager(adios2::helper::Comm &comm) { m_WriterRank = comm.Rank(); @@ -53,7 +91,7 @@ void CampaignManager::Open(const std::string &name) std::cout << "Campaign Manager " << m_WriterRank << " Open(" << name << ")\n"; } - m_Opened = true; + m_Name = name + "_" + std::to_string(m_WriterRank) + ".json"; } void CampaignManager::Record(const std::string &name, const size_t step, @@ -103,7 +141,7 @@ void CampaignManager::Record(const std::string &name, const size_t step, void CampaignManager::Close() { - if (m_Verbosity == 5) + /*if (m_Verbosity == 5) { std::cout << "Campaign Manager " << m_WriterRank << " Close()\nCampaign"; @@ -127,8 +165,16 @@ void CampaignManager::Close() } std::cout << "}\n"; } + }*/ + if (!cmap.empty()) + { + m_Output.open(m_Name, std::ofstream::out); + m_Opened = true; + m_Output << std::setw(4) << CMapToJson(cmap, m_WriterRank, m_Name) + << std::endl; + m_Output.close(); + m_Opened = false; } - m_Opened = false; } } // end namespace engine diff --git a/source/adios2/engine/campaign/CampaignManager.h b/source/adios2/engine/campaign/CampaignManager.h index 3b05eb6f6a..6b89354f89 100644 --- a/source/adios2/engine/campaign/CampaignManager.h +++ b/source/adios2/engine/campaign/CampaignManager.h @@ -19,6 +19,8 @@ #include "adios2/common/ADIOSConfig.h" #include "adios2/helper/adiosComm.h" +#include + namespace adios2 { namespace core @@ -40,9 +42,11 @@ class CampaignManager private: bool m_Opened = false; + std::string m_Name; int m_WriterRank; int m_Verbosity = 5; CampaignMap cmap; + std::ofstream m_Output; }; } // end namespace engine From 08a0715d24dc814fc2539987f630e0f7ce562934 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Mon, 22 May 2023 06:13:47 -0400 Subject: [PATCH 05/58] IO.cpp to create Campaign Reader --- source/adios2/core/IO.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/adios2/core/IO.cpp b/source/adios2/core/IO.cpp index 89764aa71b..2d55c99fbe 100644 --- a/source/adios2/core/IO.cpp +++ b/source/adios2/core/IO.cpp @@ -26,6 +26,7 @@ #include "adios2/engine/bp5/BP5Reader.h" #include "adios2/engine/bp5/BP5Writer.h" #endif +#include "adios2/engine/campaign/CampaignReader.h" #include "adios2/engine/inline/InlineReader.h" #include "adios2/engine/inline/InlineWriter.h" #include "adios2/engine/mhs/MhsReader.h" @@ -135,6 +136,9 @@ std::unordered_map Factory = { {IO::NoEngine("ERROR: nullcore engine does not support read mode"), IO::MakeEngine}}, {"plugin", {IO::MakeEngine, IO::MakeEngine}}, + {"campaign", + {IO::MakeEngine, + IO::NoEngine("ERROR: campaign engine does not support write mode")}}, }; // Synchronize access to the factory in case one thread is From fdfa1d6c9885e468a2f894c6022b47ab4fd95b6f Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Mon, 22 May 2023 08:28:57 -0400 Subject: [PATCH 06/58] add missing file --- source/adios2/engine/campaign/CampaignData.h | 50 ++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 source/adios2/engine/campaign/CampaignData.h diff --git a/source/adios2/engine/campaign/CampaignData.h b/source/adios2/engine/campaign/CampaignData.h new file mode 100644 index 0000000000..cb2f05ae12 --- /dev/null +++ b/source/adios2/engine/campaign/CampaignData.h @@ -0,0 +1,50 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * CampaignData.h + * Campaign data struct + * + * Created on: May 16, 2023 + * Author: Norbert Podhorszki pnorbert@ornl.gov + */ + +#ifndef ADIOS2_ENGINE_CAMPAIGNDATA_H_ +#define ADIOS2_ENGINE_CAMPAIGNDATA_H_ + +#include +#include +#include + +namespace adios2 +{ +namespace core +{ +namespace engine +{ + +struct CampaignRecord +{ + std::vector steps; + size_t delta_step; + std::vector times; + double delta_time; + bool varying_deltas; + + CampaignRecord() : delta_step(0), delta_time(0.0), varying_deltas(false){}; + + CampaignRecord(size_t step, double time) + : delta_step(0), delta_time(0.0), varying_deltas(false) + { + steps.push_back(step); + times.push_back(time); + }; +}; + +using CampaignMap = std::unordered_map; + +} // end namespace engine +} // end namespace core +} // end namespace adios2 + +#endif /* ADIOS2_ENGINE_CAMPAIGDATA_H_ */ From f45262d3a63b37f816904f073066ad5bd1c31272 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Mon, 22 May 2023 08:29:28 -0400 Subject: [PATCH 07/58] add explicit map for which engine supports ReadRandomAccess mode --- source/adios2/core/IO.cpp | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/source/adios2/core/IO.cpp b/source/adios2/core/IO.cpp index 2d55c99fbe..a4470d5ca2 100644 --- a/source/adios2/core/IO.cpp +++ b/source/adios2/core/IO.cpp @@ -141,6 +141,15 @@ std::unordered_map Factory = { IO::NoEngine("ERROR: campaign engine does not support write mode")}}, }; +const std::unordered_map ReadRandomAccess_Supported = { + {"bp3", false}, {"bp4", false}, {"bp5", true}, + {"dataman", false}, {"ssc", false}, {"mhs", false}, + {"sst", false}, {"daos", false}, {"effis", false}, + {"dataspaces", false}, {"hdf5", false}, {"skeleton", true}, + {"inline", false}, {"null", true}, {"nullcore", true}, + {"plugin", false}, {"campaign", true}, +}; + // Synchronize access to the factory in case one thread is // looking up while another registers additional entries. std::mutex FactoryMutex; @@ -602,11 +611,6 @@ Engine &IO::Open(const std::string &name, const Mode mode, helper::Comm comm) // << std::endl; } - if ((engineTypeLC != "bp5") && (mode_to_use == Mode::ReadRandomAccess)) - { - // only BP5 special-cases file-reader random access mode - mode_to_use = Mode::Read; - } // For the inline engine, there must be exactly 1 reader, and exactly 1 // writer. if (engineTypeLC == "inline") @@ -649,6 +653,19 @@ Engine &IO::Open(const std::string &name, const Mode mode, helper::Comm comm) } } + if (mode_to_use == Mode::ReadRandomAccess) + { + // older engines don't know about ReadRandomAccess Mode + auto it = ReadRandomAccess_Supported.find(engineTypeLC); + if (it != ReadRandomAccess_Supported.end()) + { + if (!it->second) + { + mode_to_use = Mode::Read; + } + } + } + auto f = FactoryLookup(engineTypeLC); if (f != Factory.end()) { From 957ed2ff61e9f502ed60df7947029a645fa1e88f Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Mon, 22 May 2023 08:30:37 -0400 Subject: [PATCH 08/58] save work: campaing reader copies variables from subengines, bpls is stuck on empty m_AvailableStepBlockIndexOffsets --- examples/campaign/campaign_write.cpp | 4 +- source/adios2/engine/bp3/BP3Writer.cpp | 11 +++ source/adios2/engine/bp3/BP3Writer.h | 3 + source/adios2/engine/bp4/BP4Writer.cpp | 11 +++ source/adios2/engine/bp4/BP4Writer.h | 2 + .../adios2/engine/campaign/CampaignReader.cpp | 79 ++++++++++++++++--- .../adios2/engine/campaign/CampaignReader.h | 10 +++ .../adios2/engine/campaign/CampaignReader.tcc | 19 +++++ source/adios2/engine/hdf5/HDF5WriterP.cpp | 4 + 9 files changed, 132 insertions(+), 11 deletions(-) diff --git a/examples/campaign/campaign_write.cpp b/examples/campaign/campaign_write.cpp index 6d919f09e6..35e565fbe8 100644 --- a/examples/campaign/campaign_write.cpp +++ b/examples/campaign/campaign_write.cpp @@ -158,7 +158,7 @@ int main(int argc, char *argv[]) if (rank == 0) { ioFirstRank = adiosAll.DeclareIO("OutputFirst"); - ioFirstRank.SetEngine("BP5"); + ioFirstRank.SetEngine("BP4"); } if (rank == nproc - 1) { @@ -166,7 +166,7 @@ int main(int argc, char *argv[]) ioLastRank.SetEngine("BP5"); } ioStep = adiosAll.DeclareIO("OutputStep"); - ioStep.SetEngine("BP4"); + ioStep.SetEngine("BP3"); ioNew = adiosAll.DeclareIO("OutputNew"); #ifdef ADIOS2_USE_HDF5 diff --git a/source/adios2/engine/bp3/BP3Writer.cpp b/source/adios2/engine/bp3/BP3Writer.cpp index 7537de2796..ccb73b117f 100644 --- a/source/adios2/engine/bp3/BP3Writer.cpp +++ b/source/adios2/engine/bp3/BP3Writer.cpp @@ -50,6 +50,7 @@ StepStatus BP3Writer::BeginStep(StepMode mode, const float timeoutSeconds) m_BP3Serializer.m_DeferredVariables.clear(); m_BP3Serializer.m_DeferredVariablesDataSize = 0; m_IO.m_ReadStreaming = false; + m_DidBeginStep = true; return StepStatus::OK; } @@ -106,6 +107,11 @@ void BP3Writer::EndStep() { Flush(); } + + if (m_BP3Serializer.m_RankMPI == 0) + { + m_IO.m_ADIOS.RecordOutputStep(m_Name, UnknownStep, UnknownTime); + } } void BP3Writer::Flush(const int transportIndex) @@ -266,6 +272,11 @@ void BP3Writer::DoClose(const int transportIndex) } m_BP3Serializer.DeleteBuffers(); + + if (!m_DidBeginStep && m_BP3Serializer.m_RankMPI == 0) + { + m_IO.m_ADIOS.RecordOutputStep(m_Name, UnknownStep, UnknownTime); + } } void BP3Writer::WriteProfilingJSONFile() diff --git a/source/adios2/engine/bp3/BP3Writer.h b/source/adios2/engine/bp3/BP3Writer.h index 4b9dc30585..609ca9a6ee 100644 --- a/source/adios2/engine/bp3/BP3Writer.h +++ b/source/adios2/engine/bp3/BP3Writer.h @@ -59,6 +59,9 @@ class BP3Writer : public core::Engine /** Manages the optional collective metadata files */ transportman::TransportMan m_FileMetadataManager; + // true if BeginStep was ever called + bool m_DidBeginStep = false; + void Init() final; /** Parses parameters from IO SetParameters */ diff --git a/source/adios2/engine/bp4/BP4Writer.cpp b/source/adios2/engine/bp4/BP4Writer.cpp index 7419e4dd95..18d7ac4027 100644 --- a/source/adios2/engine/bp4/BP4Writer.cpp +++ b/source/adios2/engine/bp4/BP4Writer.cpp @@ -62,6 +62,7 @@ StepStatus BP4Writer::BeginStep(StepMode mode, const float timeoutSeconds) m_BP4Serializer.m_DeferredVariables.clear(); m_BP4Serializer.m_DeferredVariablesDataSize = 0; m_IO.m_ReadStreaming = false; + m_DidBeginStep = true; return StepStatus::OK; } @@ -124,6 +125,11 @@ void BP4Writer::EndStep() { Flush(); } + + if (m_BP4Serializer.m_RankMPI == 0) + { + m_IO.m_ADIOS.RecordOutputStep(m_Name, UnknownStep, UnknownTime); + } } void BP4Writer::Flush(const int transportIndex) @@ -508,6 +514,11 @@ void BP4Writer::DoClose(const int transportIndex) /* Signal the BB thread that no more work is coming */ m_FileDrainer.Finish(); } + + if (!m_DidBeginStep && m_BP4Serializer.m_RankMPI == 0) + { + m_IO.m_ADIOS.RecordOutputStep(m_Name, UnknownStep, UnknownTime); + } // m_BP4Serializer.DeleteBuffers(); } diff --git a/source/adios2/engine/bp4/BP4Writer.h b/source/adios2/engine/bp4/BP4Writer.h index 69e9570b52..f48cf5bbee 100644 --- a/source/adios2/engine/bp4/BP4Writer.h +++ b/source/adios2/engine/bp4/BP4Writer.h @@ -90,6 +90,8 @@ class BP4Writer : public core::Engine std::vector m_ActiveFlagFileNames; int m_Verbosity = 0; + // true if BeginStep was ever called + bool m_DidBeginStep = false; void Init() final; diff --git a/source/adios2/engine/campaign/CampaignReader.cpp b/source/adios2/engine/campaign/CampaignReader.cpp index c8dae82793..985c1b6b55 100644 --- a/source/adios2/engine/campaign/CampaignReader.cpp +++ b/source/adios2/engine/campaign/CampaignReader.cpp @@ -13,8 +13,11 @@ #include "adios2/helper/adiosFunctions.h" // CSVToVector +#include #include +#include + namespace adios2 { namespace core @@ -30,7 +33,7 @@ CampaignReader::CampaignReader(IO &io, const std::string &name, const Mode mode, Init(); if (m_Verbosity == 5) { - std::cout << "Skeleton Reader " << m_ReaderRank << " Open(" << m_Name + std::cout << "Campaign Reader " << m_ReaderRank << " Open(" << m_Name << ") in constructor." << std::endl; } m_IsOpen = true; @@ -38,10 +41,10 @@ CampaignReader::CampaignReader(IO &io, const std::string &name, const Mode mode, CampaignReader::~CampaignReader() { - /* m_Skeleton deconstructor does close and finalize */ + /* CampaignReader destructor does close and finalize */ if (m_Verbosity == 5) { - std::cout << "Skeleton Reader " << m_ReaderRank << " deconstructor on " + std::cout << "Campaign Reader " << m_ReaderRank << " deconstructor on " << m_Name << "\n"; } if (m_IsOpen) @@ -60,7 +63,7 @@ StepStatus CampaignReader::BeginStep(const StepMode mode, if (m_Verbosity == 5) { - std::cout << "Skeleton Reader " << m_ReaderRank + std::cout << "Campaign Reader " << m_ReaderRank << " BeginStep() new step " << m_CurrentStep << "\n"; } @@ -69,7 +72,7 @@ StepStatus CampaignReader::BeginStep(const StepMode mode, // we return EndOfStream to the reader application if (m_CurrentStep == 2) { - std::cout << "Skeleton Reader " << m_ReaderRank + std::cout << "Campaign Reader " << m_ReaderRank << " forcefully returns End of Stream at this step\n"; return StepStatus::EndOfStream; @@ -87,7 +90,7 @@ void CampaignReader::PerformGets() { if (m_Verbosity == 5) { - std::cout << "Skeleton Reader " << m_ReaderRank + std::cout << "Campaign Reader " << m_ReaderRank << " PerformGets()\n"; } m_NeedPerformGets = false; @@ -106,7 +109,7 @@ void CampaignReader::EndStep() if (m_Verbosity == 5) { - std::cout << "Skeleton Reader " << m_ReaderRank << " EndStep()\n"; + std::cout << "Campaign Reader " << m_ReaderRank << " EndStep()\n"; } } @@ -156,16 +159,74 @@ void CampaignReader::InitParameters() void CampaignReader::InitTransports() { - // Nothing to process from m_IO.m_TransportsParameters + std::string cs = m_Comm.BroadcastFile(m_Name, "broadcast campaign file"); + nlohmann::json js = nlohmann::json::parse(cs); + std::cout << "JSON rank " << m_ReaderRank << ": " << js.size() << std::endl; + int i = 0; + for (auto &jf : js) + { + std::cout << jf << std::endl; + adios2::core::IO &io = + m_IO.m_ADIOS.DeclareIO("CampaignReader" + std::to_string(i)); + adios2::core::Engine &e = + io.Open(jf["name"], m_OpenMode, m_Comm.Duplicate()); + + m_IOs.push_back(&io); + m_Engines.push_back(&e); + + auto vmap = io.GetAvailableVariables(); + auto amap = io.GetAvailableAttributes(); + + for (auto &vr : vmap) + { + auto vname = vr.first; + std::string fname = jf["name"]; + std::string newname = fname + "/" + vname; + + const DataType type = io.InquireVariableType(vname); + + if (type == DataType::Struct) + { + } +#define declare_type(T) \ + else if (type == helper::GetDataType()) \ + { \ + Variable *vi = io.InquireVariable(vname); \ + DuplicateVariable(vi, m_IO, newname); \ + } + /* + v.m_AvailableStepsCount = vi->GetAvailableStepsCount(); \ + v.m_AvailableStepsStart = vi->GetAvailableStepsStart(); \ + v.m_ShapeID = vi->m_ShapeID; \ + v.m_SingleValue = vi->m_SingleValue; \ + v.m_ReadAsJoined = vi->m_ReadAsJoined; \ + v.m_ReadAsLocalValue = vi->m_ReadAsLocalValue; \ + v.m_RandomAccess = vi->m_RandomAccess; \ + v.m_MemSpace = vi->m_MemSpace; \ + v.m_JoinedDimPos = vi->m_JoinedDimPos; \ + v.m_AvailableStepBlockIndexOffsets = \ + vi->m_AvailableStepBlockIndexOffsets; \ + v.m_AvailableShapes = vi->m_AvailableShapes; \ + */ + ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + } + + ++i; + } } void CampaignReader::DoClose(const int transportIndex) { if (m_Verbosity == 5) { - std::cout << "Skeleton Reader " << m_ReaderRank << " Close(" << m_Name + std::cout << "Campaign Reader " << m_ReaderRank << " Close(" << m_Name << ")\n"; } + for (auto ep : m_Engines) + { + ep->Close(); + } } } // end namespace engine diff --git a/source/adios2/engine/campaign/CampaignReader.h b/source/adios2/engine/campaign/CampaignReader.h index 7f0078d392..0267d73e29 100644 --- a/source/adios2/engine/campaign/CampaignReader.h +++ b/source/adios2/engine/campaign/CampaignReader.h @@ -58,6 +58,9 @@ class CampaignReader : public Engine // EndStep must call PerformGets if necessary bool m_NeedPerformGets = false; + std::vector m_IOs; + std::vector m_Engines; + void Init() final; ///< called from constructor, gets the selected Skeleton /// transport method from settings void InitParameters() final; @@ -82,6 +85,13 @@ class CampaignReader : public Engine template void GetDeferredCommon(Variable &variable, T *data); + + /** + * Create a new variable with name `name` in `io` + * based on an existing variable. + */ + template + void DuplicateVariable(Variable *variable, IO &io, std::string &name); }; } // end namespace engine diff --git a/source/adios2/engine/campaign/CampaignReader.tcc b/source/adios2/engine/campaign/CampaignReader.tcc index d4d3252281..135b9a61fe 100644 --- a/source/adios2/engine/campaign/CampaignReader.tcc +++ b/source/adios2/engine/campaign/CampaignReader.tcc @@ -22,6 +22,25 @@ namespace core namespace engine { +template +inline void CampaignReader::DuplicateVariable(Variable *variable, IO &io, + std::string &name) +{ + auto &v = io.DefineVariable(name, variable->Shape()); + v.m_AvailableStepsCount = variable->GetAvailableStepsCount(); + v.m_AvailableStepsStart = variable->GetAvailableStepsStart(); + v.m_ShapeID = variable->m_ShapeID; + v.m_SingleValue = variable->m_SingleValue; + v.m_ReadAsJoined = variable->m_ReadAsJoined; + v.m_ReadAsLocalValue = variable->m_ReadAsLocalValue; + v.m_RandomAccess = variable->m_RandomAccess; + v.m_MemSpace = variable->m_MemSpace; + v.m_JoinedDimPos = variable->m_JoinedDimPos; + v.m_AvailableStepBlockIndexOffsets = + variable->m_AvailableStepBlockIndexOffsets; + v.m_AvailableShapes = variable->m_AvailableShapes; +} + template <> inline void CampaignReader::GetSyncCommon(Variable &variable, std::string *data) diff --git a/source/adios2/engine/hdf5/HDF5WriterP.cpp b/source/adios2/engine/hdf5/HDF5WriterP.cpp index 14c6503566..a938532fdb 100644 --- a/source/adios2/engine/hdf5/HDF5WriterP.cpp +++ b/source/adios2/engine/hdf5/HDF5WriterP.cpp @@ -46,6 +46,10 @@ void HDF5WriterP::EndStep() m_H5File.CleanUpNullVars(m_IO); m_H5File.Advance(); m_H5File.WriteAttrFromIO(m_IO); + if (!comm.Rank()) + { + m_IO.m_ADIOS.RecordOutputStep(m_Name, UnknownStep, UnknownTime); + } } void HDF5WriterP::PerformPuts() {} From 010beb7798d4e0777900f534bc6214506db227a6 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 23 May 2023 09:06:43 -0400 Subject: [PATCH 09/58] Maintain a map for each variable (original variable, engine and IO) so that we can use the actual engine and IO functions. bpls works properly showing dimensions now. Min/max is missing. Reading is not supported. BeginStep/EndStep not supported. --- source/adios2/common/ADIOSTypes.h | 4 +- source/adios2/engine/bp4/BP4Reader.cpp | 2 +- .../adios2/engine/campaign/CampaignReader.cpp | 89 +++++++++++++++---- .../adios2/engine/campaign/CampaignReader.h | 35 +++++++- .../adios2/engine/campaign/CampaignReader.tcc | 10 ++- source/utils/bpls/bpls.cpp | 34 +++---- 6 files changed, 134 insertions(+), 40 deletions(-) diff --git a/source/adios2/common/ADIOSTypes.h b/source/adios2/common/ADIOSTypes.h index 3b4a52b8d7..93f8b70519 100644 --- a/source/adios2/common/ADIOSTypes.h +++ b/source/adios2/common/ADIOSTypes.h @@ -221,11 +221,11 @@ struct MinVarInfo size_t Step; bool WasLocalValue; // writer: localValue -> reader: 1D global array int Dims; - size_t *Shape; + const size_t *Shape; bool IsValue = false; bool IsReverseDims = false; std::vector BlocksInfo; - MinVarInfo(int D, size_t *S) + MinVarInfo(const int D, const size_t *S) : Dims(D), Shape(S), IsValue(false), IsReverseDims(false), BlocksInfo({}) { } diff --git a/source/adios2/engine/bp4/BP4Reader.cpp b/source/adios2/engine/bp4/BP4Reader.cpp index 6f7cf42ed2..a7cf1f1c59 100644 --- a/source/adios2/engine/bp4/BP4Reader.cpp +++ b/source/adios2/engine/bp4/BP4Reader.cpp @@ -801,7 +801,7 @@ void BP4Reader::DoClose(const int transportIndex) std::vector::BPInfo>> \ BP4Reader::DoAllRelativeStepsBlocksInfo(const Variable &variable) const \ { \ - PERFSTUBS_SCOPED_TIMER("BP3Reader::AllRelativeStepsBlocksInfo"); \ + PERFSTUBS_SCOPED_TIMER("BP4Reader::AllRelativeStepsBlocksInfo"); \ return m_BP4Deserializer.AllRelativeStepsBlocksInfo(variable); \ } \ \ diff --git a/source/adios2/engine/campaign/CampaignReader.cpp b/source/adios2/engine/campaign/CampaignReader.cpp index 985c1b6b55..ecf4819faa 100644 --- a/source/adios2/engine/campaign/CampaignReader.cpp +++ b/source/adios2/engine/campaign/CampaignReader.cpp @@ -12,6 +12,7 @@ #include "CampaignReader.tcc" #include "adios2/helper/adiosFunctions.h" // CSVToVector +#include #include #include @@ -176,6 +177,8 @@ void CampaignReader::InitTransports() auto vmap = io.GetAvailableVariables(); auto amap = io.GetAvailableAttributes(); + VarInternalInfo internalInfo(nullptr, m_IOs.size() - 1, + m_Engines.size() - 1); for (auto &vr : vmap) { @@ -192,22 +195,10 @@ void CampaignReader::InitTransports() else if (type == helper::GetDataType()) \ { \ Variable *vi = io.InquireVariable(vname); \ - DuplicateVariable(vi, m_IO, newname); \ - } - /* - v.m_AvailableStepsCount = vi->GetAvailableStepsCount(); \ - v.m_AvailableStepsStart = vi->GetAvailableStepsStart(); \ - v.m_ShapeID = vi->m_ShapeID; \ - v.m_SingleValue = vi->m_SingleValue; \ - v.m_ReadAsJoined = vi->m_ReadAsJoined; \ - v.m_ReadAsLocalValue = vi->m_ReadAsLocalValue; \ - v.m_RandomAccess = vi->m_RandomAccess; \ - v.m_MemSpace = vi->m_MemSpace; \ - v.m_JoinedDimPos = vi->m_JoinedDimPos; \ - v.m_AvailableStepBlockIndexOffsets = \ - vi->m_AvailableStepBlockIndexOffsets; \ - v.m_AvailableShapes = vi->m_AvailableShapes; \ - */ + Variable v = \ + DuplicateVariable(vi, m_IO, newname, &e, internalInfo); \ + } + ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) #undef declare_type } @@ -229,6 +220,72 @@ void CampaignReader::DoClose(const int transportIndex) } } +// Remove the engine name from the var name, which must be of pattern +// / +/*static std::string RemoveEngineName(const std::string &varName, + const std::string &engineName) +{ + auto le = engineName.size() + 1; + auto v = varName.substr(le); + return v; +}*/ + +MinVarInfo *CampaignReader::MinBlocksInfo(const VariableBase &Var, + size_t Step) const +{ + auto it = m_VarInternalInfo.find(Var.m_Name); + if (it != m_VarInternalInfo.end()) + { + VariableBase *vb = + reinterpret_cast(it->second.originalVar); + Engine *e = m_Engines[it->second.engineIdx]; + MinVarInfo *MV = e->MinBlocksInfo(*vb, Step); + if (MV) + { + return MV; + } + } + return nullptr; +} + +#define declare_type(T) \ + std::map::BPInfo>> \ + CampaignReader::DoAllStepsBlocksInfo(const Variable &variable) const \ + { \ + PERFSTUBS_SCOPED_TIMER("CampaignReader::AllStepsBlocksInfo"); \ + auto it = m_VarInternalInfo.find(variable.m_Name); \ + Variable *v = \ + reinterpret_cast *>(it->second.originalVar); \ + Engine *e = m_Engines[it->second.engineIdx]; \ + return e->AllStepsBlocksInfo(*v); \ + } \ + \ + std::vector::BPInfo>> \ + CampaignReader::DoAllRelativeStepsBlocksInfo(const Variable &variable) \ + const \ + { \ + PERFSTUBS_SCOPED_TIMER("CampaignReader::AllRelativeStepsBlocksInfo"); \ + auto it = m_VarInternalInfo.find(variable.m_Name); \ + Variable *v = \ + reinterpret_cast *>(it->second.originalVar); \ + Engine *e = m_Engines[it->second.engineIdx]; \ + return e->AllRelativeStepsBlocksInfo(variable); \ + } \ + \ + std::vector::BPInfo> CampaignReader::DoBlocksInfo( \ + const Variable &variable, const size_t step) const \ + { \ + PERFSTUBS_SCOPED_TIMER("CampaignReader::BlocksInfo"); \ + auto it = m_VarInternalInfo.find(variable.m_Name); \ + Variable *v = \ + reinterpret_cast *>(it->second.originalVar); \ + Engine *e = m_Engines[it->second.engineIdx]; \ + return e->BlocksInfo(variable, step); \ + } + +ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + } // end namespace engine } // end namespace core } // end namespace adios2 diff --git a/source/adios2/engine/campaign/CampaignReader.h b/source/adios2/engine/campaign/CampaignReader.h index 0267d73e29..6a123ef766 100644 --- a/source/adios2/engine/campaign/CampaignReader.h +++ b/source/adios2/engine/campaign/CampaignReader.h @@ -48,12 +48,14 @@ class CampaignReader : public Engine size_t CurrentStep() const final; void EndStep() final; + MinVarInfo *MinBlocksInfo(const VariableBase &, const size_t Step) const; + private: int m_Verbosity = 0; int m_ReaderRank; // my rank in the readers' comm - // step info should be received from the writer side in BeginStep() - int m_CurrentStep = -1; + int m_CurrentStep = 0; + bool m_FirstStep = true; // EndStep must call PerformGets if necessary bool m_NeedPerformGets = false; @@ -61,6 +63,18 @@ class CampaignReader : public Engine std::vector m_IOs; std::vector m_Engines; + struct VarInternalInfo + { + void *originalVar; // Variable in the actual IO + size_t ioIdx; // actual IO in m_IOs + size_t engineIdx; // actual engine in m_Engines + VarInternalInfo(void *p, size_t i, size_t e) + : originalVar(p), ioIdx(i), engineIdx(e) + { + } + }; + std::unordered_map m_VarInternalInfo; + void Init() final; ///< called from constructor, gets the selected Skeleton /// transport method from settings void InitParameters() final; @@ -74,6 +88,19 @@ class CampaignReader : public Engine void DoClose(const int transportIndex = -1); +#define declare_type(T) \ + std::map::BPInfo>> \ + DoAllStepsBlocksInfo(const Variable &variable) const final; \ + \ + std::vector::BPInfo>> \ + DoAllRelativeStepsBlocksInfo(const Variable &) const final; \ + \ + std::vector::BPInfo> DoBlocksInfo( \ + const Variable &variable, const size_t step) const final; + + ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + /** * Called if destructor is called on an open engine. Should warn or take * any non-complex measure that might help recover. @@ -91,7 +118,9 @@ class CampaignReader : public Engine * based on an existing variable. */ template - void DuplicateVariable(Variable *variable, IO &io, std::string &name); + Variable DuplicateVariable(Variable *variable, IO &io, + std::string &name, Engine *e, + VarInternalInfo &vii); }; } // end namespace engine diff --git a/source/adios2/engine/campaign/CampaignReader.tcc b/source/adios2/engine/campaign/CampaignReader.tcc index 135b9a61fe..7e96950f00 100644 --- a/source/adios2/engine/campaign/CampaignReader.tcc +++ b/source/adios2/engine/campaign/CampaignReader.tcc @@ -23,8 +23,10 @@ namespace engine { template -inline void CampaignReader::DuplicateVariable(Variable *variable, IO &io, - std::string &name) +inline Variable CampaignReader::DuplicateVariable(Variable *variable, + IO &io, std::string &name, + Engine *e, + VarInternalInfo &vii) { auto &v = io.DefineVariable(name, variable->Shape()); v.m_AvailableStepsCount = variable->GetAvailableStepsCount(); @@ -39,6 +41,10 @@ inline void CampaignReader::DuplicateVariable(Variable *variable, IO &io, v.m_AvailableStepBlockIndexOffsets = variable->m_AvailableStepBlockIndexOffsets; v.m_AvailableShapes = variable->m_AvailableShapes; + v.m_Engine = e; + vii.originalVar = static_cast(variable); + m_VarInternalInfo.emplace(name, vii); + return v; } template <> diff --git a/source/utils/bpls/bpls.cpp b/source/utils/bpls/bpls.cpp index 462598d90a..5cd6de8002 100644 --- a/source/utils/bpls/bpls.cpp +++ b/source/utils/bpls/bpls.cpp @@ -3113,29 +3113,31 @@ Dims get_global_array_signature(core::Engine *fp, core::IO *io, core::Variable> &indices = variable->m_AvailableStepBlockIndexOffsets; auto itStep = indices.begin(); - - for (size_t step = 0; step < nsteps; step++) + if (itStep != indices.end()) { - const size_t absstep = itStep->first; - Dims d = variable->Shape(absstep - 1); - if (d.empty()) + for (size_t step = 0; step < nsteps; step++) { - continue; - } - - for (size_t k = 0; k < ndim; k++) - { - if (firstStep) + const size_t absstep = itStep->first; + Dims d = variable->Shape(absstep - 1); + if (d.empty()) { - dims[k] = d[k]; + continue; } - else if (dims[k] != d[k]) + + for (size_t k = 0; k < ndim; k++) { - dims[k] = 0; + if (firstStep) + { + dims[k] = d[k]; + } + else if (dims[k] != d[k]) + { + dims[k] = 0; + } } + firstStep = false; + ++itStep; } - firstStep = false; - ++itStep; } } } From bb4fed9efe255d3cff3e1cac1041470b72412c5f Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Wed, 24 May 2023 07:37:34 -0400 Subject: [PATCH 10/58] Support Read(sync) and block reading. --- .../adios2/engine/campaign/CampaignReader.cpp | 41 +++++++------ .../adios2/engine/campaign/CampaignReader.h | 15 ++--- .../adios2/engine/campaign/CampaignReader.tcc | 57 ++++++++----------- 3 files changed, 56 insertions(+), 57 deletions(-) diff --git a/source/adios2/engine/campaign/CampaignReader.cpp b/source/adios2/engine/campaign/CampaignReader.cpp index ecf4819faa..e7e2896f2d 100644 --- a/source/adios2/engine/campaign/CampaignReader.cpp +++ b/source/adios2/engine/campaign/CampaignReader.cpp @@ -116,19 +116,6 @@ void CampaignReader::EndStep() // PRIVATE -#define declare_type(T) \ - void CampaignReader::DoGetSync(Variable &variable, T *data) \ - { \ - GetSyncCommon(variable, data); \ - } \ - void CampaignReader::DoGetDeferred(Variable &variable, T *data) \ - { \ - GetDeferredCommon(variable, data); \ - } - -ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) -#undef declare_type - void CampaignReader::Init() { InitParameters(); @@ -195,8 +182,7 @@ void CampaignReader::InitTransports() else if (type == helper::GetDataType()) \ { \ Variable *vi = io.InquireVariable(vname); \ - Variable v = \ - DuplicateVariable(vi, m_IO, newname, &e, internalInfo); \ + Variable v = DuplicateVariable(vi, m_IO, newname, internalInfo); \ } ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) @@ -249,6 +235,22 @@ MinVarInfo *CampaignReader::MinBlocksInfo(const VariableBase &Var, } #define declare_type(T) \ + void CampaignReader::DoGetSync(Variable &variable, T *data) \ + { \ + PERFSTUBS_SCOPED_TIMER("CampaignReader::Get"); \ + auto p = TranslateToActualVariable(variable); \ + p.second->Get(*p.first, data, adios2::Mode::Sync); \ + } \ + void CampaignReader::DoGetDeferred(Variable &variable, T *data) \ + { \ + PERFSTUBS_SCOPED_TIMER("CampaignReader::Get"); \ + auto it = m_VarInternalInfo.find(variable.m_Name); \ + Variable *v = \ + reinterpret_cast *>(it->second.originalVar); \ + Engine *e = m_Engines[it->second.engineIdx]; \ + e->Get(*v, data, adios2::Mode::Deferred); \ + } \ + \ std::map::BPInfo>> \ CampaignReader::DoAllStepsBlocksInfo(const Variable &variable) const \ { \ @@ -269,7 +271,7 @@ MinVarInfo *CampaignReader::MinBlocksInfo(const VariableBase &Var, Variable *v = \ reinterpret_cast *>(it->second.originalVar); \ Engine *e = m_Engines[it->second.engineIdx]; \ - return e->AllRelativeStepsBlocksInfo(variable); \ + return e->AllRelativeStepsBlocksInfo(*v); \ } \ \ std::vector::BPInfo> CampaignReader::DoBlocksInfo( \ @@ -280,12 +282,17 @@ MinVarInfo *CampaignReader::MinBlocksInfo(const VariableBase &Var, Variable *v = \ reinterpret_cast *>(it->second.originalVar); \ Engine *e = m_Engines[it->second.engineIdx]; \ - return e->BlocksInfo(variable, step); \ + return e->BlocksInfo(*v, step); \ } ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) #undef declare_type +#define declare_type(T) + +ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + } // end namespace engine } // end namespace core } // end namespace adios2 diff --git a/source/adios2/engine/campaign/CampaignReader.h b/source/adios2/engine/campaign/CampaignReader.h index 6a123ef766..681a755204 100644 --- a/source/adios2/engine/campaign/CampaignReader.h +++ b/source/adios2/engine/campaign/CampaignReader.h @@ -107,20 +107,21 @@ class CampaignReader : public Engine */ void DestructorClose(bool Verbose) noexcept final{}; + /** + * Create a new variable with name `name` in `io` + * based on an existing variable. + */ template - void GetSyncCommon(Variable &variable, T *data); - - template - void GetDeferredCommon(Variable &variable, T *data); + Variable DuplicateVariable(Variable *variable, IO &io, + std::string &name, VarInternalInfo &vii); /** * Create a new variable with name `name` in `io` * based on an existing variable. */ template - Variable DuplicateVariable(Variable *variable, IO &io, - std::string &name, Engine *e, - VarInternalInfo &vii); + std::pair *, Engine *> + TranslateToActualVariable(Variable &variable); }; } // end namespace engine diff --git a/source/adios2/engine/campaign/CampaignReader.tcc b/source/adios2/engine/campaign/CampaignReader.tcc index 7e96950f00..a34029bee5 100644 --- a/source/adios2/engine/campaign/CampaignReader.tcc +++ b/source/adios2/engine/campaign/CampaignReader.tcc @@ -25,7 +25,6 @@ namespace engine template inline Variable CampaignReader::DuplicateVariable(Variable *variable, IO &io, std::string &name, - Engine *e, VarInternalInfo &vii) { auto &v = io.DefineVariable(name, variable->Shape()); @@ -41,45 +40,37 @@ inline Variable CampaignReader::DuplicateVariable(Variable *variable, v.m_AvailableStepBlockIndexOffsets = variable->m_AvailableStepBlockIndexOffsets; v.m_AvailableShapes = variable->m_AvailableShapes; - v.m_Engine = e; + v.m_Min = variable->m_Min; + v.m_Max = variable->m_Max; + v.m_Value = variable->m_Value; + v.m_StepsStart = variable->m_StepsStart; + v.m_StepsCount = variable->m_StepsCount; + v.m_Start = variable->m_Start; + v.m_Count = variable->m_Count; + + v.m_Engine = this; // Variable::Shape() uses this member to call engine vii.originalVar = static_cast(variable); m_VarInternalInfo.emplace(name, vii); return v; } -template <> -inline void CampaignReader::GetSyncCommon(Variable &variable, - std::string *data) -{ - variable.m_Data = data; - if (m_Verbosity == 5) - { - std::cout << "Skeleton Reader " << m_ReaderRank << " GetSync(" - << variable.m_Name << ")\n"; - } -} - -template -inline void CampaignReader::GetSyncCommon(Variable &variable, T *data) -{ - variable.m_Data = data; - if (m_Verbosity == 5) - { - std::cout << "Skeleton Reader " << m_ReaderRank << " GetSync(" - << variable.m_Name << ")\n"; - } -} - template -void CampaignReader::GetDeferredCommon(Variable &variable, T *data) +inline std::pair *, Engine *> +CampaignReader::TranslateToActualVariable(Variable &variable) { - // returns immediately - if (m_Verbosity == 5) - { - std::cout << "Skeleton Reader " << m_ReaderRank << " GetDeferred(" - << variable.m_Name << ")\n"; - } - m_NeedPerformGets = true; + auto it = m_VarInternalInfo.find(variable.m_Name); + Variable *v = reinterpret_cast *>(it->second.originalVar); + Engine *e = m_Engines[it->second.engineIdx]; + v->m_SelectionType = variable.m_SelectionType; + v->m_Start = variable.m_Start; + v->m_Count = variable.m_Count; + v->m_StepsStart = variable.m_StepsStart; + v->m_StepsCount = variable.m_StepsCount; + v->m_BlockID = variable.m_BlockID; + v->m_MemoryStart = variable.m_MemoryStart; + v->m_MemoryCount = variable.m_MemoryCount; + v->m_MemSpace = variable.m_MemSpace; + return std::make_pair(v, e); } } // end namespace engine From af3916e00fd92abd223bc6ca4824e4ad3606cbf2 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Mon, 5 Jun 2023 09:00:09 -0400 Subject: [PATCH 11/58] Save campaign data in adios-campaign/ --- source/adios2/core/ADIOS.cpp | 6 +----- source/adios2/engine/campaign/CampaignManager.cpp | 6 ++++-- source/adios2/engine/campaign/CampaignManager.h | 1 + 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/source/adios2/core/ADIOS.cpp b/source/adios2/core/ADIOS.cpp index 9e075dda38..4e2cb7d46e 100644 --- a/source/adios2/core/ADIOS.cpp +++ b/source/adios2/core/ADIOS.cpp @@ -143,11 +143,7 @@ ADIOS::ADIOS(const std::string configFile, helper::Comm comm, const std::string #ifdef ADIOS2_HAVE_KOKKOS m_GlobalServices.Init_Kokkos_API(); #endif - std::string campaignName = "campaign"; - if (adios_count > 1) - { - campaignName = "campaign" + std::to_string(adios_count); - } + std::string campaignName = "campaign_" + std::to_string(adios_count); m_CampaignManager.Open(campaignName); } diff --git a/source/adios2/engine/campaign/CampaignManager.cpp b/source/adios2/engine/campaign/CampaignManager.cpp index 9de8028b5f..e0c799a2e6 100644 --- a/source/adios2/engine/campaign/CampaignManager.cpp +++ b/source/adios2/engine/campaign/CampaignManager.cpp @@ -69,6 +69,7 @@ CampaignManager::CampaignManager(adios2::helper::Comm &comm) std::cout << "Campaign Manager " << m_WriterRank << " constructor called" << std::endl; } + helper::CreateDirectory(m_CampaignDir); } CampaignManager::~CampaignManager() @@ -86,12 +87,13 @@ CampaignManager::~CampaignManager() void CampaignManager::Open(const std::string &name) { + m_Name = m_CampaignDir + "/" + name + "_" + std::to_string(m_WriterRank) + + ".json"; if (m_Verbosity == 5) { - std::cout << "Campaign Manager " << m_WriterRank << " Open(" << name + std::cout << "Campaign Manager " << m_WriterRank << " Open(" << m_Name << ")\n"; } - m_Name = name + "_" + std::to_string(m_WriterRank) + ".json"; } void CampaignManager::Record(const std::string &name, const size_t step, diff --git a/source/adios2/engine/campaign/CampaignManager.h b/source/adios2/engine/campaign/CampaignManager.h index 6b89354f89..51e30ef62c 100644 --- a/source/adios2/engine/campaign/CampaignManager.h +++ b/source/adios2/engine/campaign/CampaignManager.h @@ -47,6 +47,7 @@ class CampaignManager int m_Verbosity = 5; CampaignMap cmap; std::ofstream m_Output; + const std::string m_CampaignDir = "adios-campaign"; }; } // end namespace engine From 68c65d92df7450d4de7ce88537709e8b14437584 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 6 Jun 2023 10:45:54 -0400 Subject: [PATCH 12/58] added (beginning of) script to create campaing archive files --- source/utils/CMakeLists.txt | 1 + .../adios2_campaign_manager.py | 62 +++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100755 source/utils/adios_campaign_manager/adios2_campaign_manager.py diff --git a/source/utils/CMakeLists.txt b/source/utils/CMakeLists.txt index 37153406c0..832c96e172 100644 --- a/source/utils/CMakeLists.txt +++ b/source/utils/CMakeLists.txt @@ -74,6 +74,7 @@ endif() if(Python_Interpreter_FOUND) add_subdirectory(bp4dbg) add_subdirectory(bp5dbg) + add_subdirectory(adios_campaign_manager) install(PROGRAMS adios2_json_pp.py RENAME adios2_json_pp DESTINATION ${CMAKE_INSTALL_BINDIR} diff --git a/source/utils/adios_campaign_manager/adios2_campaign_manager.py b/source/utils/adios_campaign_manager/adios2_campaign_manager.py new file mode 100755 index 0000000000..96bfdc9791 --- /dev/null +++ b/source/utils/adios_campaign_manager/adios2_campaign_manager.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 + +import argparse +import glob +from os.path import basename, exists, isdir + +#from adios2.adios2_campaign_manager import * + + +def SetupArgs(): + parser = argparse.ArgumentParser() + parser.add_argument( + "FILE", help="Name of the input file (.bp, .bp/md.idx, " + + ".bp/md.0 or .bp/data.XXX)") + parser.add_argument("--verbose", "-v", + help="More verbosity", action="count") + parser.add_argument("--project", "-p", + help="Project name", + required=True) + parser.add_argument("--app", "-a", + help="Application name", + required=True) + parser.add_argument("--shot", "-s", + help="Shot name", + required=True) + parser.add_argument("--campaign_store", "-c", + help="Path to local campaign store", + required=True) + args = parser.parse_args() + + # default values + args.update = False + args.CampaignFileName = args.campaign_store+"/" + \ + args.project+"_"+args.app+"_"+args.shot+".acm" + args.LocalCampaignDir = "adios-campaign" + + # print("Verbosity = {0}".format(args.verbose)) + print(f"Campaign File Name = {args.CampaignFileName}") + return args + + +def CheckFileName(args): + if not exists(args.FILE): + print("ERROR: File " + args.FILE + " does not exist", flush=True) + exit(1) + + +def DumpIndexTableFile(args): + indexFileList = glob.glob(args.idxFileName) + if len(indexFileList) > 0: + DumpIndexTable(indexFileList[0]) + else: + print("There is no BP4 Index Table file as " + args.idxFileName) + + +if __name__ == "__main__": + + args = SetupArgs() + CheckFileName(args) + # print(args) + + DumpIndexTableFile(args) From 5885c484cd032ec774db1817e4696388472a68de Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Wed, 7 Jun 2023 19:06:10 -0400 Subject: [PATCH 13/58] Pack more information in campaign archive. --- .../adios2_campaign_manager.py | 174 ++++++++++++++++-- 1 file changed, 159 insertions(+), 15 deletions(-) diff --git a/source/utils/adios_campaign_manager/adios2_campaign_manager.py b/source/utils/adios_campaign_manager/adios2_campaign_manager.py index 96bfdc9791..8156adab51 100755 --- a/source/utils/adios_campaign_manager/adios2_campaign_manager.py +++ b/source/utils/adios_campaign_manager/adios2_campaign_manager.py @@ -2,16 +2,22 @@ import argparse import glob +import json +import zipfile +from io import StringIO +from os import getcwd, remove from os.path import basename, exists, isdir +from re import sub +from socket import getfqdn +from subprocess import check_call -#from adios2.adios2_campaign_manager import * +# from adios2.adios2_campaign_manager import * def SetupArgs(): parser = argparse.ArgumentParser() parser.add_argument( - "FILE", help="Name of the input file (.bp, .bp/md.idx, " + - ".bp/md.0 or .bp/data.XXX)") + "command", help="Command: create/update/delete", choices=['create', 'update', 'delete']) parser.add_argument("--verbose", "-v", help="More verbosity", action="count") parser.add_argument("--project", "-p", @@ -32,31 +38,169 @@ def SetupArgs(): args.update = False args.CampaignFileName = args.campaign_store+"/" + \ args.project+"_"+args.app+"_"+args.shot+".acm" - args.LocalCampaignDir = "adios-campaign" + args.LocalCampaignDir = "adios-campaign/" # print("Verbosity = {0}".format(args.verbose)) print(f"Campaign File Name = {args.CampaignFileName}") return args -def CheckFileName(args): - if not exists(args.FILE): - print("ERROR: File " + args.FILE + " does not exist", flush=True) +def CheckCampaignStore(args): + if not isdir(args.campaign_store): + print("ERROR: Campaign directory " + args.campaign_store + + " does not exist", flush=True) exit(1) -def DumpIndexTableFile(args): - indexFileList = glob.glob(args.idxFileName) - if len(indexFileList) > 0: - DumpIndexTable(indexFileList[0]) +def CheckLocalCampaignDir(args): + if not isdir(args.LocalCampaignDir): + print("ERROR: Shot campaign data '" + args.LocalCampaignDir + + "' does not exist. Run this command where the code was executed.", flush=True) + exit(1) + + +def DeleteFileFromArchive(args: dict, filename: str, campaignArchive: list): + cmd = ['zip', '-d', args.CampaignFileName] + [filename] + check_call(cmd) + # reload the modified archive + campaignArchive[0] = zipfile.ZipFile(args.CampaignFileName, mode="a", + compression=zipfile.ZIP_DEFLATED, allowZip64=True) + + +def AddJsonToArchive(args: dict, jsondict: dict, campaignArchive: list, content: list): + filename = "campaign.json" + print(f"Add {filename} to archive: {campaignArchive[0].filename}") + if (filename in content): + print(f"Found existing {filename} in archive") + DeleteFileFromArchive(args, filename, campaignArchive) + + jsonStr = json.dumps(jsondict) + campaignArchive[0].writestr(filename, jsonStr) + + +def IsADIOSDataset(dataset): + if not isdir(dataset): + return False + if not exists(dataset+"/"+"md.idx"): + return False + if not exists(dataset+"/"+"data.0"): + return False + return True + + +def AddFileToArchiveIfItExists(args: dict, filename: str, campaignArchive: list): + if (exists(filename)): + campaignArchive[0].write(filename) + + +def AddDatasetToArchive(args: dict, dataset: str, campaignArchive: list): + if (IsADIOSDataset(dataset)): + print(f"Add dataset {dataset} to archive") + mdFileList = glob.glob(dataset + '/*md.*') + for f in mdFileList: + if (exists(f)): + campaignArchive[0].write(f) else: - print("There is no BP4 Index Table file as " + args.idxFileName) + print(f"Dataset {dataset} is not an ADIOS dataset. Skip") + + +def ProcessJsonFile(args, jsonlist, campaignArchive): + # with open(jsonfile) as f: + # d = json.load(f) + #print(f"Process {jsondata}:") + #d = json.load(jsonIO.getvalue()) + for entry in jsonlist: + print(f"Process entry {entry}:") + if isinstance(entry, dict): + if "name" in entry: + AddDatasetToArchive(args, entry['name'], campaignArchive) + + else: + print("your object is not a dictionary") + + +def MergeJsonFiles(jsonfiles: list): + result = list() + for f1 in jsonfiles: + with open(f1, 'r') as infile: + result.extend(json.load(infile)) + return result + + +def GetHostName(): + host = getfqdn() + if host.startswith("login"): + host = sub('^login[0-9]*\.', '', host) + if host.startswith("batch"): + host = sub('^batch[0-9]*\.', '', host) + shorthost = host.split('.')[0] + return host, shorthost if __name__ == "__main__": args = SetupArgs() - CheckFileName(args) - # print(args) + CheckCampaignStore(args) + + if (args.command == "delete"): + if exists(args.CampaignFileName): + print(f"Delete archive {args.CampaignFileName}") + remove(args.CampaignFileName) + exit(0) + else: + print(f"ERROR: archive {args.CampaignFileName} does not exist") + exit(1) + + CheckLocalCampaignDir(args) + + # List the local campaign directory + jsonFileList = glob.glob(args.LocalCampaignDir + '/*.json') + if len(jsonFileList) == 0: + print("There are no campaign data files in " + args.LocalCampaignDir) + exit(2) + + if (args.command == "create"): + print("Create archive") + mode = 'x' + if exists(args.CampaignFileName): + print(f"ERROR: archive {args.CampaignFileName} already exist") + exit(1) + elif (args.command == "update"): + print("Update archive is not implemented yet") + exit(2) + mode = 'a' + if not exists(args.CampaignFileName): + print(f"ERROR: archive {args.CampaignFileName} does not exist") + exit(1) + + campaignArchive = zipfile.ZipFile(args.CampaignFileName, mode=mode, + compression=zipfile.ZIP_DEFLATED, allowZip64=True) + content = campaignArchive.namelist() + + longHostName, shortHostName = GetHostName() + + # Pass archive in a list in case it has to be reopened (after delete) + clist = [campaignArchive] + print(f"Archive list: {content}") + jsonlist = MergeJsonFiles(jsonFileList) + jsondict = { + "id": "ACM", + "name": "ADIOS Campaign Archive", + "version": "1.0", + "stores": [ + { + "hostname": shortHostName, + "longhostname": longHostName, + "rootdir": getcwd(), + "files": jsonlist + } + ]} + + print(f"Merged json = {jsondict}") + AddJsonToArchive(args, jsondict, clist, content) + ProcessJsonFile(args, jsonlist, clist) +# for jsonfile in jsonFileList: +# AddJsonToArchive(args, jsonfile, clist, content) +# ProcessJsonFile(args, jsonfile, clist) - DumpIndexTableFile(args) + campaignArchive.close() From d89557b593c999402afe807d8f07f58c0f8144b2 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 20 Jun 2023 06:59:56 -0400 Subject: [PATCH 14/58] don't use BP3 --- examples/campaign/campaign_write.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/campaign/campaign_write.cpp b/examples/campaign/campaign_write.cpp index 35e565fbe8..7123285f54 100644 --- a/examples/campaign/campaign_write.cpp +++ b/examples/campaign/campaign_write.cpp @@ -10,7 +10,7 @@ * some existing outputs get appended to (at some specific step) and some * other new outputs are created. * - * Campaign management handles BP3/BP4/BP5 and HDF5 outputs + * Campaign management handles BP4/BP5 outputs * * Outputs: * dataAll.bp: written by every process @@ -166,7 +166,7 @@ int main(int argc, char *argv[]) ioLastRank.SetEngine("BP5"); } ioStep = adiosAll.DeclareIO("OutputStep"); - ioStep.SetEngine("BP3"); + ioStep.SetEngine("BP4"); ioNew = adiosAll.DeclareIO("OutputNew"); #ifdef ADIOS2_USE_HDF5 From c48d22b7ed06d650753336677ca9b921443e52a6 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 20 Jun 2023 07:23:53 -0400 Subject: [PATCH 15/58] Use SQLite3 database for campaign archive (instead of a zipfile) --- .../adios2_campaign_manager.py | 173 +++++++++++------- 1 file changed, 106 insertions(+), 67 deletions(-) diff --git a/source/utils/adios_campaign_manager/adios2_campaign_manager.py b/source/utils/adios_campaign_manager/adios2_campaign_manager.py index 8156adab51..7f233ac7b1 100755 --- a/source/utils/adios_campaign_manager/adios2_campaign_manager.py +++ b/source/utils/adios_campaign_manager/adios2_campaign_manager.py @@ -3,16 +3,20 @@ import argparse import glob import json -import zipfile +import sqlite3 +import zlib from io import StringIO -from os import getcwd, remove +from os import getcwd, remove, stat from os.path import basename, exists, isdir from re import sub from socket import getfqdn from subprocess import check_call +from time import time # from adios2.adios2_campaign_manager import * +ADIOS_ACM_VERSION = "1.0" + def SetupArgs(): parser = argparse.ArgumentParser() @@ -59,25 +63,6 @@ def CheckLocalCampaignDir(args): exit(1) -def DeleteFileFromArchive(args: dict, filename: str, campaignArchive: list): - cmd = ['zip', '-d', args.CampaignFileName] + [filename] - check_call(cmd) - # reload the modified archive - campaignArchive[0] = zipfile.ZipFile(args.CampaignFileName, mode="a", - compression=zipfile.ZIP_DEFLATED, allowZip64=True) - - -def AddJsonToArchive(args: dict, jsondict: dict, campaignArchive: list, content: list): - filename = "campaign.json" - print(f"Add {filename} to archive: {campaignArchive[0].filename}") - if (filename in content): - print(f"Found existing {filename} in archive") - DeleteFileFromArchive(args, filename, campaignArchive) - - jsonStr = json.dumps(jsondict) - campaignArchive[0].writestr(filename, jsonStr) - - def IsADIOSDataset(dataset): if not isdir(dataset): return False @@ -88,35 +73,84 @@ def IsADIOSDataset(dataset): return True -def AddFileToArchiveIfItExists(args: dict, filename: str, campaignArchive: list): - if (exists(filename)): - campaignArchive[0].write(filename) +def compressFile(f): + compObj = zlib.compressobj() + compressed = bytearray() + blocksize = 1073741824 # 1GB #1024*1048576 + len_orig = 0 + len_compressed = 0 + block = f.read(blocksize) + while block: + len_orig += len(block) + cBlock = compObj.compress(block) + compressed += cBlock + len_compressed += len(cBlock) + block = f.read(blocksize) + cBlock = compObj.flush() + compressed += cBlock + len_compressed += len(cBlock) + + return compressed, len_orig, len_compressed + + +def decompressBuffer(buf: bytearray): + data = zlib.decompress(buf) + return data + + +def AddFileToArchive(args: dict, filename: str, cur: sqlite3.Cursor, dsID: int): + compressed = 1 + try: + f = open(filename, 'rb') + compressed_data, len_orig, len_compressed = compressFile(f) + except IOError: + print(f"ERROR While reading file {filename}") + return -def AddDatasetToArchive(args: dict, dataset: str, campaignArchive: list): + statres = stat(filename) + ct = statres.st_ctime_ns + + cur.execute('insert into bpfile values (?, ?, ?, ?, ?, ?, ?)', + (dsID, filename, compressed, len_orig, len_compressed, ct, compressed_data)) + con.commit() + + # test + # if (filename == "dataAll.bp/md.0"): + # data = decompressBuffer(compressed_data) + # of = open("dataAll.bp-md.0", "wb") + # of.write(data) + # of.close() + + +def AddDatasetToArchive(args: dict, dataset: str, cur: sqlite3.Cursor, hostID: int, dirID: int): if (IsADIOSDataset(dataset)): print(f"Add dataset {dataset} to archive") + statres = stat(dataset) + ct = statres.st_ctime_ns + curDS = cur.execute('insert into bpdataset values (?, ?, ?, ?)', + (hostID, dirID, dataset, ct)) + + dsID = curDS.lastrowid mdFileList = glob.glob(dataset + '/*md.*') - for f in mdFileList: - if (exists(f)): - campaignArchive[0].write(f) + profileList = glob.glob(dataset + '/profiling.json') + files = mdFileList+profileList + for f in files: + AddFileToArchive(args, f, cur, dsID) else: - print(f"Dataset {dataset} is not an ADIOS dataset. Skip") + print(f"WARNING: Dataset {dataset} is not an ADIOS dataset. Skip") -def ProcessJsonFile(args, jsonlist, campaignArchive): - # with open(jsonfile) as f: - # d = json.load(f) - #print(f"Process {jsondata}:") - #d = json.load(jsonIO.getvalue()) +def ProcessJsonFile(args: dict, jsonlist: list, cur: sqlite3.Cursor, hostID: int, dirID: int): for entry in jsonlist: print(f"Process entry {entry}:") if isinstance(entry, dict): if "name" in entry: - AddDatasetToArchive(args, entry['name'], campaignArchive) + AddDatasetToArchive( + args, entry['name'], cur, hostID, dirID) else: - print("your object is not a dictionary") + print(f"WARNING: your object is not a dictionary, skip : {entry}") def MergeJsonFiles(jsonfiles: list): @@ -161,46 +195,51 @@ def GetHostName(): if (args.command == "create"): print("Create archive") - mode = 'x' if exists(args.CampaignFileName): print(f"ERROR: archive {args.CampaignFileName} already exist") exit(1) elif (args.command == "update"): - print("Update archive is not implemented yet") - exit(2) - mode = 'a' + print("Update archive") if not exists(args.CampaignFileName): print(f"ERROR: archive {args.CampaignFileName} does not exist") exit(1) - campaignArchive = zipfile.ZipFile(args.CampaignFileName, mode=mode, - compression=zipfile.ZIP_DEFLATED, allowZip64=True) - content = campaignArchive.namelist() + con = sqlite3.connect(args.CampaignFileName) + cur = con.cursor() + if (args.command == "create"): + epoch = int(time()) + cur.execute( + "create table info(id TEXT, name TEXT, version TEXT, ctime INT)") + cur.execute('insert into info values (?, ?, ?, ?)', + ("ACM", "ADIOS Campaign Archive", ADIOS_ACM_VERSION, epoch)) + cur.execute("create table host" + + "(hostname TEXT PRIMARY KEY, longhostname TEXT)") + cur.execute("create table directory" + + "(hostid INT, name TEXT, PRIMARY KEY (hostid, name))") + cur.execute("create table bpdataset" + + "(hostid INT, dirid INT, name TEXT, ctime INT" + + ", PRIMARY KEY (hostid, dirid, name))") + cur.execute("create table bpfile" + + "(bpdatasetid INT, name TEXT, compression INT, lenorig INT" + + ", lencompressed INT, ctime INT, data BLOB" + + ", PRIMARY KEY (bpdatasetid, name))") longHostName, shortHostName = GetHostName() + rootdir = getcwd() + curHost = cur.execute('insert into host values (?, ?)', + (shortHostName, longHostName)) + hostID = curHost.lastrowid + + curDir = cur.execute('insert into directory values (?, ?)', + (hostID, rootdir)) + dirID = curDir.lastrowid + con.commit() - # Pass archive in a list in case it has to be reopened (after delete) - clist = [campaignArchive] - print(f"Archive list: {content}") jsonlist = MergeJsonFiles(jsonFileList) - jsondict = { - "id": "ACM", - "name": "ADIOS Campaign Archive", - "version": "1.0", - "stores": [ - { - "hostname": shortHostName, - "longhostname": longHostName, - "rootdir": getcwd(), - "files": jsonlist - } - ]} - - print(f"Merged json = {jsondict}") - AddJsonToArchive(args, jsondict, clist, content) - ProcessJsonFile(args, jsonlist, clist) -# for jsonfile in jsonFileList: -# AddJsonToArchive(args, jsonfile, clist, content) -# ProcessJsonFile(args, jsonfile, clist) - - campaignArchive.close() + + print(f"Merged json = {jsonlist}") + ProcessJsonFile(args, jsonlist, cur, hostID, dirID) + + con.commit() + cur.close() + con.close() From 6ad334515264dcd056cb994c8a408a2bd244be7b Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 20 Jun 2023 07:25:30 -0400 Subject: [PATCH 16/58] missing cmake file for campaign manager python script --- source/utils/adios_campaign_manager/CMakeLists.txt | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 source/utils/adios_campaign_manager/CMakeLists.txt diff --git a/source/utils/adios_campaign_manager/CMakeLists.txt b/source/utils/adios_campaign_manager/CMakeLists.txt new file mode 100644 index 0000000000..01a9bce45a --- /dev/null +++ b/source/utils/adios_campaign_manager/CMakeLists.txt @@ -0,0 +1,13 @@ +install(PROGRAMS adios2_campaign_manager.py + RENAME adios2_campaign_manager + DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT adios2_scripts-runtime +) +#install( +# FILES +# adios2/bp4dbg/__init__.py +# adios2/bp4dbg/data.py +# adios2/bp4dbg/utils.py +# adios2/bp4dbg/metadata.py +# adios2/bp4dbg/idxtable.py +# DESTINATION ${CMAKE_INSTALL_PYTHONDIR}/adios2/bp4dbg COMPONENT adios2_scripts-runtime +#) From 192543e999314e29df65594811c5f61143e94315 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 20 Jun 2023 07:25:48 -0400 Subject: [PATCH 17/58] Add SQLite3 to build options, make CampaignReader dependent on it --- CMakeLists.txt | 4 +++- cmake/DetectOptions.cmake | 10 ++++++++++ source/adios2/CMakeLists.txt | 6 +++++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ad70e1817b..155d252559 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ # accompanying file Copyright.txt for details. #------------------------------------------------------------------------------# -cmake_minimum_required(VERSION 3.12) +cmake_minimum_required(VERSION 3.14) # Fail immediately if not using an out-of-source build if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR) @@ -168,6 +168,7 @@ adios_option(Endian_Reverse "Enable support for Little/Big Endian Interoperabili adios_option(Sodium "Enable support for Sodium for encryption" AUTO) adios_option(Catalyst "Enable support for in situ visualization plugin using ParaView Catalyst" AUTO) adios_option(AWSSDK "Enable support for S3 compatible storage using AWS SDK's S3 module" AUTO) +adios_option(SQLite3 "Enable support for SQLite3 required by campaign manager" AUTO) include(${PROJECT_SOURCE_DIR}/cmake/DetectOptions.cmake) if(ADIOS2_HAVE_CUDA OR ADIOS2_HAVE_Kokkos_CUDA) @@ -239,6 +240,7 @@ set(ADIOS2_CONFIG_OPTS BP5 DataMan DataSpaces HDF5 HDF5_VOL MHS SST Fortran MPI Python Blosc2 BZip2 LIBPRESSIO MGARD PNG SZ ZFP DAOS IME O_DIRECT Sodium Catalyst SysVShMem UCX ZeroMQ Profiling Endian_Reverse AWSSDK GPU_Support CUDA Kokkos Kokkos_CUDA Kokkos_HIP Kokkos_SYCL + SQLite3 ) GenerateADIOSHeaderConfig(${ADIOS2_CONFIG_OPTS}) diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index 99d2d45cb1..8f6819efa2 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -525,6 +525,16 @@ if(AWSSDK_FOUND) set(ADIOS2_HAVE_AWSSDK TRUE) endif() +# sqlite3 +if(ADIOS2_USE_SQLite3 STREQUAL AUTO) + find_package(SQLite3) +elseif(ADIOS2_USE_SQLite3) + find_package(SQLite3 REQUIRED) +endif() +if(SQLite3_FOUND) + set(ADIOS2_HAVE_SQLite3 TRUE) +endif() + # Multithreading find_package(Threads REQUIRED) diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index 9e0fb0b216..5db85449cc 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -57,7 +57,6 @@ add_library(adios2_core engine/inline/InlineWriter.cpp engine/inline/InlineWriter.tcc engine/campaign/CampaignManager.cpp - engine/campaign/CampaignReader.cpp engine/campaign/CampaignReader.tcc engine/null/NullWriter.cpp engine/null/NullWriter.tcc engine/null/NullReader.cpp engine/null/NullReader.tcc @@ -169,6 +168,11 @@ if (ADIOS2_HAVE_SST) add_subdirectory(toolkit/remote) endif() +if(ADIOS2_HAVE_SQLite3) + target_sources(adios2_core PRIVATE engine/campaign/CampaignReader.cpp engine/campaign/CampaignReader.tcc) + target_link_libraries(adios2_core PRIVATE SQLite::SQLite3) +endif() + if (ADIOS2_HAVE_BP5) target_sources(adios2_core PRIVATE engine/bp5/BP5Engine.cpp From c43bbf89fa13ca7d1421b1fe7f2a166acf3c434c Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 20 Jun 2023 07:50:01 -0400 Subject: [PATCH 18/58] campaign reader opens/closes sqlite db but it doesn't do anything yet --- .../adios2/engine/campaign/CampaignReader.cpp | 19 ++++++++++++++++++- .../adios2/engine/campaign/CampaignReader.h | 6 +++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/source/adios2/engine/campaign/CampaignReader.cpp b/source/adios2/engine/campaign/CampaignReader.cpp index e7e2896f2d..48698c390c 100644 --- a/source/adios2/engine/campaign/CampaignReader.cpp +++ b/source/adios2/engine/campaign/CampaignReader.cpp @@ -45,7 +45,7 @@ CampaignReader::~CampaignReader() /* CampaignReader destructor does close and finalize */ if (m_Verbosity == 5) { - std::cout << "Campaign Reader " << m_ReaderRank << " deconstructor on " + std::cout << "Campaign Reader " << m_ReaderRank << " destructor on " << m_Name << "\n"; } if (m_IsOpen) @@ -147,6 +147,16 @@ void CampaignReader::InitParameters() void CampaignReader::InitTransports() { + int rc = sqlite3_open(m_Name.c_str(), &m_DB); + if (rc) + { + std::string dbmsg(sqlite3_errmsg(m_DB)); + sqlite3_close(m_DB); + helper::Throw("Engine", "CampaignReader", "Open", + "Cannot open database" + m_Name + + ": " + dbmsg); + } + std::string cs = m_Comm.BroadcastFile(m_Name, "broadcast campaign file"); nlohmann::json js = nlohmann::json::parse(cs); std::cout << "JSON rank " << m_ReaderRank << ": " << js.size() << std::endl; @@ -204,6 +214,13 @@ void CampaignReader::DoClose(const int transportIndex) { ep->Close(); } + sqlite3_close(m_DB); + m_IsOpen = false; +} + +void CampaignReader::DestructorClose(bool Verbose) noexcept +{ + sqlite3_close(m_DB); } // Remove the engine name from the var name, which must be of pattern diff --git a/source/adios2/engine/campaign/CampaignReader.h b/source/adios2/engine/campaign/CampaignReader.h index 681a755204..2b0578bef6 100644 --- a/source/adios2/engine/campaign/CampaignReader.h +++ b/source/adios2/engine/campaign/CampaignReader.h @@ -18,6 +18,8 @@ #include "adios2/helper/adiosComm.h" #include "adios2/helper/adiosFunctions.h" +#include + namespace adios2 { namespace core @@ -105,7 +107,7 @@ class CampaignReader : public Engine * Called if destructor is called on an open engine. Should warn or take * any non-complex measure that might help recover. */ - void DestructorClose(bool Verbose) noexcept final{}; + void DestructorClose(bool Verbose) noexcept final; /** * Create a new variable with name `name` in `io` @@ -122,6 +124,8 @@ class CampaignReader : public Engine template std::pair *, Engine *> TranslateToActualVariable(Variable &variable); + + sqlite3 *m_DB; }; } // end namespace engine From d0c52e9a44ded976d0804cf482d1e09cdf70b9c2 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 20 Jun 2023 11:12:48 -0400 Subject: [PATCH 19/58] functions to get the Fully Qualified Domain Name, and a cluster name parsed from FQDN --- source/adios2/helper/adiosNetwork.cpp | 69 +++++++++++++++++++++++++++ source/adios2/helper/adiosNetwork.h | 13 +++++ 2 files changed, 82 insertions(+) diff --git a/source/adios2/helper/adiosNetwork.cpp b/source/adios2/helper/adiosNetwork.cpp index dce2872b02..3451d6752a 100644 --- a/source/adios2/helper/adiosNetwork.cpp +++ b/source/adios2/helper/adiosNetwork.cpp @@ -25,6 +25,10 @@ #include //AvailableIpAddresses() ioctl #include //AvailableIpAddresses() close +#include //getFQDN +#include //getFQDN +#include //getFQDN + #include namespace adios2 @@ -79,6 +83,71 @@ AvailableIpAddresses() noexcept return ips; } +std::string GetFQDN() noexcept +{ + char hostname[1024]; +#ifdef WIN32 + TCHAR infoBuf[1024]; + DWORD bufCharCount = sizeof(hostname); + memset(hostname, 0, sizeof(hostname)); + if (GetComputerName(infoBuf, &bufCharCount)) + { + for (i = 0; i < sizeof(hostname); i++) + { + hostname[i] = infoBuf[i]; + } + } + else + { + strcpy(hostname, "Unknown_Host_Name"); + } +#else + struct addrinfo hints, *info, *p; + int gai_result; + + hostname[1023] = '\0'; + gethostname(hostname, 1023); + + memset(&hints, 0, sizeof hints); + hints.ai_family = AF_UNSPEC; /*either IPV4 or IPV6*/ + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_CANONNAME; + + if ((gai_result = getaddrinfo(hostname, NULL, &hints, &info)) == 0) + { + for (p = info; p != NULL; p = p->ai_next) + { + printf("hostname: %s\n", p->ai_canonname); + if (strchr(p->ai_canonname, '.') != NULL) + { + strncpy(hostname, p->ai_canonname, sizeof(hostname)); + break; + } + } + } + else + { + strcpy(hostname, "Unknown_Host_Name"); + } + freeaddrinfo(info); +#endif + return std::string(hostname); +} + +std::string GetClusterName() noexcept +{ + std::string fqdn = GetFQDN(); + if (fqdn.rfind("login", 0) == 0) + { + fqdn.erase(0, fqdn.find('.') + 1); + } + if (fqdn.rfind("batch", 0) == 0) + { + fqdn.erase(0, fqdn.find('.') + 1); + } + return fqdn.substr(0, fqdn.find('.')); +} + void HandshakeWriter(Comm const &comm, size_t &appID, std::vector &fullAddresses, const std::string &name, const std::string &engineName, const int basePort, const int channelsPerRank, const int maxRanksPerNode, const int maxAppsPerNode) diff --git a/source/adios2/helper/adiosNetwork.h b/source/adios2/helper/adiosNetwork.h index d8e15db854..9c503b512f 100644 --- a/source/adios2/helper/adiosNetwork.h +++ b/source/adios2/helper/adiosNetwork.h @@ -36,6 +36,19 @@ class Comm; */ std::vector AvailableIpAddresses() noexcept; +/** + * returns the (a) fully qualified domain name of the current machine. + * the result is "Unknown_Host_Name" if fqdn is not found + */ +std::string GetFQDN() noexcept; + +/** + * returns a hostname from an FQDN but not the login* or batch* name, + * instead the second part from such names + * e.g. (login01.summit.ornl.gov -> summit) + */ +std::string GetClusterName() noexcept; + void HandshakeWriter(Comm const &comm, size_t &appID, std::vector &fullAddresses, const std::string &name, const std::string &engineName, const int basePort, const int channelsPerRank, const int maxRanksPerNode = 100, From 875a7ff1fee605b9077cc35ea1a47dc68644b947 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 20 Jun 2023 11:13:19 -0400 Subject: [PATCH 20/58] CampaignReader starts using SQLite3 --- .../adios2/engine/campaign/CampaignReader.cpp | 35 ++++++++++++++++++- .../adios2/engine/campaign/CampaignReader.h | 5 +-- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/source/adios2/engine/campaign/CampaignReader.cpp b/source/adios2/engine/campaign/CampaignReader.cpp index 48698c390c..f8dfd50674 100644 --- a/source/adios2/engine/campaign/CampaignReader.cpp +++ b/source/adios2/engine/campaign/CampaignReader.cpp @@ -12,6 +12,7 @@ #include "CampaignReader.tcc" #include "adios2/helper/adiosFunctions.h" // CSVToVector +#include "adios2/helper/adiosNetwork.h" // GetFQDN #include #include @@ -142,12 +143,36 @@ void CampaignReader::InitParameters() "integer in the range [0,5], in call to " "Open or Engine constructor"); } + if (key == "hostname") + { + m_Hostname = pair.second; + } } + + if (m_Hostname.empty()) + { + m_Hostname = helper::GetClusterName(); + } + // std::cout << "My Hostname is " << m_Hostname << std::endl; } +static int sqlcb_host(void *NotUsed, int argc, char **argv, char **azColName) +{ + for (int i = 0; i < argc; i++) + { + std::cout << azColName[i] << " = " << (argv[i] ? argv[i] : "NULL") + << std::endl; + } + std::cout << std::endl; + return 0; +}; + void CampaignReader::InitTransports() { - int rc = sqlite3_open(m_Name.c_str(), &m_DB); + int rc; + char *zErrMsg = 0; + + rc = sqlite3_open(m_Name.c_str(), &m_DB); if (rc) { std::string dbmsg(sqlite3_errmsg(m_DB)); @@ -157,6 +182,14 @@ void CampaignReader::InitTransports() ": " + dbmsg); } + std::string sqlcmd = "SELECT hostname, longhostname FROM host"; + rc = sqlite3_exec(m_DB, sqlcmd.c_str(), sqlcb_host, 0, &zErrMsg); + if (rc != SQLITE_OK) + { + std::cout << "SQL error: " << zErrMsg << std::endl; + sqlite3_free(zErrMsg); + } + std::string cs = m_Comm.BroadcastFile(m_Name, "broadcast campaign file"); nlohmann::json js = nlohmann::json::parse(cs); std::cout << "JSON rank " << m_ReaderRank << ": " << js.size() << std::endl; diff --git a/source/adios2/engine/campaign/CampaignReader.h b/source/adios2/engine/campaign/CampaignReader.h index 2b0578bef6..08d73878a7 100644 --- a/source/adios2/engine/campaign/CampaignReader.h +++ b/source/adios2/engine/campaign/CampaignReader.h @@ -53,8 +53,9 @@ class CampaignReader : public Engine MinVarInfo *MinBlocksInfo(const VariableBase &, const size_t Step) const; private: - int m_Verbosity = 0; - int m_ReaderRank; // my rank in the readers' comm + int m_Verbosity = 0; // runtime parameter Verbose + std::string m_Hostname; // runtime parameter Hostname + int m_ReaderRank; // my rank in the readers' comm int m_CurrentStep = 0; bool m_FirstStep = true; From 0e35446fcd2b7e137bed07e6bcd22a96b154517c Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Mon, 26 Jun 2023 11:19:47 -0400 Subject: [PATCH 21/58] CampaignReader processes sqlite3 database. Able to open local files (based on hostname match) --- source/adios2/CMakeLists.txt | 5 +- source/adios2/engine/campaign/CampaignData.h | 49 +++++--- .../engine/campaign/CampaignManager.cpp | 2 +- .../adios2/engine/campaign/CampaignManager.h | 4 +- .../adios2/engine/campaign/CampaignReader.cpp | 108 +++++++++++------- .../adios2/engine/campaign/CampaignReader.h | 2 + .../adios2_campaign_manager.py | 9 +- 7 files changed, 113 insertions(+), 66 deletions(-) diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index 5db85449cc..1492005925 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -169,7 +169,10 @@ if (ADIOS2_HAVE_SST) endif() if(ADIOS2_HAVE_SQLite3) - target_sources(adios2_core PRIVATE engine/campaign/CampaignReader.cpp engine/campaign/CampaignReader.tcc) + target_sources(adios2_core PRIVATE + engine/campaign/CampaignReader.cpp + engine/campaign/CampaignReader.tcc + engine/campaign/CampaignData.cpp) target_link_libraries(adios2_core PRIVATE SQLite::SQLite3) endif() diff --git a/source/adios2/engine/campaign/CampaignData.h b/source/adios2/engine/campaign/CampaignData.h index cb2f05ae12..337deb9bb4 100644 --- a/source/adios2/engine/campaign/CampaignData.h +++ b/source/adios2/engine/campaign/CampaignData.h @@ -3,7 +3,7 @@ * accompanying file Copyright.txt for details. * * CampaignData.h - * Campaign data struct + * Campaign data from database * * Created on: May 16, 2023 * Author: Norbert Podhorszki pnorbert@ornl.gov @@ -16,6 +16,8 @@ #include #include +#include + namespace adios2 { namespace core @@ -23,25 +25,38 @@ namespace core namespace engine { -struct CampaignRecord +struct CampaignHost +{ + std::string hostname; + std::string longhostname; + std::vector directory; +}; + +struct CampaignBPFile +{ + std::string name; + size_t bpDatasetIdx; + bool compressed; + size_t lengthOriginal; + size_t lengthCompressed; +}; + +struct CampaignBPDataset +{ + std::string name; + size_t hostIdx; + size_t dirIdx; + std::vector files; +}; + +struct CampaignData { - std::vector steps; - size_t delta_step; - std::vector times; - double delta_time; - bool varying_deltas; - - CampaignRecord() : delta_step(0), delta_time(0.0), varying_deltas(false){}; - - CampaignRecord(size_t step, double time) - : delta_step(0), delta_time(0.0), varying_deltas(false) - { - steps.push_back(step); - times.push_back(time); - }; + std::string version; + std::vector hosts; + std::vector bpdatasets; }; -using CampaignMap = std::unordered_map; +void ReadCampaignData(sqlite3 *db, CampaignData &cd); } // end namespace engine } // end namespace core diff --git a/source/adios2/engine/campaign/CampaignManager.cpp b/source/adios2/engine/campaign/CampaignManager.cpp index e0c799a2e6..0d70ce72ba 100644 --- a/source/adios2/engine/campaign/CampaignManager.cpp +++ b/source/adios2/engine/campaign/CampaignManager.cpp @@ -25,7 +25,7 @@ namespace core namespace engine { -static std::string CMapToJson(const CampaignMap &cmap, const int rank, +static std::string CMapToJson(const CampaignRecordMap &cmap, const int rank, const std::string name) { nlohmann::json j = nlohmann::json::array(); diff --git a/source/adios2/engine/campaign/CampaignManager.h b/source/adios2/engine/campaign/CampaignManager.h index 51e30ef62c..209a4449c2 100644 --- a/source/adios2/engine/campaign/CampaignManager.h +++ b/source/adios2/engine/campaign/CampaignManager.h @@ -15,7 +15,7 @@ #ifndef ADIOS2_ENGINE_CAMPAIGNMANAGER_H_ #define ADIOS2_ENGINE_CAMPAIGNMANAGER_H_ -#include "CampaignData.h" +#include "CampaignRecord.h" #include "adios2/common/ADIOSConfig.h" #include "adios2/helper/adiosComm.h" @@ -45,7 +45,7 @@ class CampaignManager std::string m_Name; int m_WriterRank; int m_Verbosity = 5; - CampaignMap cmap; + CampaignRecordMap cmap; std::ofstream m_Output; const std::string m_CampaignDir = "adios-campaign"; }; diff --git a/source/adios2/engine/campaign/CampaignReader.cpp b/source/adios2/engine/campaign/CampaignReader.cpp index f8dfd50674..8d2511fd03 100644 --- a/source/adios2/engine/campaign/CampaignReader.cpp +++ b/source/adios2/engine/campaign/CampaignReader.cpp @@ -156,17 +156,6 @@ void CampaignReader::InitParameters() // std::cout << "My Hostname is " << m_Hostname << std::endl; } -static int sqlcb_host(void *NotUsed, int argc, char **argv, char **azColName) -{ - for (int i = 0; i < argc; i++) - { - std::cout << azColName[i] << " = " << (argv[i] ? argv[i] : "NULL") - << std::endl; - } - std::cout << std::endl; - return 0; -}; - void CampaignReader::InitTransports() { int rc; @@ -182,45 +171,72 @@ void CampaignReader::InitTransports() ": " + dbmsg); } - std::string sqlcmd = "SELECT hostname, longhostname FROM host"; - rc = sqlite3_exec(m_DB, sqlcmd.c_str(), sqlcb_host, 0, &zErrMsg); - if (rc != SQLITE_OK) + ReadCampaignData(m_DB, m_CampaignData); + + std::cout << "Local hostname = " << m_Hostname << "\n"; + std::cout << "Database result:\n version = " << m_CampaignData.version + << "\n hosts:\n"; + + for (size_t hostidx = 0; hostidx < m_CampaignData.hosts.size(); ++hostidx) { - std::cout << "SQL error: " << zErrMsg << std::endl; - sqlite3_free(zErrMsg); + CampaignHost &h = m_CampaignData.hosts[hostidx]; + std::cout << " host =" << h.hostname + << " long name = " << h.longhostname << " directories: \n"; + for (size_t diridx = 0; diridx < h.directory.size(); ++diridx) + { + std::cout << " dir = " << h.directory[diridx] << "\n"; + } + } + std::cout << " datasets:\n"; + for (auto &ds : m_CampaignData.bpdatasets) + { + std::cout << " " << m_CampaignData.hosts[ds.hostIdx].hostname << ":" + << m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + << PathSeparator << ds.name << "\n"; + for (auto &bpf : ds.files) + { + std::cout << " file: " << bpf.name << "\n"; + } } - std::string cs = m_Comm.BroadcastFile(m_Name, "broadcast campaign file"); - nlohmann::json js = nlohmann::json::parse(cs); - std::cout << "JSON rank " << m_ReaderRank << ": " << js.size() << std::endl; + // std::string cs = m_Comm.BroadcastFile(m_Name, "broadcast campaign file"); + // nlohmann::json js = nlohmann::json::parse(cs); + // std::cout << "JSON rank " << m_ReaderRank << ": " << js.size() << + // std::endl; int i = 0; - for (auto &jf : js) + for (auto &ds : m_CampaignData.bpdatasets) { - std::cout << jf << std::endl; - adios2::core::IO &io = - m_IO.m_ADIOS.DeclareIO("CampaignReader" + std::to_string(i)); - adios2::core::Engine &e = - io.Open(jf["name"], m_OpenMode, m_Comm.Duplicate()); - - m_IOs.push_back(&io); - m_Engines.push_back(&e); + if (m_CampaignData.hosts[ds.hostIdx].hostname == m_Hostname) + { + std::string localPath = + m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + + PathSeparator + ds.name; + std::cout << "Open local file " << localPath << "\n"; - auto vmap = io.GetAvailableVariables(); - auto amap = io.GetAvailableAttributes(); - VarInternalInfo internalInfo(nullptr, m_IOs.size() - 1, - m_Engines.size() - 1); + adios2::core::IO &io = + m_IO.m_ADIOS.DeclareIO("CampaignReader" + std::to_string(i)); + adios2::core::Engine &e = + io.Open(localPath, m_OpenMode, m_Comm.Duplicate()); - for (auto &vr : vmap) - { - auto vname = vr.first; - std::string fname = jf["name"]; - std::string newname = fname + "/" + vname; + m_IOs.push_back(&io); + m_Engines.push_back(&e); - const DataType type = io.InquireVariableType(vname); + auto vmap = io.GetAvailableVariables(); + auto amap = io.GetAvailableAttributes(); + VarInternalInfo internalInfo(nullptr, m_IOs.size() - 1, + m_Engines.size() - 1); - if (type == DataType::Struct) + for (auto &vr : vmap) { - } + auto vname = vr.first; + std::string fname = ds.name; + std::string newname = fname + "/" + vname; + + const DataType type = io.InquireVariableType(vname); + + if (type == DataType::Struct) + { + } #define declare_type(T) \ else if (type == helper::GetDataType()) \ { \ @@ -228,10 +244,18 @@ void CampaignReader::InitTransports() Variable v = DuplicateVariable(vi, m_IO, newname, internalInfo); \ } - ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) + ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) #undef declare_type + } + } + else + { + std::string remotePath = + m_CampaignData.hosts[ds.hostIdx].hostname + ":" + + m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + + PathSeparator + ds.name; + std::cout << "Cannot yet Open remote file " << remotePath << "\n"; } - ++i; } } diff --git a/source/adios2/engine/campaign/CampaignReader.h b/source/adios2/engine/campaign/CampaignReader.h index 08d73878a7..09cbbcd244 100644 --- a/source/adios2/engine/campaign/CampaignReader.h +++ b/source/adios2/engine/campaign/CampaignReader.h @@ -12,6 +12,7 @@ #ifndef ADIOS2_ENGINE_CAMPAIGNREADER_H_ #define ADIOS2_ENGINE_CAMPAIGNREADER_H_ +#include "CampaignData.h" #include "adios2/common/ADIOSConfig.h" #include "adios2/core/ADIOS.h" #include "adios2/core/Engine.h" @@ -127,6 +128,7 @@ class CampaignReader : public Engine TranslateToActualVariable(Variable &variable); sqlite3 *m_DB; + CampaignData m_CampaignData; }; } // end namespace engine diff --git a/source/utils/adios_campaign_manager/adios2_campaign_manager.py b/source/utils/adios_campaign_manager/adios2_campaign_manager.py index 7f233ac7b1..208c182fe2 100755 --- a/source/utils/adios_campaign_manager/adios2_campaign_manager.py +++ b/source/utils/adios_campaign_manager/adios2_campaign_manager.py @@ -6,7 +6,7 @@ import sqlite3 import zlib from io import StringIO -from os import getcwd, remove, stat +from os import chdir, getcwd, remove, stat from os.path import basename, exists, isdir from re import sub from socket import getfqdn @@ -132,11 +132,14 @@ def AddDatasetToArchive(args: dict, dataset: str, cur: sqlite3.Cursor, hostID: i (hostID, dirID, dataset, ct)) dsID = curDS.lastrowid - mdFileList = glob.glob(dataset + '/*md.*') - profileList = glob.glob(dataset + '/profiling.json') + cwd = getcwd() + chdir(dataset) + mdFileList = glob.glob('*md.*') + profileList = glob.glob('profiling.json') files = mdFileList+profileList for f in files: AddFileToArchive(args, f, cur, dsID) + chdir(cwd) else: print(f"WARNING: Dataset {dataset} is not an ADIOS dataset. Skip") From 2f482aec7e5509dc87a26d3a9568037749b9f895 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Fri, 30 Jun 2023 13:42:07 -0400 Subject: [PATCH 22/58] Save compressed metadata to disk and read from there. --- source/adios2/engine/campaign/CampaignData.h | 5 ++ .../adios2/engine/campaign/CampaignReader.cpp | 77 +++++++++++-------- .../adios2/engine/campaign/CampaignReader.h | 7 +- 3 files changed, 55 insertions(+), 34 deletions(-) diff --git a/source/adios2/engine/campaign/CampaignData.h b/source/adios2/engine/campaign/CampaignData.h index 337deb9bb4..0ea1bf5f61 100644 --- a/source/adios2/engine/campaign/CampaignData.h +++ b/source/adios2/engine/campaign/CampaignData.h @@ -12,6 +12,8 @@ #ifndef ADIOS2_ENGINE_CAMPAIGNDATA_H_ #define ADIOS2_ENGINE_CAMPAIGNDATA_H_ +#include +#include #include #include #include @@ -58,6 +60,9 @@ struct CampaignData void ReadCampaignData(sqlite3 *db, CampaignData &cd); +void SaveToFile(sqlite3 *db, const std::string &path, + const CampaignBPFile &bpfile); + } // end namespace engine } // end namespace core } // end namespace adios2 diff --git a/source/adios2/engine/campaign/CampaignReader.cpp b/source/adios2/engine/campaign/CampaignReader.cpp index 8d2511fd03..b86f8bbd3f 100644 --- a/source/adios2/engine/campaign/CampaignReader.cpp +++ b/source/adios2/engine/campaign/CampaignReader.cpp @@ -13,6 +13,7 @@ #include "adios2/helper/adiosFunctions.h" // CSVToVector #include "adios2/helper/adiosNetwork.h" // GetFQDN +#include "adios2/helper/adiosSystem.h" // CreateDirectory #include #include @@ -147,6 +148,10 @@ void CampaignReader::InitParameters() { m_Hostname = pair.second; } + if (key == "cachepath") + { + m_CachePath = pair.second; + } } if (m_Hostname.empty()) @@ -206,37 +211,56 @@ void CampaignReader::InitTransports() int i = 0; for (auto &ds : m_CampaignData.bpdatasets) { - if (m_CampaignData.hosts[ds.hostIdx].hostname == m_Hostname) + std::string localPath; + if (m_CampaignData.hosts[ds.hostIdx].hostname != m_Hostname) { - std::string localPath = + std::string remotePath = + m_CampaignData.hosts[ds.hostIdx].hostname + ":" + m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + PathSeparator + ds.name; + std::cout << "Open remote file " << remotePath << "\n"; + localPath = m_CachePath + PathSeparator + + m_CampaignData.hosts[ds.hostIdx].hostname + + PathSeparator + ds.name; + helper::CreateDirectory(localPath); + for (auto &bpf : ds.files) + { + std::cout << " save file " << remotePath << "/" << bpf.name + << " to " << localPath << "/" << bpf.name << "\n"; + SaveToFile(m_DB, localPath + PathSeparator + bpf.name, bpf); + } + } + else + { + localPath = m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + + PathSeparator + ds.name; std::cout << "Open local file " << localPath << "\n"; + } - adios2::core::IO &io = - m_IO.m_ADIOS.DeclareIO("CampaignReader" + std::to_string(i)); - adios2::core::Engine &e = - io.Open(localPath, m_OpenMode, m_Comm.Duplicate()); + adios2::core::IO &io = + m_IO.m_ADIOS.DeclareIO("CampaignReader" + std::to_string(i)); + adios2::core::Engine &e = + io.Open(localPath, m_OpenMode, m_Comm.Duplicate()); - m_IOs.push_back(&io); - m_Engines.push_back(&e); + m_IOs.push_back(&io); + m_Engines.push_back(&e); - auto vmap = io.GetAvailableVariables(); - auto amap = io.GetAvailableAttributes(); - VarInternalInfo internalInfo(nullptr, m_IOs.size() - 1, - m_Engines.size() - 1); + auto vmap = io.GetAvailableVariables(); + auto amap = io.GetAvailableAttributes(); + VarInternalInfo internalInfo(nullptr, m_IOs.size() - 1, + m_Engines.size() - 1); - for (auto &vr : vmap) - { - auto vname = vr.first; - std::string fname = ds.name; - std::string newname = fname + "/" + vname; + for (auto &vr : vmap) + { + auto vname = vr.first; + std::string fname = ds.name; + std::string newname = fname + "/" + vname; - const DataType type = io.InquireVariableType(vname); + const DataType type = io.InquireVariableType(vname); - if (type == DataType::Struct) - { - } + if (type == DataType::Struct) + { + } #define declare_type(T) \ else if (type == helper::GetDataType()) \ { \ @@ -244,17 +268,8 @@ void CampaignReader::InitTransports() Variable v = DuplicateVariable(vi, m_IO, newname, internalInfo); \ } - ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) + ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) #undef declare_type - } - } - else - { - std::string remotePath = - m_CampaignData.hosts[ds.hostIdx].hostname + ":" + - m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + - PathSeparator + ds.name; - std::cout << "Cannot yet Open remote file " << remotePath << "\n"; } ++i; } diff --git a/source/adios2/engine/campaign/CampaignReader.h b/source/adios2/engine/campaign/CampaignReader.h index 09cbbcd244..0264e55f73 100644 --- a/source/adios2/engine/campaign/CampaignReader.h +++ b/source/adios2/engine/campaign/CampaignReader.h @@ -54,9 +54,10 @@ class CampaignReader : public Engine MinVarInfo *MinBlocksInfo(const VariableBase &, const size_t Step) const; private: - int m_Verbosity = 0; // runtime parameter Verbose - std::string m_Hostname; // runtime parameter Hostname - int m_ReaderRank; // my rank in the readers' comm + int m_Verbosity = 0; // runtime parameter Verbose + std::string m_Hostname; // runtime parameter Hostname + std::string m_CachePath = "/tmp"; // runtime parameter CachePath + int m_ReaderRank; // my rank in the readers' comm int m_CurrentStep = 0; bool m_FirstStep = true; From bb16f6af22ba91b34aa3a8de36b52f617219dd1a Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Fri, 30 Jun 2023 14:26:36 -0400 Subject: [PATCH 23/58] check ctime of individual files and save from DB only if it is older --- source/adios2/engine/campaign/CampaignData.h | 1 + source/adios2/engine/campaign/CampaignReader.cpp | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/source/adios2/engine/campaign/CampaignData.h b/source/adios2/engine/campaign/CampaignData.h index 0ea1bf5f61..5150fe487c 100644 --- a/source/adios2/engine/campaign/CampaignData.h +++ b/source/adios2/engine/campaign/CampaignData.h @@ -41,6 +41,7 @@ struct CampaignBPFile bool compressed; size_t lengthOriginal; size_t lengthCompressed; + long ctime; }; struct CampaignBPDataset diff --git a/source/adios2/engine/campaign/CampaignReader.cpp b/source/adios2/engine/campaign/CampaignReader.cpp index b86f8bbd3f..3a633809ae 100644 --- a/source/adios2/engine/campaign/CampaignReader.cpp +++ b/source/adios2/engine/campaign/CampaignReader.cpp @@ -225,8 +225,9 @@ void CampaignReader::InitTransports() helper::CreateDirectory(localPath); for (auto &bpf : ds.files) { - std::cout << " save file " << remotePath << "/" << bpf.name - << " to " << localPath << "/" << bpf.name << "\n"; + /*std::cout << " save file " << remotePath << "/" << + bpf.name + << " to " << localPath << "/" << bpf.name << "\n";*/ SaveToFile(m_DB, localPath + PathSeparator + bpf.name, bpf); } } From a1732a77571c5fba5bd321c8f8c64bd62b901c30 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Wed, 12 Jul 2023 14:30:32 -0400 Subject: [PATCH 24/58] add missing file --- .../adios2/engine/campaign/CampaignData.cpp | 382 ++++++++++++++++++ 1 file changed, 382 insertions(+) create mode 100644 source/adios2/engine/campaign/CampaignData.cpp diff --git a/source/adios2/engine/campaign/CampaignData.cpp b/source/adios2/engine/campaign/CampaignData.cpp new file mode 100644 index 0000000000..847221afff --- /dev/null +++ b/source/adios2/engine/campaign/CampaignData.cpp @@ -0,0 +1,382 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * CampaignData.cpp + * Campaign data struct + * + * Created on: May 16, 2023 + * Author: Norbert Podhorszki pnorbert@ornl.gov + */ + +#include "CampaignData.h" +#include "adios2/helper/adiosLog.h" +#include "adios2/helper/adiosString.h" + +#include +#include +#include +#include +#include +#include + +namespace adios2 +{ +namespace core +{ +namespace engine +{ + +/* + * Data from processes to be recorded + */ + +static int sqlcb_info(void *p, int argc, char **argv, char **azColName) +{ + CampaignData *cdp = reinterpret_cast(p); + cdp->version = std::string(argv[0]); + return 0; +}; + +static int sqlcb_host(void *p, int argc, char **argv, char **azColName) +{ + CampaignData *cdp = reinterpret_cast(p); + CampaignHost ch; + /* + std::cout << "SQL: host record: "; + for (int i = 0; i < argc; i++) + { + std::cout << azColName[i] << " = " << (argv[i] ? argv[i] : "NULL") + << std::endl; + } + std::cout << std::endl; + */ + ch.hostname = std::string(argv[0]); + ch.longhostname = std::string(argv[1]); + cdp->hosts.push_back(ch); + return 0; +}; + +static int sqlcb_directory(void *p, int argc, char **argv, char **azColName) +{ + CampaignData *cdp = reinterpret_cast(p); + size_t hostid = helper::StringToSizeT(std::string(argv[0]), + "SQL callback convert text to int"); + size_t hostidx = + hostid - 1; // SQL rows start from 1, vector idx start from 0 + cdp->hosts[hostidx].directory.push_back(argv[1]); + return 0; +}; + +static int sqlcb_bpdataset(void *p, int argc, char **argv, char **azColName) +{ + CampaignData *cdp = reinterpret_cast(p); + CampaignBPDataset cds; + size_t hostid = helper::StringToSizeT(std::string(argv[0]), + "SQL callback convert text to int"); + size_t dirid = helper::StringToSizeT(std::string(argv[0]), + "SQL callback convert text to int"); + cds.hostIdx = hostid - 1; // SQL rows start from 1, vector idx start from 0 + cds.dirIdx = dirid - 1; + cds.name = argv[2]; + cdp->bpdatasets.push_back(cds); + return 0; +}; + +static int sqlcb_bpfile(void *p, int argc, char **argv, char **azColName) +{ + CampaignData *cdp = reinterpret_cast(p); + CampaignBPFile cf; + size_t dsid = helper::StringToSizeT(std::string(argv[0]), + "SQL callback convert text to int"); + cf.bpDatasetIdx = dsid - 1; + cf.name = std::string(argv[1]); + int comp = helper::StringTo(std::string(argv[2]), + "SQL callback convert text to int"); + cf.compressed = (bool)comp; + cf.lengthOriginal = helper::StringToSizeT( + std::string(argv[3]), "SQL callback convert text to int"); + cf.lengthCompressed = helper::StringToSizeT( + std::string(argv[4]), "SQL callback convert text to int"); + cf.ctime = helper::StringTo(std::string(argv[5]), + "SQL callback convert ctime to int"); + + CampaignBPDataset &cds = cdp->bpdatasets[cf.bpDatasetIdx]; + cds.files.push_back(cf); + return 0; +}; + +void ReadCampaignData(sqlite3 *db, CampaignData &cd) +{ + int rc; + char *zErrMsg = 0; + std::string sqlcmd; + + sqlcmd = "SELECT version FROM info"; + rc = sqlite3_exec(db, sqlcmd.c_str(), sqlcb_info, &cd, &zErrMsg); + if (rc != SQLITE_OK) + { + std::cout << "SQL error: " << zErrMsg << std::endl; + std::string m(zErrMsg); + helper::Throw( + "Engine", "CampaignReader", "ReadCampaignData", + "SQL error on reading info records:" + m); + sqlite3_free(zErrMsg); + } + + sqlcmd = "SELECT hostname, longhostname FROM host"; + rc = sqlite3_exec(db, sqlcmd.c_str(), sqlcb_host, &cd, &zErrMsg); + if (rc != SQLITE_OK) + { + std::cout << "SQL error: " << zErrMsg << std::endl; + std::string m(zErrMsg); + helper::Throw( + "Engine", "CampaignReader", "ReadCampaignData", + "SQL error on reading host records:" + m); + sqlite3_free(zErrMsg); + } + + sqlcmd = "SELECT hostid, name FROM directory"; + rc = sqlite3_exec(db, sqlcmd.c_str(), sqlcb_directory, &cd, &zErrMsg); + if (rc != SQLITE_OK) + { + std::cout << "SQL error: " << zErrMsg << std::endl; + std::string m(zErrMsg); + helper::Throw( + "Engine", "CampaignReader", "ReadCampaignData", + "SQL error on reading directory records:" + m); + sqlite3_free(zErrMsg); + } + + sqlcmd = "SELECT hostid, dirid, name FROM bpdataset"; + rc = sqlite3_exec(db, sqlcmd.c_str(), sqlcb_bpdataset, &cd, &zErrMsg); + if (rc != SQLITE_OK) + { + std::cout << "SQL error: " << zErrMsg << std::endl; + std::string m(zErrMsg); + helper::Throw( + "Engine", "CampaignReader", "ReadCampaignData", + "SQL error on reading bpdataset records:" + m); + sqlite3_free(zErrMsg); + } + + sqlcmd = + "SELECT bpdatasetid, name, compression, lenorig, lencompressed, ctime " + "FROM bpfile"; + rc = sqlite3_exec(db, sqlcmd.c_str(), sqlcb_bpfile, &cd, &zErrMsg); + if (rc != SQLITE_OK) + { + std::cout << "SQL error: " << zErrMsg << std::endl; + std::string m(zErrMsg); + helper::Throw( + "Engine", "CampaignReader", "ReadCampaignData", + "SQL error on reading bpfile records:" + m); + sqlite3_free(zErrMsg); + } +} + +static int sqlcb_bpfile_data(void *p, int argc, char **argv, char **azColName) +{ + CampaignData *cdp = reinterpret_cast(p); + CampaignBPFile cf; + size_t dsid = helper::StringToSizeT(std::string(argv[0]), + "SQL callback convert text to int"); + cf.bpDatasetIdx = dsid - 1; + cf.name = std::string(argv[1]); + int comp = helper::StringTo(std::string(argv[2]), + "SQL callback convert text to int"); + cf.compressed = (bool)comp; + cf.lengthOriginal = helper::StringToSizeT( + std::string(argv[3]), "SQL callback convert text to int"); + cf.lengthCompressed = helper::StringToSizeT( + std::string(argv[4]), "SQL callback convert text to int"); + + CampaignBPDataset &cds = cdp->bpdatasets[cf.bpDatasetIdx]; + cds.files.push_back(cf); + return 0; +}; + +/* Decompress from in-memory source to file dest until stream ends or EOF. + inf() returns Z_OK on success, Z_MEM_ERROR if memory could not be + allocated for processing, Z_DATA_ERROR if the deflate data is + invalid or incomplete, Z_VERSION_ERROR if the version of zlib.h and + the version of the library linked do not match, or Z_ERRNO if there + is an error reading or writing the files. */ +int inflateToFile(const unsigned char *source, const size_t blobsize, + std::ofstream *dest) +{ + constexpr size_t CHUNK = 16777216; + int ret; + unsigned have; + z_stream strm; + + std::vector out(CHUNK); + + /* allocate inflate state */ + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = 0; + strm.next_in = Z_NULL; + ret = inflateInit(&strm); + if (ret != Z_OK) + return ret; + + /* decompress until deflate stream ends or end of file */ + unsigned char *p = const_cast(source); + std::cout << " Inflate = " << (void *)p << " size = " << blobsize << "\n"; + do + { + strm.avail_in = (uInt)(blobsize > CHUNK ? CHUNK : blobsize); + strm.next_in = p; + + std::cout << " next_in = " << (void *)strm.next_in + << " avail_in = " << strm.avail_in << "\n"; + /* run inflate() on input until output buffer not full */ + do + { + strm.avail_out = CHUNK; + strm.next_out = out.data(); + std::cout << " next_out = " << (void *)strm.next_out + << " avail_out = " << strm.avail_out << "\n"; + ret = inflate(&strm, Z_NO_FLUSH); + switch (ret) + { + case Z_NEED_DICT: + ret = Z_DATA_ERROR; /* and fall through */ + case Z_STREAM_ERROR: + case Z_DATA_ERROR: + case Z_MEM_ERROR: + (void)inflateEnd(&strm); + return ret; + } + have = CHUNK - strm.avail_out; + std::cout << " have = " << have << "\n"; + dest->write(reinterpret_cast(out.data()), have); + + } while (strm.avail_out == 0); + + /* done when inflate() says it's done */ + } while (ret != Z_STREAM_END); + + /* clean up and return */ + (void)inflateEnd(&strm); + return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR; +} + +static long timeToSec(long ct) +{ + long t; + if (ct > 99999999999999999) + { + /* nanosec to sec */ + t = ct / 1000000000; + } + else if (ct > 99999999999999) + { + /* microsec to sec */ + t = ct / 1000000; + } + else if (ct > 99999999999) + { + /* millisec to sec */ + t = ct / 1000; + } + else + { + t = ct; + } + return t; +} + +static bool isFileNewer(const std::string path, long ctime) +{ + int result; +#ifdef _WIN32 + struct _stat s; + result = _stat(path.c_str(), &s); +#else + struct stat s; + result = stat(path.c_str(), &s); +#endif + if (result != 0) + { + return false; + } + + long ct = s.st_ctime; + long ctSec = timeToSec(ct); + long ctimeSec = timeToSec(ctime); + + /*std::cout << " Stat(" << path << "): size = " << s.st_size + << " ct = " << ctSec << " ctime = " << ctimeSec << "\n";*/ + return (ctSec > ctimeSec); +} + +void SaveToFile(sqlite3 *db, const std::string &path, + const CampaignBPFile &bpfile) +{ + if (isFileNewer(path, bpfile.ctime)) + { + return; + } + + int rc; + char *zErrMsg = 0; + std::string sqlcmd; + std::string id = std::to_string(bpfile.bpDatasetIdx + 1); + + sqlite3_stmt *statement; + sqlcmd = "SELECT data FROM bpfile WHERE bpdatasetid = " + id + + " AND name = '" + bpfile.name + "'"; + // std::cout << "SQL statement: " << sqlcmd << "\n"; + rc = + sqlite3_prepare_v2(db, sqlcmd.c_str(), sqlcmd.size(), &statement, NULL); + if (rc != SQLITE_OK) + { + std::cout << "SQL error: " << zErrMsg << std::endl; + std::string m(zErrMsg); + helper::Throw( + "Engine", "CampaignReader", "SaveToFIle", + "SQL error on reading info records:" + m); + sqlite3_free(zErrMsg); + } + + int result = 0; + result = sqlite3_step(statement); + if (result != SQLITE_ROW) + { + helper::Throw( + "Engine", "CampaignReader", "SaveToFIle", + "Did not find record for :" + bpfile.name); + } + + int iBlobsize = sqlite3_column_bytes(statement, 0); + const void *p = sqlite3_column_blob(statement, 0); + + /*std::cout << "-- Retrieved from DB data of " << bpfile.name + << " size = " << iBlobsize + << " compressed = " << bpfile.compressed + << " compressed size = " << bpfile.lengthCompressed + << " original size = " << bpfile.lengthOriginal << " blob = " << p + << "\n";*/ + + size_t blobsize = static_cast(iBlobsize); + std::ofstream f; + f.rdbuf()->pubsetbuf(0, 0); + f.open(path, std::ios::out | std::ios::binary); + if (bpfile.compressed) + { + const unsigned char *ptr = static_cast(p); + inflateToFile(ptr, blobsize, &f); + } + else + { + f.write(static_cast(p), blobsize); + } + f.close(); +} + +} // end namespace engine +} // end namespace core +} // end namespace adios2 From 14225cdd90e1ace9d4ceb8264d40b4da7aa22a83 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Wed, 12 Jul 2023 14:36:17 -0400 Subject: [PATCH 25/58] add another missing file --- .../adios2/engine/campaign/CampaignRecord.h | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 source/adios2/engine/campaign/CampaignRecord.h diff --git a/source/adios2/engine/campaign/CampaignRecord.h b/source/adios2/engine/campaign/CampaignRecord.h new file mode 100644 index 0000000000..e7deb0f78a --- /dev/null +++ b/source/adios2/engine/campaign/CampaignRecord.h @@ -0,0 +1,56 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * CampaignRecord.h + * Campaign record struct + * + * Created on: May 16, 2023 + * Author: Norbert Podhorszki pnorbert@ornl.gov + */ + +#ifndef ADIOS2_ENGINE_CAMPAIGNRECORD_H_ +#define ADIOS2_ENGINE_CAMPAIGNRECORD_H_ + +#include +#include +#include + +#include + +namespace adios2 +{ +namespace core +{ +namespace engine +{ + +/* + * Data from processes to be recorded + */ + +struct CampaignRecord +{ + std::vector steps; + size_t delta_step; + std::vector times; + double delta_time; + bool varying_deltas; + + CampaignRecord() : delta_step(0), delta_time(0.0), varying_deltas(false){}; + + CampaignRecord(size_t step, double time) + : delta_step(0), delta_time(0.0), varying_deltas(false) + { + steps.push_back(step); + times.push_back(time); + }; +}; + +using CampaignRecordMap = std::unordered_map; + +} // end namespace engine +} // end namespace core +} // end namespace adios2 + +#endif /* ADIOS2_ENGINE_CAMPAIGRECORD_H_ */ From f5877bf5b46fab77a17aaea7c89c5fdeb6aa0f11 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Wed, 12 Jul 2023 16:37:43 -0400 Subject: [PATCH 26/58] enable using multiple directories in same host --- .../adios2/engine/campaign/CampaignData.cpp | 2 +- .../adios2/engine/campaign/CampaignManager.h | 2 +- .../adios2/engine/campaign/CampaignReader.cpp | 54 +++++++++++-------- .../adios2_campaign_manager.py | 30 +++++++++-- 4 files changed, 62 insertions(+), 26 deletions(-) diff --git a/source/adios2/engine/campaign/CampaignData.cpp b/source/adios2/engine/campaign/CampaignData.cpp index 847221afff..d22d21370a 100644 --- a/source/adios2/engine/campaign/CampaignData.cpp +++ b/source/adios2/engine/campaign/CampaignData.cpp @@ -74,7 +74,7 @@ static int sqlcb_bpdataset(void *p, int argc, char **argv, char **azColName) CampaignBPDataset cds; size_t hostid = helper::StringToSizeT(std::string(argv[0]), "SQL callback convert text to int"); - size_t dirid = helper::StringToSizeT(std::string(argv[0]), + size_t dirid = helper::StringToSizeT(std::string(argv[1]), "SQL callback convert text to int"); cds.hostIdx = hostid - 1; // SQL rows start from 1, vector idx start from 0 cds.dirIdx = dirid - 1; diff --git a/source/adios2/engine/campaign/CampaignManager.h b/source/adios2/engine/campaign/CampaignManager.h index 209a4449c2..a39c448606 100644 --- a/source/adios2/engine/campaign/CampaignManager.h +++ b/source/adios2/engine/campaign/CampaignManager.h @@ -44,7 +44,7 @@ class CampaignManager bool m_Opened = false; std::string m_Name; int m_WriterRank; - int m_Verbosity = 5; + int m_Verbosity = 0; CampaignRecordMap cmap; std::ofstream m_Output; const std::string m_CampaignDir = "adios-campaign"; diff --git a/source/adios2/engine/campaign/CampaignReader.cpp b/source/adios2/engine/campaign/CampaignReader.cpp index 3a633809ae..00551d738e 100644 --- a/source/adios2/engine/campaign/CampaignReader.cpp +++ b/source/adios2/engine/campaign/CampaignReader.cpp @@ -178,29 +178,35 @@ void CampaignReader::InitTransports() ReadCampaignData(m_DB, m_CampaignData); - std::cout << "Local hostname = " << m_Hostname << "\n"; - std::cout << "Database result:\n version = " << m_CampaignData.version - << "\n hosts:\n"; - - for (size_t hostidx = 0; hostidx < m_CampaignData.hosts.size(); ++hostidx) + if (m_Verbosity == 1) { - CampaignHost &h = m_CampaignData.hosts[hostidx]; - std::cout << " host =" << h.hostname - << " long name = " << h.longhostname << " directories: \n"; - for (size_t diridx = 0; diridx < h.directory.size(); ++diridx) + std::cout << "Local hostname = " << m_Hostname << "\n"; + std::cout << "Database result:\n version = " << m_CampaignData.version + << "\n hosts:\n"; + + for (size_t hostidx = 0; hostidx < m_CampaignData.hosts.size(); + ++hostidx) { - std::cout << " dir = " << h.directory[diridx] << "\n"; + CampaignHost &h = m_CampaignData.hosts[hostidx]; + std::cout << " host =" << h.hostname + << " long name = " << h.longhostname + << " directories: \n"; + for (size_t diridx = 0; diridx < h.directory.size(); ++diridx) + { + std::cout << " dir = " << h.directory[diridx] << "\n"; + } } - } - std::cout << " datasets:\n"; - for (auto &ds : m_CampaignData.bpdatasets) - { - std::cout << " " << m_CampaignData.hosts[ds.hostIdx].hostname << ":" - << m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] - << PathSeparator << ds.name << "\n"; - for (auto &bpf : ds.files) + std::cout << " datasets:\n"; + for (auto &ds : m_CampaignData.bpdatasets) { - std::cout << " file: " << bpf.name << "\n"; + std::cout << " " << m_CampaignData.hosts[ds.hostIdx].hostname + << ":" + << m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + << PathSeparator << ds.name << "\n"; + for (auto &bpf : ds.files) + { + std::cout << " file: " << bpf.name << "\n"; + } } } @@ -218,7 +224,10 @@ void CampaignReader::InitTransports() m_CampaignData.hosts[ds.hostIdx].hostname + ":" + m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + PathSeparator + ds.name; - std::cout << "Open remote file " << remotePath << "\n"; + if (m_Verbosity == 1) + { + std::cout << "Open remote file " << remotePath << "\n"; + } localPath = m_CachePath + PathSeparator + m_CampaignData.hosts[ds.hostIdx].hostname + PathSeparator + ds.name; @@ -235,7 +244,10 @@ void CampaignReader::InitTransports() { localPath = m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + PathSeparator + ds.name; - std::cout << "Open local file " << localPath << "\n"; + if (m_Verbosity == 1) + { + std::cout << "Open local file " << localPath << "\n"; + } } adios2::core::IO &io = diff --git a/source/utils/adios_campaign_manager/adios2_campaign_manager.py b/source/utils/adios_campaign_manager/adios2_campaign_manager.py index 208c182fe2..817c3c14bf 100755 --- a/source/utils/adios_campaign_manager/adios2_campaign_manager.py +++ b/source/utils/adios_campaign_manager/adios2_campaign_manager.py @@ -36,6 +36,10 @@ def SetupArgs(): parser.add_argument("--campaign_store", "-c", help="Path to local campaign store", required=True) + parser.add_argument("--hostname", "-n", + help="Host name unique for hosts in a campaign", + required=False) + args = parser.parse_args() # default values @@ -174,6 +178,21 @@ def GetHostName(): return host, shorthost +def AddHostName(longHostName, shortHostName): + res = cur.execute( + 'select rowid from host where hostname = "' + shortHostName + '"') + row = res.fetchone() + if row != None: + hostID = row[0] + print(f"Found host {shortHostName} in database, rowid = {hostID}") + else: + curHost = cur.execute('insert into host values (?, ?)', + (shortHostName, longHostName)) + hostID = curHost.lastrowid + print(f"Inserted host {shortHostName} into database, rowid = {hostID}") + return hostID + + if __name__ == "__main__": args = SetupArgs() @@ -228,10 +247,15 @@ def GetHostName(): ", PRIMARY KEY (bpdatasetid, name))") longHostName, shortHostName = GetHostName() + if args.hostname != None: + shortHostName = args.hostname + + hostID = AddHostName(longHostName, shortHostName) + rootdir = getcwd() - curHost = cur.execute('insert into host values (?, ?)', - (shortHostName, longHostName)) - hostID = curHost.lastrowid + # curHost = cur.execute('insert into host values (?, ?)', + # (shortHostName, longHostName)) + #hostID = curHost.lastrowid curDir = cur.execute('insert into directory values (?, ?)', (hostID, rootdir)) From 026f915f7fb3a2cf24c60ca183834aa0df47e1e1 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Wed, 12 Jul 2023 16:57:10 -0400 Subject: [PATCH 27/58] fix print --- source/adios2/engine/campaign/CampaignReader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/adios2/engine/campaign/CampaignReader.cpp b/source/adios2/engine/campaign/CampaignReader.cpp index 00551d738e..c2d5434ae6 100644 --- a/source/adios2/engine/campaign/CampaignReader.cpp +++ b/source/adios2/engine/campaign/CampaignReader.cpp @@ -190,7 +190,7 @@ void CampaignReader::InitTransports() CampaignHost &h = m_CampaignData.hosts[hostidx]; std::cout << " host =" << h.hostname << " long name = " << h.longhostname - << " directories: \n"; + << "\n directories: \n"; for (size_t diridx = 0; diridx < h.directory.size(); ++diridx) { std::cout << " dir = " << h.directory[diridx] << "\n"; From 6f53bb26e2a4d29dda29f2bcd960f66dbe9fd2f6 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Wed, 12 Jul 2023 17:50:13 -0400 Subject: [PATCH 28/58] unfix print --- source/adios2/engine/campaign/CampaignReader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/adios2/engine/campaign/CampaignReader.cpp b/source/adios2/engine/campaign/CampaignReader.cpp index c2d5434ae6..00551d738e 100644 --- a/source/adios2/engine/campaign/CampaignReader.cpp +++ b/source/adios2/engine/campaign/CampaignReader.cpp @@ -190,7 +190,7 @@ void CampaignReader::InitTransports() CampaignHost &h = m_CampaignData.hosts[hostidx]; std::cout << " host =" << h.hostname << " long name = " << h.longhostname - << "\n directories: \n"; + << " directories: \n"; for (size_t diridx = 0; diridx < h.directory.size(); ++diridx) { std::cout << " dir = " << h.directory[diridx] << "\n"; From a3eba6df09b5b5c260a0998e3a8a198a0dc9ec42 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Wed, 12 Jul 2023 17:50:23 -0400 Subject: [PATCH 29/58] add info command --- .../adios2_campaign_manager.py | 138 +++++++++++------- 1 file changed, 89 insertions(+), 49 deletions(-) diff --git a/source/utils/adios_campaign_manager/adios2_campaign_manager.py b/source/utils/adios_campaign_manager/adios2_campaign_manager.py index 817c3c14bf..976011681c 100755 --- a/source/utils/adios_campaign_manager/adios2_campaign_manager.py +++ b/source/utils/adios_campaign_manager/adios2_campaign_manager.py @@ -5,6 +5,7 @@ import json import sqlite3 import zlib +from datetime import datetime, timezone from io import StringIO from os import chdir, getcwd, remove, stat from os.path import basename, exists, isdir @@ -21,7 +22,7 @@ def SetupArgs(): parser = argparse.ArgumentParser() parser.add_argument( - "command", help="Command: create/update/delete", choices=['create', 'update', 'delete']) + "command", help="Command: create/update/delete", choices=['create', 'update', 'delete', 'info']) parser.add_argument("--verbose", "-v", help="More verbosity", action="count") parser.add_argument("--project", "-p", @@ -113,7 +114,7 @@ def AddFileToArchive(args: dict, filename: str, cur: sqlite3.Cursor, dsID: int): return statres = stat(filename) - ct = statres.st_ctime_ns + ct = statres.st_ctime cur.execute('insert into bpfile values (?, ?, ?, ?, ?, ?, ?)', (dsID, filename, compressed, len_orig, len_compressed, ct, compressed_data)) @@ -131,7 +132,7 @@ def AddDatasetToArchive(args: dict, dataset: str, cur: sqlite3.Cursor, hostID: i if (IsADIOSDataset(dataset)): print(f"Add dataset {dataset} to archive") statres = stat(dataset) - ct = statres.st_ctime_ns + ct = statres.st_ctime curDS = cur.execute('insert into bpdataset values (?, ?, ?, ?)', (hostID, dirID, dataset, ct)) @@ -160,14 +161,6 @@ def ProcessJsonFile(args: dict, jsonlist: list, cur: sqlite3.Cursor, hostID: int print(f"WARNING: your object is not a dictionary, skip : {entry}") -def MergeJsonFiles(jsonfiles: list): - result = list() - for f1 in jsonfiles: - with open(f1, 'r') as infile: - result.extend(json.load(infile)) - return result - - def GetHostName(): host = getfqdn() if host.startswith("login"): @@ -193,6 +186,83 @@ def AddHostName(longHostName, shortHostName): return hostID +def Info(args: dict, cur: sqlite3.Cursor): + res = cur.execute('select id, name, version, ctime from info') + info = res.fetchone() + t = datetime.fromtimestamp(float(info[3])) + print(f"{info[1]}, version {info[2]}, created on {t}") + + res = cur.execute('select rowid, hostname, longhostname from host') + hosts = res.fetchall() + for host in hosts: + print(f"hostname = {host[1]} longhostname = {host[2]}") + res2 = cur.execute( + 'select rowid, name from directory where hostid = "'+str(host[0])+'"') + dirs = res2.fetchall() + for dir in dirs: + print(f" dir = {dir[1]}") + res3 = cur.execute( + 'select rowid, name, ctime from bpdataset where hostid = "'+str(host[0]) + + '" and dirid = "'+str(dir[0])+'"') + bpdatasets = res2.fetchall() + for bpdataset in bpdatasets: + t = datetime.fromtimestamp(float(bpdataset[2])) + print(f" dataset = {bpdataset[1]} created on {t}") + + +def Update(args: dict, cur: sqlite3.Cursor): + longHostName, shortHostName = GetHostName() + if args.hostname != None: + shortHostName = args.hostname + + hostID = AddHostName(longHostName, shortHostName) + + rootdir = getcwd() + # curHost = cur.execute('insert into host values (?, ?)', + # (shortHostName, longHostName)) + # hostID = curHost.lastrowid + + curDir = cur.execute('insert into directory values (?, ?)', + (hostID, rootdir)) + dirID = curDir.lastrowid + con.commit() + + jsonlist = MergeJsonFiles(jsonFileList) + + print(f"Merged json = {jsonlist}") + ProcessJsonFile(args, jsonlist, cur, hostID, dirID) + + con.commit() + + +def Create(args: dict, cur: sqlite3.Cursor): + epoch = int(time()) + cur.execute( + "create table info(id TEXT, name TEXT, version TEXT, ctime INT)") + cur.execute('insert into info values (?, ?, ?, ?)', + ("ACM", "ADIOS Campaign Archive", ADIOS_ACM_VERSION, epoch)) + cur.execute("create table host" + + "(hostname TEXT PRIMARY KEY, longhostname TEXT)") + cur.execute("create table directory" + + "(hostid INT, name TEXT, PRIMARY KEY (hostid, name))") + cur.execute("create table bpdataset" + + "(hostid INT, dirid INT, name TEXT, ctime INT" + + ", PRIMARY KEY (hostid, dirid, name))") + cur.execute("create table bpfile" + + "(bpdatasetid INT, name TEXT, compression INT, lenorig INT" + + ", lencompressed INT, ctime INT, data BLOB" + + ", PRIMARY KEY (bpdatasetid, name))") + Update(args, cur) + + +def MergeJsonFiles(jsonfiles: list): + result = list() + for f1 in jsonfiles: + with open(f1, 'r') as infile: + result.extend(json.load(infile)) + return result + + if __name__ == "__main__": args = SetupArgs() @@ -220,53 +290,23 @@ def AddHostName(longHostName, shortHostName): if exists(args.CampaignFileName): print(f"ERROR: archive {args.CampaignFileName} already exist") exit(1) - elif (args.command == "update"): - print("Update archive") + elif (args.command == "update" or args.command == 'info'): + print(f"{args.command} archive") if not exists(args.CampaignFileName): print(f"ERROR: archive {args.CampaignFileName} does not exist") exit(1) con = sqlite3.connect(args.CampaignFileName) cur = con.cursor() - if (args.command == "create"): - epoch = int(time()) - cur.execute( - "create table info(id TEXT, name TEXT, version TEXT, ctime INT)") - cur.execute('insert into info values (?, ?, ?, ?)', - ("ACM", "ADIOS Campaign Archive", ADIOS_ACM_VERSION, epoch)) - cur.execute("create table host" + - "(hostname TEXT PRIMARY KEY, longhostname TEXT)") - cur.execute("create table directory" + - "(hostid INT, name TEXT, PRIMARY KEY (hostid, name))") - cur.execute("create table bpdataset" + - "(hostid INT, dirid INT, name TEXT, ctime INT" + - ", PRIMARY KEY (hostid, dirid, name))") - cur.execute("create table bpfile" + - "(bpdatasetid INT, name TEXT, compression INT, lenorig INT" + - ", lencompressed INT, ctime INT, data BLOB" + - ", PRIMARY KEY (bpdatasetid, name))") - longHostName, shortHostName = GetHostName() - if args.hostname != None: - shortHostName = args.hostname + if (args.command == "info"): + Info(args, cur) - hostID = AddHostName(longHostName, shortHostName) + elif (args.command == "create"): + Create(args, cur) - rootdir = getcwd() - # curHost = cur.execute('insert into host values (?, ?)', - # (shortHostName, longHostName)) - #hostID = curHost.lastrowid - - curDir = cur.execute('insert into directory values (?, ?)', - (hostID, rootdir)) - dirID = curDir.lastrowid - con.commit() - - jsonlist = MergeJsonFiles(jsonFileList) - - print(f"Merged json = {jsonlist}") - ProcessJsonFile(args, jsonlist, cur, hostID, dirID) + elif (args.command == "update"): + Update(args, cur) - con.commit() cur.close() con.close() From 0730209be781a49040191f645230c6297ac7c57b Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Wed, 12 Jul 2023 18:08:28 -0400 Subject: [PATCH 30/58] don't check campaign files when running info --- .../adios2_campaign_manager.py | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/source/utils/adios_campaign_manager/adios2_campaign_manager.py b/source/utils/adios_campaign_manager/adios2_campaign_manager.py index 976011681c..8b310c52b7 100755 --- a/source/utils/adios_campaign_manager/adios2_campaign_manager.py +++ b/source/utils/adios_campaign_manager/adios2_campaign_manager.py @@ -277,14 +277,6 @@ def MergeJsonFiles(jsonfiles: list): print(f"ERROR: archive {args.CampaignFileName} does not exist") exit(1) - CheckLocalCampaignDir(args) - - # List the local campaign directory - jsonFileList = glob.glob(args.LocalCampaignDir + '/*.json') - if len(jsonFileList) == 0: - print("There are no campaign data files in " + args.LocalCampaignDir) - exit(2) - if (args.command == "create"): print("Create archive") if exists(args.CampaignFileName): @@ -301,12 +293,18 @@ def MergeJsonFiles(jsonfiles: list): if (args.command == "info"): Info(args, cur) - - elif (args.command == "create"): - Create(args, cur) - - elif (args.command == "update"): - Update(args, cur) + else: + CheckLocalCampaignDir(args) + # List the local campaign directory + jsonFileList = glob.glob(args.LocalCampaignDir + '/*.json') + if len(jsonFileList) == 0: + print("There are no campaign data files in " + args.LocalCampaignDir) + exit(2) + + if (args.command == "create"): + Create(args, cur) + elif (args.command == "update"): + Update(args, cur) cur.close() con.close() From 334eeca14efac08075e540620bbff5de2ac3c774 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Mon, 31 Jul 2023 09:56:53 -0400 Subject: [PATCH 31/58] compile some functions unconditionally on all platforms --- source/adios2/helper/adiosNetwork.cpp | 119 ++++++++++++++------------ source/adios2/helper/adiosNetwork.h | 29 ++++--- 2 files changed, 78 insertions(+), 70 deletions(-) diff --git a/source/adios2/helper/adiosNetwork.cpp b/source/adios2/helper/adiosNetwork.cpp index 3451d6752a..33872dce3e 100644 --- a/source/adios2/helper/adiosNetwork.cpp +++ b/source/adios2/helper/adiosNetwork.cpp @@ -12,6 +12,13 @@ #include "adios2/helper/adiosComm.h" #include "adios2/toolkit/transport/file/FileFStream.h" +#include // memcpy +#include // gethostname + +#include //getFQDN +#include //getFQDN +#include //getFQDN + #ifndef _WIN32 #if defined(ADIOS2_HAVE_DATAMAN) || defined(ADIOS2_HAVE_TABLE) @@ -21,68 +28,18 @@ #include //AvailableIpAddresses() inet_ntoa #include //AvailableIpAddresses() struct if_nameindex #include //AvailableIpAddresses() struct sockaddr_in -#include //AvailableIpAddresses() strncp #include //AvailableIpAddresses() ioctl -#include //AvailableIpAddresses() close - -#include //getFQDN -#include //getFQDN -#include //getFQDN #include +#endif // ADIOS2_HAVE_DATAMAN || ADIOS2_HAVE_TABLE +#endif // _WIN32 + namespace adios2 { namespace helper { -#if defined(__clang__) -#if __has_feature(memory_sanitizer) -// Memory Sanitizer fails to recognize that if_nameindex initializes -// the memory in the array behind the pointer it returns. -__attribute__((no_sanitize("memory"))) -#endif -#endif -std::vector -AvailableIpAddresses() noexcept -{ - std::vector ips; - int socket_handler = -1; - if ((socket_handler = socket(PF_INET, SOCK_DGRAM, 0)) < 0) - { - return ips; - } - struct if_nameindex *head = if_nameindex(); - if (!head) - { - close(socket_handler); - return ips; - } - for (struct if_nameindex *p = head; !(p->if_index == 0 && p->if_name == NULL); ++p) - { - struct ifreq req; - strncpy(req.ifr_name, p->if_name, IFNAMSIZ); - if (ioctl(socket_handler, SIOCGIFADDR, &req) < 0) - { - if (errno == EADDRNOTAVAIL) - { - continue; - } - if_freenameindex(head); - close(socket_handler); - return ips; - } - const std::string ip = inet_ntoa(((struct sockaddr_in *)&req.ifr_addr)->sin_addr); - if (ip != "127.0.0.1") - { - ips.emplace_back(ip); - } - } - if_freenameindex(head); - close(socket_handler); - return ips; -} - std::string GetFQDN() noexcept { char hostname[1024]; @@ -148,6 +105,56 @@ std::string GetClusterName() noexcept return fqdn.substr(0, fqdn.find('.')); } +#ifndef _WIN32 +#if defined(ADIOS2_HAVE_DATAMAN) || defined(ADIOS2_HAVE_TABLE) + +#if defined(__clang__) +#if __has_feature(memory_sanitizer) +// Memory Sanitizer fails to recognize that if_nameindex initializes +// the memory in the array behind the pointer it returns. +__attribute__((no_sanitize("memory"))) +#endif +#endif +std::vector +AvailableIpAddresses() noexcept +{ + std::vector ips; + int socket_handler = -1; + if ((socket_handler = socket(PF_INET, SOCK_DGRAM, 0)) < 0) + { + return ips; + } + struct if_nameindex *head = if_nameindex(); + if (!head) + { + close(socket_handler); + return ips; + } + for (struct if_nameindex *p = head; !(p->if_index == 0 && p->if_name == NULL); ++p) + { + struct ifreq req; + strncpy(req.ifr_name, p->if_name, IFNAMSIZ); + if (ioctl(socket_handler, SIOCGIFADDR, &req) < 0) + { + if (errno == EADDRNOTAVAIL) + { + continue; + } + if_freenameindex(head); + close(socket_handler); + return ips; + } + const std::string ip = inet_ntoa(((struct sockaddr_in *)&req.ifr_addr)->sin_addr); + if (ip != "127.0.0.1") + { + ips.emplace_back(ip); + } + } + if_freenameindex(head); + close(socket_handler); + return ips; +} + void HandshakeWriter(Comm const &comm, size_t &appID, std::vector &fullAddresses, const std::string &name, const std::string &engineName, const int basePort, const int channelsPerRank, const int maxRanksPerNode, const int maxAppsPerNode) @@ -308,8 +315,8 @@ void HandshakeReader(Comm const &comm, size_t &appID, std::vector & fullAddresses = j.get>(); } +#endif // ADIOS2_HAVE_DATAMAN || ADIOS2_HAVE_TABLE +#endif // _WIN32 + } // end namespace helper } // end namespace adios2 - -#endif -#endif diff --git a/source/adios2/helper/adiosNetwork.h b/source/adios2/helper/adiosNetwork.h index 9c503b512f..2b349e1387 100644 --- a/source/adios2/helper/adiosNetwork.h +++ b/source/adios2/helper/adiosNetwork.h @@ -12,12 +12,6 @@ #ifndef ADIOS2_HELPER_ADIOSNETWORK_H_ #define ADIOS2_HELPER_ADIOSNETWORK_H_ -#ifndef _WIN32 -// The function implementations use which does not -// work with all of the compilers we support. Provide these functions -// only when one of the engines that needs them is enabled. -#if defined(ADIOS2_HAVE_DATAMAN) || defined(ADIOS2_HAVE_TABLE) - /// \cond EXCLUDE_FROM_DOXYGEN #include #include @@ -30,12 +24,6 @@ namespace helper class Comm; -/** - * returns a vector of strings with all available IP addresses on the node - * @return vector of strings - */ -std::vector AvailableIpAddresses() noexcept; - /** * returns the (a) fully qualified domain name of the current machine. * the result is "Unknown_Host_Name" if fqdn is not found @@ -49,6 +37,18 @@ std::string GetFQDN() noexcept; */ std::string GetClusterName() noexcept; +#ifndef _WIN32 +// The function implementations use which does not +// work with all of the compilers we support. Provide these functions +// only when one of the engines that needs them is enabled. +#if defined(ADIOS2_HAVE_DATAMAN) || defined(ADIOS2_HAVE_TABLE) + +/** + * returns a vector of strings with all available IP addresses on the node + * @return vector of strings + */ +std::vector AvailableIpAddresses() noexcept; + void HandshakeWriter(Comm const &comm, size_t &appID, std::vector &fullAddresses, const std::string &name, const std::string &engineName, const int basePort, const int channelsPerRank, const int maxRanksPerNode = 100, @@ -57,9 +57,10 @@ void HandshakeWriter(Comm const &comm, size_t &appID, std::vector & void HandshakeReader(Comm const &comm, size_t &appID, std::vector &fullAddresses, const std::string &name, const std::string &engineName); +#endif // ADIOS2_HAVE_DATAMAN || ADIOS2_HAVE_TABLE +#endif // _WIN32 + } // end namespace helper } // end namespace adios2 -#endif // ADIOS2_HAVE_DATAMAN || ADIOS2_HAVE_TABLE -#endif // _WIN32 #endif // ADIOS2_HELPER_ADIOSNETWORK_H_ From 6049c7aee8e78baf012003b16b85c4841636159e Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 8 Aug 2023 07:05:17 -0400 Subject: [PATCH 32/58] flake the campaign-manager python script --- .../adios2_campaign_manager.py | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/source/utils/adios_campaign_manager/adios2_campaign_manager.py b/source/utils/adios_campaign_manager/adios2_campaign_manager.py index 8b310c52b7..12d70e60d6 100755 --- a/source/utils/adios_campaign_manager/adios2_campaign_manager.py +++ b/source/utils/adios_campaign_manager/adios2_campaign_manager.py @@ -5,13 +5,11 @@ import json import sqlite3 import zlib -from datetime import datetime, timezone -from io import StringIO +from datetime import datetime from os import chdir, getcwd, remove, stat -from os.path import basename, exists, isdir +from os.path import exists, isdir from re import sub from socket import getfqdn -from subprocess import check_call from time import time # from adios2.adios2_campaign_manager import * @@ -22,7 +20,8 @@ def SetupArgs(): parser = argparse.ArgumentParser() parser.add_argument( - "command", help="Command: create/update/delete", choices=['create', 'update', 'delete', 'info']) + "command", help="Command: create/update/delete", + choices=['create', 'update', 'delete', 'info']) parser.add_argument("--verbose", "-v", help="More verbosity", action="count") parser.add_argument("--project", "-p", @@ -45,8 +44,8 @@ def SetupArgs(): # default values args.update = False - args.CampaignFileName = args.campaign_store+"/" + \ - args.project+"_"+args.app+"_"+args.shot+".acm" + args.CampaignFileName = args.campaign_store + "/" + \ + args.project + "_" + args.app + "_" + args.shot + ".acm" args.LocalCampaignDir = "adios-campaign/" # print("Verbosity = {0}".format(args.verbose)) @@ -71,9 +70,9 @@ def CheckLocalCampaignDir(args): def IsADIOSDataset(dataset): if not isdir(dataset): return False - if not exists(dataset+"/"+"md.idx"): + if not exists(dataset + "/" + "md.idx"): return False - if not exists(dataset+"/"+"data.0"): + if not exists(dataset + "/" + "data.0"): return False return True @@ -141,7 +140,7 @@ def AddDatasetToArchive(args: dict, dataset: str, cur: sqlite3.Cursor, hostID: i chdir(dataset) mdFileList = glob.glob('*md.*') profileList = glob.glob('profiling.json') - files = mdFileList+profileList + files = mdFileList + profileList for f in files: AddFileToArchive(args, f, cur, dsID) chdir(cwd) @@ -175,7 +174,7 @@ def AddHostName(longHostName, shortHostName): res = cur.execute( 'select rowid from host where hostname = "' + shortHostName + '"') row = res.fetchone() - if row != None: + if row is not None: hostID = row[0] print(f"Found host {shortHostName} in database, rowid = {hostID}") else: @@ -197,14 +196,14 @@ def Info(args: dict, cur: sqlite3.Cursor): for host in hosts: print(f"hostname = {host[1]} longhostname = {host[2]}") res2 = cur.execute( - 'select rowid, name from directory where hostid = "'+str(host[0])+'"') + 'select rowid, name from directory where hostid = "' + str(host[0]) + '"') dirs = res2.fetchall() for dir in dirs: print(f" dir = {dir[1]}") res3 = cur.execute( - 'select rowid, name, ctime from bpdataset where hostid = "'+str(host[0]) + - '" and dirid = "'+str(dir[0])+'"') - bpdatasets = res2.fetchall() + 'select rowid, name, ctime from bpdataset where hostid = "' + str(host[0]) + + '" and dirid = "' + str(dir[0]) + '"') + bpdatasets = res3.fetchall() for bpdataset in bpdatasets: t = datetime.fromtimestamp(float(bpdataset[2])) print(f" dataset = {bpdataset[1]} created on {t}") @@ -212,7 +211,7 @@ def Info(args: dict, cur: sqlite3.Cursor): def Update(args: dict, cur: sqlite3.Cursor): longHostName, shortHostName = GetHostName() - if args.hostname != None: + if args.hostname is not None: shortHostName = args.hostname hostID = AddHostName(longHostName, shortHostName) From 8d2e6172095950212171e524a68a025916d53ce5 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 8 Aug 2023 07:11:38 -0400 Subject: [PATCH 33/58] do not convert all parameter values to lower case. Full paths may contain upper case for a reason. --- source/adios2/engine/bp5/BP5Engine.cpp | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/source/adios2/engine/bp5/BP5Engine.cpp b/source/adios2/engine/bp5/BP5Engine.cpp index ce489b2fbf..115d2ce688 100644 --- a/source/adios2/engine/bp5/BP5Engine.cpp +++ b/source/adios2/engine/bp5/BP5Engine.cpp @@ -139,8 +139,7 @@ void BP5Engine::ParseParams(IO &io, struct BP5Params &Params) for (auto &p : io.m_Parameters) { const std::string key = helper::LowerCase(p.first); - const std::string value = helper::LowerCase(p.second); - params_lowercase[key] = value; + params_lowercase[key] = p.second; } auto lf_SetBoolParameter = [&](const std::string key, bool ¶meter, bool def) { @@ -240,8 +239,7 @@ void BP5Engine::ParseParams(IO &io, struct BP5Params &Params) parameter = def; if (itKey != params_lowercase.end()) { - std::string value = itKey->second; - std::transform(value.begin(), value.end(), value.begin(), ::tolower); + const std::string value = helper::LowerCase(itKey->second); if (value == "malloc") { parameter = (int)BufferVType::MallocVType; @@ -266,8 +264,7 @@ void BP5Engine::ParseParams(IO &io, struct BP5Params &Params) parameter = def; if (itKey != params_lowercase.end()) { - std::string value = itKey->second; - std::transform(value.begin(), value.end(), value.begin(), ::tolower); + const std::string value = helper::LowerCase(itKey->second); if (value == "everyonewrites" || value == "auto") { parameter = (int)AggregationType::EveryoneWrites; @@ -297,8 +294,7 @@ void BP5Engine::ParseParams(IO &io, struct BP5Params &Params) parameter = def; if (itKey != params_lowercase.end()) { - std::string value = itKey->second; - std::transform(value.begin(), value.end(), value.begin(), ::tolower); + const std::string value = helper::LowerCase(itKey->second); if (value == "guided" || value == "auto" || value == "on" || value == "true") { parameter = (int)AsyncWrite::Guided; From ffcb09bcc062d7ca5b809cd977efdb930144787d Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 8 Aug 2023 08:50:02 -0400 Subject: [PATCH 34/58] Use RemoteDataPath parameter for BP5 to open remote data files. clang-format campaign source files. --- .../adios2/engine/campaign/CampaignData.cpp | 100 ++++----- source/adios2/engine/campaign/CampaignData.h | 3 +- .../adios2/engine/campaign/CampaignReader.cpp | 194 ++++++++---------- .../adios2/engine/campaign/CampaignReader.h | 40 ++-- .../adios2/engine/campaign/CampaignReader.tcc | 8 +- .../adios2/engine/campaign/CampaignRecord.h | 3 +- 6 files changed, 147 insertions(+), 201 deletions(-) diff --git a/source/adios2/engine/campaign/CampaignData.cpp b/source/adios2/engine/campaign/CampaignData.cpp index d22d21370a..9c056135b0 100644 --- a/source/adios2/engine/campaign/CampaignData.cpp +++ b/source/adios2/engine/campaign/CampaignData.cpp @@ -60,10 +60,8 @@ static int sqlcb_host(void *p, int argc, char **argv, char **azColName) static int sqlcb_directory(void *p, int argc, char **argv, char **azColName) { CampaignData *cdp = reinterpret_cast(p); - size_t hostid = helper::StringToSizeT(std::string(argv[0]), - "SQL callback convert text to int"); - size_t hostidx = - hostid - 1; // SQL rows start from 1, vector idx start from 0 + size_t hostid = helper::StringToSizeT(std::string(argv[0]), "SQL callback convert text to int"); + size_t hostidx = hostid - 1; // SQL rows start from 1, vector idx start from 0 cdp->hosts[hostidx].directory.push_back(argv[1]); return 0; }; @@ -72,10 +70,8 @@ static int sqlcb_bpdataset(void *p, int argc, char **argv, char **azColName) { CampaignData *cdp = reinterpret_cast(p); CampaignBPDataset cds; - size_t hostid = helper::StringToSizeT(std::string(argv[0]), - "SQL callback convert text to int"); - size_t dirid = helper::StringToSizeT(std::string(argv[1]), - "SQL callback convert text to int"); + size_t hostid = helper::StringToSizeT(std::string(argv[0]), "SQL callback convert text to int"); + size_t dirid = helper::StringToSizeT(std::string(argv[1]), "SQL callback convert text to int"); cds.hostIdx = hostid - 1; // SQL rows start from 1, vector idx start from 0 cds.dirIdx = dirid - 1; cds.name = argv[2]; @@ -87,19 +83,16 @@ static int sqlcb_bpfile(void *p, int argc, char **argv, char **azColName) { CampaignData *cdp = reinterpret_cast(p); CampaignBPFile cf; - size_t dsid = helper::StringToSizeT(std::string(argv[0]), - "SQL callback convert text to int"); + size_t dsid = helper::StringToSizeT(std::string(argv[0]), "SQL callback convert text to int"); cf.bpDatasetIdx = dsid - 1; cf.name = std::string(argv[1]); - int comp = helper::StringTo(std::string(argv[2]), - "SQL callback convert text to int"); + int comp = helper::StringTo(std::string(argv[2]), "SQL callback convert text to int"); cf.compressed = (bool)comp; - cf.lengthOriginal = helper::StringToSizeT( - std::string(argv[3]), "SQL callback convert text to int"); - cf.lengthCompressed = helper::StringToSizeT( - std::string(argv[4]), "SQL callback convert text to int"); - cf.ctime = helper::StringTo(std::string(argv[5]), - "SQL callback convert ctime to int"); + cf.lengthOriginal = + helper::StringToSizeT(std::string(argv[3]), "SQL callback convert text to int"); + cf.lengthCompressed = + helper::StringToSizeT(std::string(argv[4]), "SQL callback convert text to int"); + cf.ctime = helper::StringTo(std::string(argv[5]), "SQL callback convert ctime to int"); CampaignBPDataset &cds = cdp->bpdatasets[cf.bpDatasetIdx]; cds.files.push_back(cf); @@ -118,9 +111,8 @@ void ReadCampaignData(sqlite3 *db, CampaignData &cd) { std::cout << "SQL error: " << zErrMsg << std::endl; std::string m(zErrMsg); - helper::Throw( - "Engine", "CampaignReader", "ReadCampaignData", - "SQL error on reading info records:" + m); + helper::Throw("Engine", "CampaignReader", "ReadCampaignData", + "SQL error on reading info records:" + m); sqlite3_free(zErrMsg); } @@ -130,9 +122,8 @@ void ReadCampaignData(sqlite3 *db, CampaignData &cd) { std::cout << "SQL error: " << zErrMsg << std::endl; std::string m(zErrMsg); - helper::Throw( - "Engine", "CampaignReader", "ReadCampaignData", - "SQL error on reading host records:" + m); + helper::Throw("Engine", "CampaignReader", "ReadCampaignData", + "SQL error on reading host records:" + m); sqlite3_free(zErrMsg); } @@ -142,9 +133,8 @@ void ReadCampaignData(sqlite3 *db, CampaignData &cd) { std::cout << "SQL error: " << zErrMsg << std::endl; std::string m(zErrMsg); - helper::Throw( - "Engine", "CampaignReader", "ReadCampaignData", - "SQL error on reading directory records:" + m); + helper::Throw("Engine", "CampaignReader", "ReadCampaignData", + "SQL error on reading directory records:" + m); sqlite3_free(zErrMsg); } @@ -154,23 +144,20 @@ void ReadCampaignData(sqlite3 *db, CampaignData &cd) { std::cout << "SQL error: " << zErrMsg << std::endl; std::string m(zErrMsg); - helper::Throw( - "Engine", "CampaignReader", "ReadCampaignData", - "SQL error on reading bpdataset records:" + m); + helper::Throw("Engine", "CampaignReader", "ReadCampaignData", + "SQL error on reading bpdataset records:" + m); sqlite3_free(zErrMsg); } - sqlcmd = - "SELECT bpdatasetid, name, compression, lenorig, lencompressed, ctime " - "FROM bpfile"; + sqlcmd = "SELECT bpdatasetid, name, compression, lenorig, lencompressed, ctime " + "FROM bpfile"; rc = sqlite3_exec(db, sqlcmd.c_str(), sqlcb_bpfile, &cd, &zErrMsg); if (rc != SQLITE_OK) { std::cout << "SQL error: " << zErrMsg << std::endl; std::string m(zErrMsg); - helper::Throw( - "Engine", "CampaignReader", "ReadCampaignData", - "SQL error on reading bpfile records:" + m); + helper::Throw("Engine", "CampaignReader", "ReadCampaignData", + "SQL error on reading bpfile records:" + m); sqlite3_free(zErrMsg); } } @@ -179,17 +166,15 @@ static int sqlcb_bpfile_data(void *p, int argc, char **argv, char **azColName) { CampaignData *cdp = reinterpret_cast(p); CampaignBPFile cf; - size_t dsid = helper::StringToSizeT(std::string(argv[0]), - "SQL callback convert text to int"); + size_t dsid = helper::StringToSizeT(std::string(argv[0]), "SQL callback convert text to int"); cf.bpDatasetIdx = dsid - 1; cf.name = std::string(argv[1]); - int comp = helper::StringTo(std::string(argv[2]), - "SQL callback convert text to int"); + int comp = helper::StringTo(std::string(argv[2]), "SQL callback convert text to int"); cf.compressed = (bool)comp; - cf.lengthOriginal = helper::StringToSizeT( - std::string(argv[3]), "SQL callback convert text to int"); - cf.lengthCompressed = helper::StringToSizeT( - std::string(argv[4]), "SQL callback convert text to int"); + cf.lengthOriginal = + helper::StringToSizeT(std::string(argv[3]), "SQL callback convert text to int"); + cf.lengthCompressed = + helper::StringToSizeT(std::string(argv[4]), "SQL callback convert text to int"); CampaignBPDataset &cds = cdp->bpdatasets[cf.bpDatasetIdx]; cds.files.push_back(cf); @@ -202,8 +187,7 @@ static int sqlcb_bpfile_data(void *p, int argc, char **argv, char **azColName) invalid or incomplete, Z_VERSION_ERROR if the version of zlib.h and the version of the library linked do not match, or Z_ERRNO if there is an error reading or writing the files. */ -int inflateToFile(const unsigned char *source, const size_t blobsize, - std::ofstream *dest) +int inflateToFile(const unsigned char *source, const size_t blobsize, std::ofstream *dest) { constexpr size_t CHUNK = 16777216; int ret; @@ -230,8 +214,8 @@ int inflateToFile(const unsigned char *source, const size_t blobsize, strm.avail_in = (uInt)(blobsize > CHUNK ? CHUNK : blobsize); strm.next_in = p; - std::cout << " next_in = " << (void *)strm.next_in - << " avail_in = " << strm.avail_in << "\n"; + std::cout << " next_in = " << (void *)strm.next_in << " avail_in = " << strm.avail_in + << "\n"; /* run inflate() on input until output buffer not full */ do { @@ -313,8 +297,7 @@ static bool isFileNewer(const std::string path, long ctime) return (ctSec > ctimeSec); } -void SaveToFile(sqlite3 *db, const std::string &path, - const CampaignBPFile &bpfile) +void SaveToFile(sqlite3 *db, const std::string &path, const CampaignBPFile &bpfile) { if (isFileNewer(path, bpfile.ctime)) { @@ -327,18 +310,16 @@ void SaveToFile(sqlite3 *db, const std::string &path, std::string id = std::to_string(bpfile.bpDatasetIdx + 1); sqlite3_stmt *statement; - sqlcmd = "SELECT data FROM bpfile WHERE bpdatasetid = " + id + - " AND name = '" + bpfile.name + "'"; + sqlcmd = + "SELECT data FROM bpfile WHERE bpdatasetid = " + id + " AND name = '" + bpfile.name + "'"; // std::cout << "SQL statement: " << sqlcmd << "\n"; - rc = - sqlite3_prepare_v2(db, sqlcmd.c_str(), sqlcmd.size(), &statement, NULL); + rc = sqlite3_prepare_v2(db, sqlcmd.c_str(), sqlcmd.size(), &statement, NULL); if (rc != SQLITE_OK) { std::cout << "SQL error: " << zErrMsg << std::endl; std::string m(zErrMsg); - helper::Throw( - "Engine", "CampaignReader", "SaveToFIle", - "SQL error on reading info records:" + m); + helper::Throw("Engine", "CampaignReader", "SaveToFIle", + "SQL error on reading info records:" + m); sqlite3_free(zErrMsg); } @@ -346,9 +327,8 @@ void SaveToFile(sqlite3 *db, const std::string &path, result = sqlite3_step(statement); if (result != SQLITE_ROW) { - helper::Throw( - "Engine", "CampaignReader", "SaveToFIle", - "Did not find record for :" + bpfile.name); + helper::Throw("Engine", "CampaignReader", "SaveToFIle", + "Did not find record for :" + bpfile.name); } int iBlobsize = sqlite3_column_bytes(statement, 0); diff --git a/source/adios2/engine/campaign/CampaignData.h b/source/adios2/engine/campaign/CampaignData.h index 5150fe487c..8d360abfa4 100644 --- a/source/adios2/engine/campaign/CampaignData.h +++ b/source/adios2/engine/campaign/CampaignData.h @@ -61,8 +61,7 @@ struct CampaignData void ReadCampaignData(sqlite3 *db, CampaignData &cd); -void SaveToFile(sqlite3 *db, const std::string &path, - const CampaignBPFile &bpfile); +void SaveToFile(sqlite3 *db, const std::string &path, const CampaignBPFile &bpfile); } // end namespace engine } // end namespace core diff --git a/source/adios2/engine/campaign/CampaignReader.cpp b/source/adios2/engine/campaign/CampaignReader.cpp index 00551d738e..4a5f7591b1 100644 --- a/source/adios2/engine/campaign/CampaignReader.cpp +++ b/source/adios2/engine/campaign/CampaignReader.cpp @@ -28,16 +28,15 @@ namespace core namespace engine { -CampaignReader::CampaignReader(IO &io, const std::string &name, const Mode mode, - helper::Comm comm) +CampaignReader::CampaignReader(IO &io, const std::string &name, const Mode mode, helper::Comm comm) : Engine("CampaignReader", io, name, mode, std::move(comm)) { m_ReaderRank = m_Comm.Rank(); Init(); if (m_Verbosity == 5) { - std::cout << "Campaign Reader " << m_ReaderRank << " Open(" << m_Name - << ") in constructor." << std::endl; + std::cout << "Campaign Reader " << m_ReaderRank << " Open(" << m_Name << ") in constructor." + << std::endl; } m_IsOpen = true; } @@ -47,8 +46,7 @@ CampaignReader::~CampaignReader() /* CampaignReader destructor does close and finalize */ if (m_Verbosity == 5) { - std::cout << "Campaign Reader " << m_ReaderRank << " destructor on " - << m_Name << "\n"; + std::cout << "Campaign Reader " << m_ReaderRank << " destructor on " << m_Name << "\n"; } if (m_IsOpen) { @@ -57,8 +55,7 @@ CampaignReader::~CampaignReader() m_IsOpen = false; } -StepStatus CampaignReader::BeginStep(const StepMode mode, - const float timeoutSeconds) +StepStatus CampaignReader::BeginStep(const StepMode mode, const float timeoutSeconds) { // step info should be received from the writer side in BeginStep() // so this forced increase should not be here @@ -66,8 +63,8 @@ StepStatus CampaignReader::BeginStep(const StepMode mode, if (m_Verbosity == 5) { - std::cout << "Campaign Reader " << m_ReaderRank - << " BeginStep() new step " << m_CurrentStep << "\n"; + std::cout << "Campaign Reader " << m_ReaderRank << " BeginStep() new step " + << m_CurrentStep << "\n"; } // If we reach the end of stream (writer is gone or explicitly tells the @@ -93,8 +90,7 @@ void CampaignReader::PerformGets() { if (m_Verbosity == 5) { - std::cout << "Campaign Reader " << m_ReaderRank - << " PerformGets()\n"; + std::cout << "Campaign Reader " << m_ReaderRank << " PerformGets()\n"; } m_NeedPerformGets = false; } @@ -138,11 +134,10 @@ void CampaignReader::InitParameters() { m_Verbosity = std::stoi(value); if (m_Verbosity < 0 || m_Verbosity > 5) - helper::Throw( - "Engine", "CampaignReader", "InitParameters", - "Method verbose argument must be an " - "integer in the range [0,5], in call to " - "Open or Engine constructor"); + helper::Throw("Engine", "CampaignReader", "InitParameters", + "Method verbose argument must be an " + "integer in the range [0,5], in call to " + "Open or Engine constructor"); } if (key == "hostname") { @@ -172,8 +167,7 @@ void CampaignReader::InitTransports() std::string dbmsg(sqlite3_errmsg(m_DB)); sqlite3_close(m_DB); helper::Throw("Engine", "CampaignReader", "Open", - "Cannot open database" + m_Name + - ": " + dbmsg); + "Cannot open database" + m_Name + ": " + dbmsg); } ReadCampaignData(m_DB, m_CampaignData); @@ -181,15 +175,12 @@ void CampaignReader::InitTransports() if (m_Verbosity == 1) { std::cout << "Local hostname = " << m_Hostname << "\n"; - std::cout << "Database result:\n version = " << m_CampaignData.version - << "\n hosts:\n"; + std::cout << "Database result:\n version = " << m_CampaignData.version << "\n hosts:\n"; - for (size_t hostidx = 0; hostidx < m_CampaignData.hosts.size(); - ++hostidx) + for (size_t hostidx = 0; hostidx < m_CampaignData.hosts.size(); ++hostidx) { CampaignHost &h = m_CampaignData.hosts[hostidx]; - std::cout << " host =" << h.hostname - << " long name = " << h.longhostname + std::cout << " host =" << h.hostname << " long name = " << h.longhostname << " directories: \n"; for (size_t diridx = 0; diridx < h.directory.size(); ++diridx) { @@ -199,10 +190,9 @@ void CampaignReader::InitTransports() std::cout << " datasets:\n"; for (auto &ds : m_CampaignData.bpdatasets) { - std::cout << " " << m_CampaignData.hosts[ds.hostIdx].hostname - << ":" - << m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] - << PathSeparator << ds.name << "\n"; + std::cout << " " << m_CampaignData.hosts[ds.hostIdx].hostname << ":" + << m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] << PathSeparator + << ds.name << "\n"; for (auto &bpf : ds.files) { std::cout << " file: " << bpf.name << "\n"; @@ -217,51 +207,48 @@ void CampaignReader::InitTransports() int i = 0; for (auto &ds : m_CampaignData.bpdatasets) { + adios2::core::IO &io = m_IO.m_ADIOS.DeclareIO("CampaignReader" + std::to_string(i)); std::string localPath; if (m_CampaignData.hosts[ds.hostIdx].hostname != m_Hostname) { - std::string remotePath = - m_CampaignData.hosts[ds.hostIdx].hostname + ":" + - m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + - PathSeparator + ds.name; + const std::string remotePath = + m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + PathSeparator + ds.name; + const std::string remoteURL = + m_CampaignData.hosts[ds.hostIdx].hostname + ":" + remotePath; if (m_Verbosity == 1) { - std::cout << "Open remote file " << remotePath << "\n"; + std::cout << "Open remote file " << remoteURL << "\n"; } - localPath = m_CachePath + PathSeparator + - m_CampaignData.hosts[ds.hostIdx].hostname + + localPath = m_CachePath + PathSeparator + m_CampaignData.hosts[ds.hostIdx].hostname + PathSeparator + ds.name; helper::CreateDirectory(localPath); for (auto &bpf : ds.files) { - /*std::cout << " save file " << remotePath << "/" << + /*std::cout << " save file " << remoteURL << "/" << bpf.name << " to " << localPath << "/" << bpf.name << "\n";*/ SaveToFile(m_DB, localPath + PathSeparator + bpf.name, bpf); } + io.SetParameter("RemoteDataPath", remotePath); } else { - localPath = m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + - PathSeparator + ds.name; + localPath = + m_CampaignData.hosts[ds.hostIdx].directory[ds.dirIdx] + PathSeparator + ds.name; if (m_Verbosity == 1) { std::cout << "Open local file " << localPath << "\n"; } } - adios2::core::IO &io = - m_IO.m_ADIOS.DeclareIO("CampaignReader" + std::to_string(i)); - adios2::core::Engine &e = - io.Open(localPath, m_OpenMode, m_Comm.Duplicate()); + adios2::core::Engine &e = io.Open(localPath, m_OpenMode, m_Comm.Duplicate()); m_IOs.push_back(&io); m_Engines.push_back(&e); auto vmap = io.GetAvailableVariables(); auto amap = io.GetAvailableAttributes(); - VarInternalInfo internalInfo(nullptr, m_IOs.size() - 1, - m_Engines.size() - 1); + VarInternalInfo internalInfo(nullptr, m_IOs.size() - 1, m_Engines.size() - 1); for (auto &vr : vmap) { @@ -274,11 +261,11 @@ void CampaignReader::InitTransports() if (type == DataType::Struct) { } -#define declare_type(T) \ - else if (type == helper::GetDataType()) \ - { \ - Variable *vi = io.InquireVariable(vname); \ - Variable v = DuplicateVariable(vi, m_IO, newname, internalInfo); \ +#define declare_type(T) \ + else if (type == helper::GetDataType()) \ + { \ + Variable *vi = io.InquireVariable(vname); \ + Variable v = DuplicateVariable(vi, m_IO, newname, internalInfo); \ } ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) @@ -292,8 +279,7 @@ void CampaignReader::DoClose(const int transportIndex) { if (m_Verbosity == 5) { - std::cout << "Campaign Reader " << m_ReaderRank << " Close(" << m_Name - << ")\n"; + std::cout << "Campaign Reader " << m_ReaderRank << " Close(" << m_Name << ")\n"; } for (auto ep : m_Engines) { @@ -303,10 +289,7 @@ void CampaignReader::DoClose(const int transportIndex) m_IsOpen = false; } -void CampaignReader::DestructorClose(bool Verbose) noexcept -{ - sqlite3_close(m_DB); -} +void CampaignReader::DestructorClose(bool Verbose) noexcept { sqlite3_close(m_DB); } // Remove the engine name from the var name, which must be of pattern // / @@ -318,14 +301,12 @@ void CampaignReader::DestructorClose(bool Verbose) noexcept return v; }*/ -MinVarInfo *CampaignReader::MinBlocksInfo(const VariableBase &Var, - size_t Step) const +MinVarInfo *CampaignReader::MinBlocksInfo(const VariableBase &Var, size_t Step) const { auto it = m_VarInternalInfo.find(Var.m_Name); if (it != m_VarInternalInfo.end()) { - VariableBase *vb = - reinterpret_cast(it->second.originalVar); + VariableBase *vb = reinterpret_cast(it->second.originalVar); Engine *e = m_Engines[it->second.engineIdx]; MinVarInfo *MV = e->MinBlocksInfo(*vb, Step); if (MV) @@ -336,55 +317,50 @@ MinVarInfo *CampaignReader::MinBlocksInfo(const VariableBase &Var, return nullptr; } -#define declare_type(T) \ - void CampaignReader::DoGetSync(Variable &variable, T *data) \ - { \ - PERFSTUBS_SCOPED_TIMER("CampaignReader::Get"); \ - auto p = TranslateToActualVariable(variable); \ - p.second->Get(*p.first, data, adios2::Mode::Sync); \ - } \ - void CampaignReader::DoGetDeferred(Variable &variable, T *data) \ - { \ - PERFSTUBS_SCOPED_TIMER("CampaignReader::Get"); \ - auto it = m_VarInternalInfo.find(variable.m_Name); \ - Variable *v = \ - reinterpret_cast *>(it->second.originalVar); \ - Engine *e = m_Engines[it->second.engineIdx]; \ - e->Get(*v, data, adios2::Mode::Deferred); \ - } \ - \ - std::map::BPInfo>> \ - CampaignReader::DoAllStepsBlocksInfo(const Variable &variable) const \ - { \ - PERFSTUBS_SCOPED_TIMER("CampaignReader::AllStepsBlocksInfo"); \ - auto it = m_VarInternalInfo.find(variable.m_Name); \ - Variable *v = \ - reinterpret_cast *>(it->second.originalVar); \ - Engine *e = m_Engines[it->second.engineIdx]; \ - return e->AllStepsBlocksInfo(*v); \ - } \ - \ - std::vector::BPInfo>> \ - CampaignReader::DoAllRelativeStepsBlocksInfo(const Variable &variable) \ - const \ - { \ - PERFSTUBS_SCOPED_TIMER("CampaignReader::AllRelativeStepsBlocksInfo"); \ - auto it = m_VarInternalInfo.find(variable.m_Name); \ - Variable *v = \ - reinterpret_cast *>(it->second.originalVar); \ - Engine *e = m_Engines[it->second.engineIdx]; \ - return e->AllRelativeStepsBlocksInfo(*v); \ - } \ - \ - std::vector::BPInfo> CampaignReader::DoBlocksInfo( \ - const Variable &variable, const size_t step) const \ - { \ - PERFSTUBS_SCOPED_TIMER("CampaignReader::BlocksInfo"); \ - auto it = m_VarInternalInfo.find(variable.m_Name); \ - Variable *v = \ - reinterpret_cast *>(it->second.originalVar); \ - Engine *e = m_Engines[it->second.engineIdx]; \ - return e->BlocksInfo(*v, step); \ +#define declare_type(T) \ + void CampaignReader::DoGetSync(Variable &variable, T *data) \ + { \ + PERFSTUBS_SCOPED_TIMER("CampaignReader::Get"); \ + auto p = TranslateToActualVariable(variable); \ + p.second->Get(*p.first, data, adios2::Mode::Sync); \ + } \ + void CampaignReader::DoGetDeferred(Variable &variable, T *data) \ + { \ + PERFSTUBS_SCOPED_TIMER("CampaignReader::Get"); \ + auto it = m_VarInternalInfo.find(variable.m_Name); \ + Variable *v = reinterpret_cast *>(it->second.originalVar); \ + Engine *e = m_Engines[it->second.engineIdx]; \ + e->Get(*v, data, adios2::Mode::Deferred); \ + } \ + \ + std::map::BPInfo>> \ + CampaignReader::DoAllStepsBlocksInfo(const Variable &variable) const \ + { \ + PERFSTUBS_SCOPED_TIMER("CampaignReader::AllStepsBlocksInfo"); \ + auto it = m_VarInternalInfo.find(variable.m_Name); \ + Variable *v = reinterpret_cast *>(it->second.originalVar); \ + Engine *e = m_Engines[it->second.engineIdx]; \ + return e->AllStepsBlocksInfo(*v); \ + } \ + \ + std::vector::BPInfo>> \ + CampaignReader::DoAllRelativeStepsBlocksInfo(const Variable &variable) const \ + { \ + PERFSTUBS_SCOPED_TIMER("CampaignReader::AllRelativeStepsBlocksInfo"); \ + auto it = m_VarInternalInfo.find(variable.m_Name); \ + Variable *v = reinterpret_cast *>(it->second.originalVar); \ + Engine *e = m_Engines[it->second.engineIdx]; \ + return e->AllRelativeStepsBlocksInfo(*v); \ + } \ + \ + std::vector::BPInfo> CampaignReader::DoBlocksInfo( \ + const Variable &variable, const size_t step) const \ + { \ + PERFSTUBS_SCOPED_TIMER("CampaignReader::BlocksInfo"); \ + auto it = m_VarInternalInfo.find(variable.m_Name); \ + Variable *v = reinterpret_cast *>(it->second.originalVar); \ + Engine *e = m_Engines[it->second.engineIdx]; \ + return e->BlocksInfo(*v, step); \ } ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) diff --git a/source/adios2/engine/campaign/CampaignReader.h b/source/adios2/engine/campaign/CampaignReader.h index 0264e55f73..eac4ce7c43 100644 --- a/source/adios2/engine/campaign/CampaignReader.h +++ b/source/adios2/engine/campaign/CampaignReader.h @@ -41,12 +41,10 @@ class CampaignReader : public Engine * @param method * @param hostLanguage */ - CampaignReader(IO &adios, const std::string &name, const Mode mode, - helper::Comm comm); + CampaignReader(IO &adios, const std::string &name, const Mode mode, helper::Comm comm); ~CampaignReader(); - StepStatus BeginStep(StepMode mode = StepMode::Read, - const float timeoutSeconds = -1.0) final; + StepStatus BeginStep(StepMode mode = StepMode::Read, const float timeoutSeconds = -1.0) final; void PerformGets() final; size_t CurrentStep() const final; void EndStep() final; @@ -73,10 +71,7 @@ class CampaignReader : public Engine void *originalVar; // Variable in the actual IO size_t ioIdx; // actual IO in m_IOs size_t engineIdx; // actual engine in m_Engines - VarInternalInfo(void *p, size_t i, size_t e) - : originalVar(p), ioIdx(i), engineIdx(e) - { - } + VarInternalInfo(void *p, size_t i, size_t e) : originalVar(p), ioIdx(i), engineIdx(e) {} }; std::unordered_map m_VarInternalInfo; @@ -85,23 +80,23 @@ class CampaignReader : public Engine void InitParameters() final; void InitTransports() final; -#define declare_type(T) \ - void DoGetSync(Variable &, T *) final; \ +#define declare_type(T) \ + void DoGetSync(Variable &, T *) final; \ void DoGetDeferred(Variable &, T *) final; ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) #undef declare_type void DoClose(const int transportIndex = -1); -#define declare_type(T) \ - std::map::BPInfo>> \ - DoAllStepsBlocksInfo(const Variable &variable) const final; \ - \ - std::vector::BPInfo>> \ - DoAllRelativeStepsBlocksInfo(const Variable &) const final; \ - \ - std::vector::BPInfo> DoBlocksInfo( \ - const Variable &variable, const size_t step) const final; +#define declare_type(T) \ + std::map::BPInfo>> DoAllStepsBlocksInfo( \ + const Variable &variable) const final; \ + \ + std::vector::BPInfo>> DoAllRelativeStepsBlocksInfo( \ + const Variable &) const final; \ + \ + std::vector::BPInfo> DoBlocksInfo(const Variable &variable, \ + const size_t step) const final; ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) #undef declare_type @@ -117,16 +112,15 @@ class CampaignReader : public Engine * based on an existing variable. */ template - Variable DuplicateVariable(Variable *variable, IO &io, - std::string &name, VarInternalInfo &vii); + Variable DuplicateVariable(Variable *variable, IO &io, std::string &name, + VarInternalInfo &vii); /** * Create a new variable with name `name` in `io` * based on an existing variable. */ template - std::pair *, Engine *> - TranslateToActualVariable(Variable &variable); + std::pair *, Engine *> TranslateToActualVariable(Variable &variable); sqlite3 *m_DB; CampaignData m_CampaignData; diff --git a/source/adios2/engine/campaign/CampaignReader.tcc b/source/adios2/engine/campaign/CampaignReader.tcc index a34029bee5..d635a1a976 100644 --- a/source/adios2/engine/campaign/CampaignReader.tcc +++ b/source/adios2/engine/campaign/CampaignReader.tcc @@ -23,9 +23,8 @@ namespace engine { template -inline Variable CampaignReader::DuplicateVariable(Variable *variable, - IO &io, std::string &name, - VarInternalInfo &vii) +inline Variable CampaignReader::DuplicateVariable(Variable *variable, IO &io, + std::string &name, VarInternalInfo &vii) { auto &v = io.DefineVariable(name, variable->Shape()); v.m_AvailableStepsCount = variable->GetAvailableStepsCount(); @@ -37,8 +36,7 @@ inline Variable CampaignReader::DuplicateVariable(Variable *variable, v.m_RandomAccess = variable->m_RandomAccess; v.m_MemSpace = variable->m_MemSpace; v.m_JoinedDimPos = variable->m_JoinedDimPos; - v.m_AvailableStepBlockIndexOffsets = - variable->m_AvailableStepBlockIndexOffsets; + v.m_AvailableStepBlockIndexOffsets = variable->m_AvailableStepBlockIndexOffsets; v.m_AvailableShapes = variable->m_AvailableShapes; v.m_Min = variable->m_Min; v.m_Max = variable->m_Max; diff --git a/source/adios2/engine/campaign/CampaignRecord.h b/source/adios2/engine/campaign/CampaignRecord.h index e7deb0f78a..57d0d0ff50 100644 --- a/source/adios2/engine/campaign/CampaignRecord.h +++ b/source/adios2/engine/campaign/CampaignRecord.h @@ -39,8 +39,7 @@ struct CampaignRecord CampaignRecord() : delta_step(0), delta_time(0.0), varying_deltas(false){}; - CampaignRecord(size_t step, double time) - : delta_step(0), delta_time(0.0), varying_deltas(false) + CampaignRecord(size_t step, double time) : delta_step(0), delta_time(0.0), varying_deltas(false) { steps.push_back(step); times.push_back(time); From 86bc6881dcf7b4fe8099d86e46faa624ef746163 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 8 Aug 2023 09:21:39 -0400 Subject: [PATCH 35/58] remove debug prints --- .../adios2/engine/campaign/CampaignData.cpp | 6 -- .../engine/campaign/CampaignManager.cpp | 85 ++++--------------- .../adios2_campaign_manager.py | 4 +- 3 files changed, 18 insertions(+), 77 deletions(-) diff --git a/source/adios2/engine/campaign/CampaignData.cpp b/source/adios2/engine/campaign/CampaignData.cpp index 9c056135b0..f2a68474dc 100644 --- a/source/adios2/engine/campaign/CampaignData.cpp +++ b/source/adios2/engine/campaign/CampaignData.cpp @@ -208,21 +208,16 @@ int inflateToFile(const unsigned char *source, const size_t blobsize, std::ofstr /* decompress until deflate stream ends or end of file */ unsigned char *p = const_cast(source); - std::cout << " Inflate = " << (void *)p << " size = " << blobsize << "\n"; do { strm.avail_in = (uInt)(blobsize > CHUNK ? CHUNK : blobsize); strm.next_in = p; - std::cout << " next_in = " << (void *)strm.next_in << " avail_in = " << strm.avail_in - << "\n"; /* run inflate() on input until output buffer not full */ do { strm.avail_out = CHUNK; strm.next_out = out.data(); - std::cout << " next_out = " << (void *)strm.next_out - << " avail_out = " << strm.avail_out << "\n"; ret = inflate(&strm, Z_NO_FLUSH); switch (ret) { @@ -235,7 +230,6 @@ int inflateToFile(const unsigned char *source, const size_t blobsize, std::ofstr return ret; } have = CHUNK - strm.avail_out; - std::cout << " have = " << have << "\n"; dest->write(reinterpret_cast(out.data()), have); } while (strm.avail_out == 0); diff --git a/source/adios2/engine/campaign/CampaignManager.cpp b/source/adios2/engine/campaign/CampaignManager.cpp index 0d70ce72ba..d63dfae6ab 100644 --- a/source/adios2/engine/campaign/CampaignManager.cpp +++ b/source/adios2/engine/campaign/CampaignManager.cpp @@ -25,37 +25,17 @@ namespace core namespace engine { -static std::string CMapToJson(const CampaignRecordMap &cmap, const int rank, - const std::string name) +static std::string CMapToJson(const CampaignRecordMap &cmap, const int rank, const std::string name) { nlohmann::json j = nlohmann::json::array(); - std::cout << "Campaign Manager " << rank << " Close(" << name - << ")\nCampaign"; for (auto &r : cmap) { - nlohmann::json c = - nlohmann::json{{"name", r.first}, - {"varying_deltas", r.second.varying_deltas}, - {"delta_step", r.second.delta_step}, - {"delta_time", r.second.delta_time}, - {"steps", r.second.steps}, - {"times", r.second.times}}; - std::cout << "name = " << r.first << "\n"; - std::cout << " varying_deltas = " << r.second.varying_deltas << "\n"; - std::cout << " delta_step = " << r.second.delta_step << "\n"; - std::cout << " delta_time = " << r.second.delta_time << "\n"; - std::cout << " steps = {"; - for (const auto s : r.second.steps) - { - std::cout << s << " "; - } - std::cout << "}\n"; - std::cout << " times = {"; - for (const auto t : r.second.times) - { - std::cout << t << " "; - } - std::cout << "}\n"; + nlohmann::json c = nlohmann::json{{"name", r.first}, + {"varying_deltas", r.second.varying_deltas}, + {"delta_step", r.second.delta_step}, + {"delta_time", r.second.delta_time}, + {"steps", r.second.steps}, + {"times", r.second.times}}; j.push_back(c); } return nlohmann::to_string(j); @@ -66,8 +46,7 @@ CampaignManager::CampaignManager(adios2::helper::Comm &comm) m_WriterRank = comm.Rank(); if (m_Verbosity == 5) { - std::cout << "Campaign Manager " << m_WriterRank - << " constructor called" << std::endl; + std::cout << "Campaign Manager " << m_WriterRank << " constructor called" << std::endl; } helper::CreateDirectory(m_CampaignDir); } @@ -76,8 +55,7 @@ CampaignManager::~CampaignManager() { if (m_Verbosity == 5) { - std::cout << "Campaign Manager " << m_WriterRank - << " desctructor called\n"; + std::cout << "Campaign Manager " << m_WriterRank << " desctructor called\n"; } if (m_Opened) { @@ -87,23 +65,19 @@ CampaignManager::~CampaignManager() void CampaignManager::Open(const std::string &name) { - m_Name = m_CampaignDir + "/" + name + "_" + std::to_string(m_WriterRank) + - ".json"; + m_Name = m_CampaignDir + "/" + name + "_" + std::to_string(m_WriterRank) + ".json"; if (m_Verbosity == 5) { - std::cout << "Campaign Manager " << m_WriterRank << " Open(" << m_Name - << ")\n"; + std::cout << "Campaign Manager " << m_WriterRank << " Open(" << m_Name << ")\n"; } } -void CampaignManager::Record(const std::string &name, const size_t step, - const double time) +void CampaignManager::Record(const std::string &name, const size_t step, const double time) { if (m_Verbosity == 5) { - std::cout << "Campaign Manager " << m_WriterRank - << " Record name = " << name << " step = " << step - << " time = " << time << "\n"; + std::cout << "Campaign Manager " << m_WriterRank << " Record name = " << name + << " step = " << step << " time = " << time << "\n"; } auto r = cmap.find(name); if (r != cmap.end()) @@ -121,8 +95,7 @@ void CampaignManager::Record(const std::string &name, const size_t step, } else { - size_t old_delta_step = - r->second.steps.back() - r->second.steps.rbegin()[1]; + size_t old_delta_step = r->second.steps.back() - r->second.steps.rbegin()[1]; if (old_delta_step != delta_step) { r->second.delta_step = 0; @@ -143,37 +116,11 @@ void CampaignManager::Record(const std::string &name, const size_t step, void CampaignManager::Close() { - /*if (m_Verbosity == 5) - { - std::cout << "Campaign Manager " << m_WriterRank - << " Close()\nCampaign"; - for (const auto &r : cmap) - { - std::cout << "name = " << r.first << "\n"; - std::cout << " varying_deltas = " << r.second.varying_deltas - << "\n"; - std::cout << " delta_step = " << r.second.delta_step << "\n"; - std::cout << " delta_time = " << r.second.delta_time << "\n"; - std::cout << " steps = {"; - for (const auto s : r.second.steps) - { - std::cout << s << " "; - } - std::cout << "}\n"; - std::cout << " times = {"; - for (const auto t : r.second.times) - { - std::cout << t << " "; - } - std::cout << "}\n"; - } - }*/ if (!cmap.empty()) { m_Output.open(m_Name, std::ofstream::out); m_Opened = true; - m_Output << std::setw(4) << CMapToJson(cmap, m_WriterRank, m_Name) - << std::endl; + m_Output << std::setw(4) << CMapToJson(cmap, m_WriterRank, m_Name) << std::endl; m_Output.close(); m_Opened = false; } diff --git a/source/utils/adios_campaign_manager/adios2_campaign_manager.py b/source/utils/adios_campaign_manager/adios2_campaign_manager.py index 12d70e60d6..38d5255742 100755 --- a/source/utils/adios_campaign_manager/adios2_campaign_manager.py +++ b/source/utils/adios_campaign_manager/adios2_campaign_manager.py @@ -150,7 +150,7 @@ def AddDatasetToArchive(args: dict, dataset: str, cur: sqlite3.Cursor, hostID: i def ProcessJsonFile(args: dict, jsonlist: list, cur: sqlite3.Cursor, hostID: int, dirID: int): for entry in jsonlist: - print(f"Process entry {entry}:") + # print(f"Process entry {entry}:") if isinstance(entry, dict): if "name" in entry: AddDatasetToArchive( @@ -228,7 +228,7 @@ def Update(args: dict, cur: sqlite3.Cursor): jsonlist = MergeJsonFiles(jsonFileList) - print(f"Merged json = {jsonlist}") + # print(f"Merged json = {jsonlist}") ProcessJsonFile(args, jsonlist, cur, hostID, dirID) con.commit() From ad9f23c0dbf8cdd389adee0e91039b6b1651c796 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 8 Aug 2023 09:42:24 -0400 Subject: [PATCH 36/58] remote unused variables and functions --- .../adios2/engine/campaign/CampaignData.cpp | 19 ------------------- .../adios2/engine/campaign/CampaignReader.cpp | 2 -- .../adios2/engine/campaign/CampaignReader.h | 1 - 3 files changed, 22 deletions(-) diff --git a/source/adios2/engine/campaign/CampaignData.cpp b/source/adios2/engine/campaign/CampaignData.cpp index f2a68474dc..8e62e8ca84 100644 --- a/source/adios2/engine/campaign/CampaignData.cpp +++ b/source/adios2/engine/campaign/CampaignData.cpp @@ -162,25 +162,6 @@ void ReadCampaignData(sqlite3 *db, CampaignData &cd) } } -static int sqlcb_bpfile_data(void *p, int argc, char **argv, char **azColName) -{ - CampaignData *cdp = reinterpret_cast(p); - CampaignBPFile cf; - size_t dsid = helper::StringToSizeT(std::string(argv[0]), "SQL callback convert text to int"); - cf.bpDatasetIdx = dsid - 1; - cf.name = std::string(argv[1]); - int comp = helper::StringTo(std::string(argv[2]), "SQL callback convert text to int"); - cf.compressed = (bool)comp; - cf.lengthOriginal = - helper::StringToSizeT(std::string(argv[3]), "SQL callback convert text to int"); - cf.lengthCompressed = - helper::StringToSizeT(std::string(argv[4]), "SQL callback convert text to int"); - - CampaignBPDataset &cds = cdp->bpdatasets[cf.bpDatasetIdx]; - cds.files.push_back(cf); - return 0; -}; - /* Decompress from in-memory source to file dest until stream ends or EOF. inf() returns Z_OK on success, Z_MEM_ERROR if memory could not be allocated for processing, Z_DATA_ERROR if the deflate data is diff --git a/source/adios2/engine/campaign/CampaignReader.cpp b/source/adios2/engine/campaign/CampaignReader.cpp index 4a5f7591b1..0beb88bbc9 100644 --- a/source/adios2/engine/campaign/CampaignReader.cpp +++ b/source/adios2/engine/campaign/CampaignReader.cpp @@ -159,8 +159,6 @@ void CampaignReader::InitParameters() void CampaignReader::InitTransports() { int rc; - char *zErrMsg = 0; - rc = sqlite3_open(m_Name.c_str(), &m_DB); if (rc) { diff --git a/source/adios2/engine/campaign/CampaignReader.h b/source/adios2/engine/campaign/CampaignReader.h index eac4ce7c43..711099fbd1 100644 --- a/source/adios2/engine/campaign/CampaignReader.h +++ b/source/adios2/engine/campaign/CampaignReader.h @@ -58,7 +58,6 @@ class CampaignReader : public Engine int m_ReaderRank; // my rank in the readers' comm int m_CurrentStep = 0; - bool m_FirstStep = true; // EndStep must call PerformGets if necessary bool m_NeedPerformGets = false; From 3af1909469921e9b3e79cc972050baa423f1f4b8 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 8 Aug 2023 09:46:23 -0400 Subject: [PATCH 37/58] reformat --- examples/campaign/campaign_write.cpp | 103 ++++++++++----------------- source/adios2/core/ADIOS.h | 3 +- source/adios2/core/IO.cpp | 11 ++- 3 files changed, 43 insertions(+), 74 deletions(-) diff --git a/examples/campaign/campaign_write.cpp b/examples/campaign/campaign_write.cpp index 7123285f54..29c397a72c 100644 --- a/examples/campaign/campaign_write.cpp +++ b/examples/campaign/campaign_write.cpp @@ -71,8 +71,7 @@ unsigned int convertToUint(std::string varName, char *arg) unsigned int retval = std::strtoul(arg, &end, 10); if (end[0] || errno == ERANGE) { - throw std::invalid_argument("Invalid value given for " + varName + - ": " + std::string(arg)); + throw std::invalid_argument("Invalid value given for " + varName + ": " + std::string(arg)); } return retval; } @@ -84,12 +83,10 @@ int main(int argc, char *argv[]) printUsage(); return -1; } - nSteps = - adios2::helper::StringToSizeT(argv[1], "when parsing argument steps"); + nSteps = adios2::helper::StringToSizeT(argv[1], "when parsing argument steps"); if (argc > 2) { - startStep = adios2::helper::StringToSizeT( - argv[2], "when parsing argument start_step"); + startStep = adios2::helper::StringToSizeT(argv[2], "when parsing argument start_step"); } /*nSteps = convertToUint("steps", argv[1]); if (argc > 2) @@ -206,75 +203,55 @@ int main(int argc, char *argv[]) } adios2::Variable varXAll = ioAll.DefineVariable( - "x", {static_cast(nproc), Nx}, - {static_cast(rank), 0}, {1, Nx}); + "x", {static_cast(nproc), Nx}, {static_cast(rank), 0}, {1, Nx}); adios2::Variable varYAll = ioAll.DefineVariable( - "y", {static_cast(nproc), Ny}, - {static_cast(rank), 0}, {1, Ny}); + "y", {static_cast(nproc), Ny}, {static_cast(rank), 0}, {1, Ny}); adios2::Variable varZAll = ioAll.DefineVariable( - "z", {static_cast(nproc), Nz}, - {static_cast(rank), 0}, {1, Nz}); - adios2::Variable varStepAll = - ioAll.DefineVariable("AdiosStep"); - adios2::Variable varPhysStepAll = - ioAll.DefineVariable("iteration"); - adios2::Variable varPhysTimeAll = - ioAll.DefineVariable("time"); - ioAll.DefineAttribute("comment", - "Written by all processes"); + "z", {static_cast(nproc), Nz}, {static_cast(rank), 0}, {1, Nz}); + adios2::Variable varStepAll = ioAll.DefineVariable("AdiosStep"); + adios2::Variable varPhysStepAll = ioAll.DefineVariable("iteration"); + adios2::Variable varPhysTimeAll = ioAll.DefineVariable("time"); + ioAll.DefineAttribute("comment", "Written by all processes"); adios2::Variable varZStep = ioStep.DefineVariable( - "z", {(unsigned int)nproc, Nz}, {static_cast(rank), 0}, - {1, Nz}); - adios2::Variable varStepStep = - ioStep.DefineVariable("AdiosStep"); - adios2::Variable varPhysStepStep = - ioStep.DefineVariable("iteration"); - adios2::Variable varPhysTimeStep = - ioStep.DefineVariable("time"); + "z", {(unsigned int)nproc, Nz}, {static_cast(rank), 0}, {1, Nz}); + adios2::Variable varStepStep = ioStep.DefineVariable("AdiosStep"); + adios2::Variable varPhysStepStep = ioStep.DefineVariable("iteration"); + adios2::Variable varPhysTimeStep = ioStep.DefineVariable("time"); adios2::Variable varYNew = ioStep.DefineVariable( - "y", {(unsigned int)nproc, Ny}, {static_cast(rank), 0}, - {1, Ny}); - adios2::Variable varStepNew = - ioNew.DefineVariable("AdiosStep"); - adios2::Variable varPhysStepNew = - ioNew.DefineVariable("iteration"); - adios2::Variable varPhysTimeNew = - ioNew.DefineVariable("time"); + "y", {(unsigned int)nproc, Ny}, {static_cast(rank), 0}, {1, Ny}); + adios2::Variable varStepNew = ioNew.DefineVariable("AdiosStep"); + adios2::Variable varPhysStepNew = ioNew.DefineVariable("iteration"); + adios2::Variable varPhysTimeNew = ioNew.DefineVariable("time"); writerAll = ioAll.Open("dataAll.bp", mode); adios2::Variable varXFirstRank; if (rank == 0) { - varXFirstRank = - ioFirstRank.DefineVariable("x", {Nx}, {0}, {Nx}); - ioFirstRank.DefineAttribute("comment", - "Written by rank 0"); - writerFirstRank = - ioFirstRank.Open("dataFirstRank.bp", mode, commFirstRank); + varXFirstRank = ioFirstRank.DefineVariable("x", {Nx}, {0}, {Nx}); + ioFirstRank.DefineAttribute("comment", "Written by rank 0"); + writerFirstRank = ioFirstRank.Open("dataFirstRank.bp", mode, commFirstRank); } adios2::Variable varXLastRank; if (rank == nproc - 1) { - varXLastRank = - ioLastRank.DefineVariable("x", {Nx}, {0}, {Nx}); - ioLastRank.DefineAttribute( - "comment", "Written by rank " + std::to_string(nproc - 1)); - writerLastRank = - ioLastRank.Open("dataLastRank.bp", mode, commLastRank); + varXLastRank = ioLastRank.DefineVariable("x", {Nx}, {0}, {Nx}); + ioLastRank.DefineAttribute("comment", + "Written by rank " + std::to_string(nproc - 1)); + writerLastRank = ioLastRank.Open("dataLastRank.bp", mode, commLastRank); } // the other ADIOS object (valid on rank 1..N, not on rank 0) adios2::Variable varXAnother; if (rank > 0) { - varXAnother = ioAnother.DefineVariable( - "x", {static_cast(nproc - 1), Nx}); - ioAnother.DefineAttribute( - "comment", "Written by ranks 1.." + std::to_string(nproc - 1)); + varXAnother = + ioAnother.DefineVariable("x", {static_cast(nproc - 1), Nx}); + ioAnother.DefineAttribute("comment", "Written by ranks 1.." + + std::to_string(nproc - 1)); writerAnother = ioAnother.Open("dataAnother.bp", mode); } @@ -306,8 +283,8 @@ int main(int argc, char *argv[]) writerAll.EndStep(); - adios2::Engine writerStep = ioStep.Open( - "dataStep" + std::to_string(step) + ".bp", adios2::Mode::Write); + adios2::Engine writerStep = + ioStep.Open("dataStep" + std::to_string(step) + ".bp", adios2::Mode::Write); writerStep.Put(varZStep, z.data()); writerStep.Put(varStepStep, step); writerStep.Put(varPhysStepStep, physicalStep); @@ -318,8 +295,7 @@ int main(int argc, char *argv[]) { writerAnother.BeginStep(); varXAnother.SetSelection(adios2::Box( - {static_cast(rank - 1), 0}, - {1, static_cast(Nx)})); + {static_cast(rank - 1), 0}, {1, static_cast(Nx)})); writerAnother.Put(varXAnother, x.data()); writerAnother.EndStep(); } @@ -341,8 +317,7 @@ int main(int argc, char *argv[]) if (startStep > 0 && step == startStep) { adios2::Engine writerNew = - ioNew.Open("dataNew" + std::to_string(step) + ".bp", - adios2::Mode::Write); + ioNew.Open("dataNew" + std::to_string(step) + ".bp", adios2::Mode::Write); writerNew.Put(varYNew, y.data()); writerNew.Put(varStepNew, step); writerNew.Put(varPhysStepNew, physicalStep); @@ -382,14 +357,10 @@ int main(int argc, char *argv[]) adios2::Variable varXFinal = ioFinal.DefineVariable("x_on_rank_1", {Nx}, {0}, {Nx}); - adios2::Variable varStepFinal = - ioFinal.DefineVariable("AdiosStep"); - adios2::Variable varPhysStepFinal = - ioFinal.DefineVariable("iteration"); - adios2::Variable varPhysTimeFinal = - ioFinal.DefineVariable("time"); - ioFinal.DefineAttribute( - "comment", "Written by rank 1 at end of run"); + adios2::Variable varStepFinal = ioFinal.DefineVariable("AdiosStep"); + adios2::Variable varPhysStepFinal = ioFinal.DefineVariable("iteration"); + adios2::Variable varPhysTimeFinal = ioFinal.DefineVariable("time"); + ioFinal.DefineAttribute("comment", "Written by rank 1 at end of run"); adios2::Engine writerFinal = ioFinal.Open("dataFinal.bp", mode); writerFinal.Put(varXFinal, x.data()); diff --git a/source/adios2/core/ADIOS.h b/source/adios2/core/ADIOS.h index e6d7940b48..cee4f65664 100644 --- a/source/adios2/core/ADIOS.h +++ b/source/adios2/core/ADIOS.h @@ -161,8 +161,7 @@ class ADIOS * in main thread. Useful when using Async IO */ void ExitComputationBlock() noexcept; - void RecordOutputStep(const std::string &name, - const size_t step = UnknownStep, + void RecordOutputStep(const std::string &name, const size_t step = UnknownStep, const double time = UnknownTime); private: diff --git a/source/adios2/core/IO.cpp b/source/adios2/core/IO.cpp index a4470d5ca2..8300cba1ee 100644 --- a/source/adios2/core/IO.cpp +++ b/source/adios2/core/IO.cpp @@ -142,12 +142,11 @@ std::unordered_map Factory = { }; const std::unordered_map ReadRandomAccess_Supported = { - {"bp3", false}, {"bp4", false}, {"bp5", true}, - {"dataman", false}, {"ssc", false}, {"mhs", false}, - {"sst", false}, {"daos", false}, {"effis", false}, - {"dataspaces", false}, {"hdf5", false}, {"skeleton", true}, - {"inline", false}, {"null", true}, {"nullcore", true}, - {"plugin", false}, {"campaign", true}, + {"bp3", false}, {"bp4", false}, {"bp5", true}, {"dataman", false}, + {"ssc", false}, {"mhs", false}, {"sst", false}, {"daos", false}, + {"effis", false}, {"dataspaces", false}, {"hdf5", false}, {"skeleton", true}, + {"inline", false}, {"null", true}, {"nullcore", true}, {"plugin", false}, + {"campaign", true}, }; // Synchronize access to the factory in case one thread is From c01ace4ff329071c60e7af69ffe0e3aac673495f Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 8 Aug 2023 09:49:14 -0400 Subject: [PATCH 38/58] \\. for \. --- .../utils/adios_campaign_manager/adios2_campaign_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/utils/adios_campaign_manager/adios2_campaign_manager.py b/source/utils/adios_campaign_manager/adios2_campaign_manager.py index 38d5255742..09857ecc4a 100755 --- a/source/utils/adios_campaign_manager/adios2_campaign_manager.py +++ b/source/utils/adios_campaign_manager/adios2_campaign_manager.py @@ -163,9 +163,9 @@ def ProcessJsonFile(args: dict, jsonlist: list, cur: sqlite3.Cursor, hostID: int def GetHostName(): host = getfqdn() if host.startswith("login"): - host = sub('^login[0-9]*\.', '', host) + host = sub('^login[0-9]*\\.', '', host) if host.startswith("batch"): - host = sub('^batch[0-9]*\.', '', host) + host = sub('^batch[0-9]*\\.', '', host) shorthost = host.split('.')[0] return host, shorthost From 8eba0dcb21844b684b795f79e515344c91c07e11 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 8 Aug 2023 09:56:28 -0400 Subject: [PATCH 39/58] don't include sqlite3.h in CampaignRecord.h yet --- source/adios2/engine/campaign/CampaignRecord.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/adios2/engine/campaign/CampaignRecord.h b/source/adios2/engine/campaign/CampaignRecord.h index 57d0d0ff50..21829fa72e 100644 --- a/source/adios2/engine/campaign/CampaignRecord.h +++ b/source/adios2/engine/campaign/CampaignRecord.h @@ -16,8 +16,6 @@ #include #include -#include - namespace adios2 { namespace core From 1208433d13be23cb481bb2da70fc72d9780e4cfd Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Fri, 1 Sep 2023 08:13:16 -0400 Subject: [PATCH 40/58] fixes for compiling --- source/adios2/engine/campaign/CampaignManager.cpp | 2 +- source/adios2/helper/adiosNetwork.cpp | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/source/adios2/engine/campaign/CampaignManager.cpp b/source/adios2/engine/campaign/CampaignManager.cpp index d63dfae6ab..ddc6a6bc0b 100644 --- a/source/adios2/engine/campaign/CampaignManager.cpp +++ b/source/adios2/engine/campaign/CampaignManager.cpp @@ -87,7 +87,7 @@ void CampaignManager::Record(const std::string &name, const size_t step, const d size_t delta_step = step - last_step; double last_time = r->second.times.back(); double delta_time = time - last_time; - int nsteps = r->second.steps.size(); + auto nsteps = r->second.steps.size(); if (nsteps == 1) { r->second.delta_step = delta_step; diff --git a/source/adios2/helper/adiosNetwork.cpp b/source/adios2/helper/adiosNetwork.cpp index 33872dce3e..33b88a2c52 100644 --- a/source/adios2/helper/adiosNetwork.cpp +++ b/source/adios2/helper/adiosNetwork.cpp @@ -13,13 +13,15 @@ #include "adios2/toolkit/transport/file/FileFStream.h" #include // memcpy -#include // gethostname #include //getFQDN #include //getFQDN #include //getFQDN #ifndef _WIN32 + +#include // gethostname + #if defined(ADIOS2_HAVE_DATAMAN) || defined(ADIOS2_HAVE_TABLE) #include From 3af9b558686c6d8a20da493b2d383bae3246d3e6 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Fri, 1 Sep 2023 08:26:09 -0400 Subject: [PATCH 41/58] StringTo does not exist, use instead --- source/adios2/engine/campaign/CampaignData.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/adios2/engine/campaign/CampaignData.cpp b/source/adios2/engine/campaign/CampaignData.cpp index 8e62e8ca84..13a0ebd8c3 100644 --- a/source/adios2/engine/campaign/CampaignData.cpp +++ b/source/adios2/engine/campaign/CampaignData.cpp @@ -92,7 +92,8 @@ static int sqlcb_bpfile(void *p, int argc, char **argv, char **azColName) helper::StringToSizeT(std::string(argv[3]), "SQL callback convert text to int"); cf.lengthCompressed = helper::StringToSizeT(std::string(argv[4]), "SQL callback convert text to int"); - cf.ctime = helper::StringTo(std::string(argv[5]), "SQL callback convert ctime to int"); + cf.ctime = static_cast( + helper::StringTo(std::string(argv[5]), "SQL callback convert ctime to int")); CampaignBPDataset &cds = cdp->bpdatasets[cf.bpDatasetIdx]; cds.files.push_back(cf); From 0a3e0bb27ae8a5546ea52f09cd212d476dcf7d24 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Fri, 1 Sep 2023 08:26:44 -0400 Subject: [PATCH 42/58] fix comm name --- source/adios2/engine/hdf5/HDF5WriterP.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/adios2/engine/hdf5/HDF5WriterP.cpp b/source/adios2/engine/hdf5/HDF5WriterP.cpp index a938532fdb..d113f0f314 100644 --- a/source/adios2/engine/hdf5/HDF5WriterP.cpp +++ b/source/adios2/engine/hdf5/HDF5WriterP.cpp @@ -46,7 +46,7 @@ void HDF5WriterP::EndStep() m_H5File.CleanUpNullVars(m_IO); m_H5File.Advance(); m_H5File.WriteAttrFromIO(m_IO); - if (!comm.Rank()) + if (!m_Comm.Rank()) { m_IO.m_ADIOS.RecordOutputStep(m_Name, UnknownStep, UnknownTime); } From 934ae46180a14b1f9bfb3f2d1d529c78f6b241cc Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Fri, 1 Sep 2023 08:51:50 -0400 Subject: [PATCH 43/58] don't include netdb on windows --- source/adios2/CMakeLists.txt | 2 +- source/adios2/helper/adiosNetwork.cpp | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index 7043a9feab..f4172b4d77 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -195,7 +195,7 @@ if(ADIOS2_HAVE_SQLite3) engine/campaign/CampaignReader.cpp engine/campaign/CampaignReader.tcc engine/campaign/CampaignData.cpp) - target_link_libraries(adios2_core PRIVATE SQLite::SQLite3) + target_link_libraries(adios2_core PRIVATE SQLite::SQLite3 ZLIB::ZLIB) endif() if(ADIOS2_HAVE_DAOS) diff --git a/source/adios2/helper/adiosNetwork.cpp b/source/adios2/helper/adiosNetwork.cpp index 33b88a2c52..756254b933 100644 --- a/source/adios2/helper/adiosNetwork.cpp +++ b/source/adios2/helper/adiosNetwork.cpp @@ -14,13 +14,12 @@ #include // memcpy +#ifndef _WIN32 + #include //getFQDN #include //getFQDN #include //getFQDN - -#ifndef _WIN32 - -#include // gethostname +#include // gethostname #if defined(ADIOS2_HAVE_DATAMAN) || defined(ADIOS2_HAVE_TABLE) From c6ae903db71b1cf20d89e370e5b850f0f46cbb81 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Fri, 1 Sep 2023 08:52:16 -0400 Subject: [PATCH 44/58] when campaign reader uses sqlite3 it also needs zlib --- cmake/DetectOptions.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index f58517a6c8..e8b6d09718 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -550,7 +550,9 @@ elseif(ADIOS2_USE_SQLite3) find_package(SQLite3 REQUIRED) endif() if(SQLite3_FOUND) + message(STATUS "Sqlite3 found. Requires ZLIB") set(ADIOS2_HAVE_SQLite3 TRUE) + find_package(ZLIB REQUIRED) endif() # Multithreading From e26b736050aa08340eac88d224a7008eb2ce32ef Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Fri, 17 Nov 2023 13:45:16 -0500 Subject: [PATCH 45/58] remove merge conflict line --- CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3205c66ff6..071cf5ec05 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -173,7 +173,6 @@ adios_option(Catalyst "Enable support for in situ visualization plugin using P adios_option(SQLite3 "Enable support for SQLite3 required by campaign manager" OFF) adios_option(AWSSDK "Enable support for S3 compatible storage using AWS SDK's S3 module" OFF) adios_option(Derived_Variable "Enable support for derived variables" OFF) ->>>>>>> master include(${PROJECT_SOURCE_DIR}/cmake/DetectOptions.cmake) if(ADIOS2_HAVE_CUDA OR ADIOS2_HAVE_Kokkos_CUDA) From a6a13c1aa49c50f6e6beea91c5ecb2e1333c15e1 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Mon, 20 Nov 2023 12:01:27 -0500 Subject: [PATCH 46/58] Replace cmake option SQLite3 for Campaign, which depends on SQLite3 and ZLIB. --- CMakeLists.txt | 7 ++++--- cmake/DetectOptions.cmake | 30 ++++++++++++++++++++---------- source/adios2/CMakeLists.txt | 2 +- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 071cf5ec05..de866e8363 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -170,7 +170,7 @@ adios_option(Profiling "Enable support for profiling" AUTO) adios_option(Endian_Reverse "Enable support for Little/Big Endian Interoperability" AUTO) adios_option(Sodium "Enable support for Sodium for encryption" AUTO) adios_option(Catalyst "Enable support for in situ visualization plugin using ParaView Catalyst" AUTO) -adios_option(SQLite3 "Enable support for SQLite3 required by campaign manager" OFF) +adios_option(Campaign "Enable support for Campaigns (requires SQLite3 and ZLIB)" OFF) adios_option(AWSSDK "Enable support for S3 compatible storage using AWS SDK's S3 module" OFF) adios_option(Derived_Variable "Enable support for derived variables" OFF) include(${PROJECT_SOURCE_DIR}/cmake/DetectOptions.cmake) @@ -246,7 +246,7 @@ set(ADIOS2_CONFIG_OPTS DataMan DataSpaces HDF5 HDF5_VOL MHS SST Fortran MPI Python Blosc2 BZip2 LIBPRESSIO MGARD PNG SZ ZFP DAOS IME O_DIRECT Sodium Catalyst SysVShMem UCX ZeroMQ Profiling Endian_Reverse Derived_Variable AWSSDK GPU_Support CUDA Kokkos - Kokkos_CUDA Kokkos_HIP Kokkos_SYCL SQLite3 + Kokkos_CUDA Kokkos_HIP Kokkos_SYCL Campaign ) GenerateADIOSHeaderConfig(${ADIOS2_CONFIG_OPTS}) @@ -412,7 +412,7 @@ message(" Examples: ${ADIOS2_BUILD_EXAMPLES}") message(" Build Options:") foreach(opt IN LISTS ADIOS2_CONFIG_OPTS) - message_pad(" ${opt}" 16 label) + message_pad(" ${opt}" 25 label) if(${ADIOS2_HAVE_${opt}}) message("${label}: ON") else() @@ -425,3 +425,4 @@ message(" RDMA Transport for Staging: Available") else() message(" RDMA Transport for Staging: Unconfigured") endif() + diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index 33684354c0..86f115f4e8 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -561,17 +561,27 @@ if(AWSSDK_FOUND) set(ADIOS2_HAVE_AWSSDK TRUE) endif() -# sqlite3 -if(ADIOS2_USE_SQLite3 STREQUAL AUTO) - find_package(SQLite3) -elseif(ADIOS2_USE_SQLite3) - find_package(SQLite3 REQUIRED) -endif() -if(SQLite3_FOUND) - message(STATUS "Sqlite3 found. Requires ZLIB") - set(ADIOS2_HAVE_SQLite3 TRUE) +# Campaign Management +if(ADIOS2_USE_Campaign STREQUAL AUTO) + find_package(SQLite3) + if (SQLite3_FOUND) + find_package(ZLIB) + endif() + if (SQLite3_FOUND AND ZLIB_FOUND) + message(STATUS "Sqlite3 and ZLIB found. Turn on Campaign Management") + set(ADIOS2_HAVE_Campaign TRUE) + endif() +elseif(ADIOS2_USE_Campaign) + find_package(SQLite3 REQUIRED) + if (SQLite3_FOUND) find_package(ZLIB REQUIRED) -endif() + endif() + if (SQLite3_FOUND AND ZLIB_FOUND) + message(STATUS "Sqlite3 and ZLIB found. Turn on Campaign Management") + set(ADIOS2_HAVE_Campaign TRUE) + endif() +endif() + # Multithreading find_package(Threads REQUIRED) diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index 200e05a437..3646c59051 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -205,7 +205,7 @@ if (ADIOS2_HAVE_SST) add_subdirectory(toolkit/remote) endif() -if(ADIOS2_HAVE_SQLite3) +if(ADIOS2_HAVE_Campaign) target_sources(adios2_core PRIVATE engine/campaign/CampaignReader.cpp engine/campaign/CampaignReader.tcc From 986eecd9c88effdef9acfffd73192776ab46b8fe Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Mon, 20 Nov 2023 14:49:22 -0500 Subject: [PATCH 47/58] Rename campaign extension to .aca, Adios Campaign Archive --- .../utils/adios_campaign_manager/adios2_campaign_manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/utils/adios_campaign_manager/adios2_campaign_manager.py b/source/utils/adios_campaign_manager/adios2_campaign_manager.py index 09857ecc4a..0b48d02dad 100755 --- a/source/utils/adios_campaign_manager/adios2_campaign_manager.py +++ b/source/utils/adios_campaign_manager/adios2_campaign_manager.py @@ -14,7 +14,7 @@ # from adios2.adios2_campaign_manager import * -ADIOS_ACM_VERSION = "1.0" +ADIOS_ACA_VERSION = "1.0" def SetupArgs(): @@ -45,7 +45,7 @@ def SetupArgs(): # default values args.update = False args.CampaignFileName = args.campaign_store + "/" + \ - args.project + "_" + args.app + "_" + args.shot + ".acm" + args.project + "_" + args.app + "_" + args.shot + ".aca" args.LocalCampaignDir = "adios-campaign/" # print("Verbosity = {0}".format(args.verbose)) @@ -239,7 +239,7 @@ def Create(args: dict, cur: sqlite3.Cursor): cur.execute( "create table info(id TEXT, name TEXT, version TEXT, ctime INT)") cur.execute('insert into info values (?, ?, ?, ?)', - ("ACM", "ADIOS Campaign Archive", ADIOS_ACM_VERSION, epoch)) + ("ACA", "ADIOS Campaign Archive", ADIOS_ACA_VERSION, epoch)) cur.execute("create table host" + "(hostname TEXT PRIMARY KEY, longhostname TEXT)") cur.execute("create table directory" + From 05326965c50be7c9951e1fa2e41e9114a3ffd1e7 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Mon, 20 Nov 2023 14:49:53 -0500 Subject: [PATCH 48/58] Fix building source without campaign support --- source/adios2/core/IO.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/source/adios2/core/IO.cpp b/source/adios2/core/IO.cpp index 7f8673ca20..bbc797038d 100644 --- a/source/adios2/core/IO.cpp +++ b/source/adios2/core/IO.cpp @@ -24,7 +24,6 @@ #include "adios2/engine/bp4/BP4Writer.h" #include "adios2/engine/bp5/BP5Reader.h" #include "adios2/engine/bp5/BP5Writer.h" -#include "adios2/engine/campaign/CampaignReader.h" #include "adios2/engine/inline/InlineReader.h" #include "adios2/engine/inline/InlineWriter.h" #include "adios2/engine/mhs/MhsReader.h" @@ -55,6 +54,10 @@ #include "adios2/engine/daos/DaosWriter.h" #endif +#ifdef ADIOS2_HAVE_CAMPAIGN // external dependencies +#include "adios2/engine/campaign/CampaignReader.h" +#endif + namespace adios2 { namespace core @@ -127,9 +130,16 @@ std::unordered_map Factory = { {IO::NoEngine("ERROR: nullcore engine does not support read mode"), IO::MakeEngine}}, {"plugin", {IO::MakeEngine, IO::MakeEngine}}, + {"campaign", +#ifdef ADIOS2_HAVE_CAMPAIGN {IO::MakeEngine, - IO::NoEngine("ERROR: campaign engine does not support write mode")}}, + IO::NoEngine("ERROR: campaign engine does not support write mode")} +#else + IO::NoEngineEntry("ERROR: this version didn't compile with " + "support for campaign management, can't use Campaign engine\n") +#endif + }, }; const std::unordered_map ReadRandomAccess_Supported = { From ab9205cc1ee79ed8708e636bf8248bb77c1297a4 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Mon, 15 May 2023 08:00:23 -0400 Subject: [PATCH 49/58] Added CampaignManager to be created by ADIOS object. Added CampaignReader engine. - convert info to json at Close() and write out. - add example to write many different files from different communicators and different ADIOS objects. - add explicit map for which engine supports ReadRandomAccess mode - save campaign data (in json) in adios-campaign/ - use SQLite3 database for campaign archive, compresses metadata - CampaignReader uses SQLite3 - check ctime of individual files and save from DB only if it is older - enable using multiple directories in same host - new parameter RemoteDataPath for BP5 engine, and Campaign Engine controls this parameter when opening a remote file (with metadata files locally present in another (local) path) --- cmake/DetectOptions.cmake | 1 - source/adios2/engine/campaign/CampaignReader.cpp | 1 - source/adios2/helper/adiosNetwork.cpp | 5 ++--- .../utils/adios_campaign_manager/adios2_campaign_manager.py | 4 ---- 4 files changed, 2 insertions(+), 9 deletions(-) diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index 86f115f4e8..bbf253bb49 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -582,7 +582,6 @@ elseif(ADIOS2_USE_Campaign) endif() endif() - # Multithreading find_package(Threads REQUIRED) diff --git a/source/adios2/engine/campaign/CampaignReader.cpp b/source/adios2/engine/campaign/CampaignReader.cpp index 0beb88bbc9..f825783142 100644 --- a/source/adios2/engine/campaign/CampaignReader.cpp +++ b/source/adios2/engine/campaign/CampaignReader.cpp @@ -159,7 +159,6 @@ void CampaignReader::InitParameters() void CampaignReader::InitTransports() { int rc; - rc = sqlite3_open(m_Name.c_str(), &m_DB); if (rc) { std::string dbmsg(sqlite3_errmsg(m_DB)); diff --git a/source/adios2/helper/adiosNetwork.cpp b/source/adios2/helper/adiosNetwork.cpp index 756254b933..33872dce3e 100644 --- a/source/adios2/helper/adiosNetwork.cpp +++ b/source/adios2/helper/adiosNetwork.cpp @@ -13,14 +13,13 @@ #include "adios2/toolkit/transport/file/FileFStream.h" #include // memcpy - -#ifndef _WIN32 +#include // gethostname #include //getFQDN #include //getFQDN #include //getFQDN -#include // gethostname +#ifndef _WIN32 #if defined(ADIOS2_HAVE_DATAMAN) || defined(ADIOS2_HAVE_TABLE) #include diff --git a/source/utils/adios_campaign_manager/adios2_campaign_manager.py b/source/utils/adios_campaign_manager/adios2_campaign_manager.py index 0b48d02dad..5b4b096517 100755 --- a/source/utils/adios_campaign_manager/adios2_campaign_manager.py +++ b/source/utils/adios_campaign_manager/adios2_campaign_manager.py @@ -16,7 +16,6 @@ ADIOS_ACA_VERSION = "1.0" - def SetupArgs(): parser = argparse.ArgumentParser() parser.add_argument( @@ -39,7 +38,6 @@ def SetupArgs(): parser.add_argument("--hostname", "-n", help="Host name unique for hosts in a campaign", required=False) - args = parser.parse_args() # default values @@ -134,7 +132,6 @@ def AddDatasetToArchive(args: dict, dataset: str, cur: sqlite3.Cursor, hostID: i ct = statres.st_ctime curDS = cur.execute('insert into bpdataset values (?, ?, ?, ?)', (hostID, dirID, dataset, ct)) - dsID = curDS.lastrowid cwd = getcwd() chdir(dataset) @@ -155,7 +152,6 @@ def ProcessJsonFile(args: dict, jsonlist: list, cur: sqlite3.Cursor, hostID: int if "name" in entry: AddDatasetToArchive( args, entry['name'], cur, hostID, dirID) - else: print(f"WARNING: your object is not a dictionary, skip : {entry}") From e8b561b977ee45de9b672cfe6905dff86605e8a9 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 21 Nov 2023 10:09:50 -0500 Subject: [PATCH 50/58] add back deleted line --- source/adios2/engine/campaign/CampaignReader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/adios2/engine/campaign/CampaignReader.cpp b/source/adios2/engine/campaign/CampaignReader.cpp index f825783142..c06dad0a96 100644 --- a/source/adios2/engine/campaign/CampaignReader.cpp +++ b/source/adios2/engine/campaign/CampaignReader.cpp @@ -158,7 +158,7 @@ void CampaignReader::InitParameters() void CampaignReader::InitTransports() { - int rc; + int rc = sqlite3_open(m_Name.c_str(), &m_DB); if (rc) { std::string dbmsg(sqlite3_errmsg(m_DB)); From 3d50e737316ad25bee616358c52cf2c25051d162 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 21 Nov 2023 11:40:40 -0500 Subject: [PATCH 51/58] do not include unistd on win32 --- source/adios2/helper/adiosNetwork.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/adios2/helper/adiosNetwork.cpp b/source/adios2/helper/adiosNetwork.cpp index 33872dce3e..c458f80f05 100644 --- a/source/adios2/helper/adiosNetwork.cpp +++ b/source/adios2/helper/adiosNetwork.cpp @@ -13,18 +13,20 @@ #include "adios2/toolkit/transport/file/FileFStream.h" #include // memcpy -#include // gethostname #include //getFQDN #include //getFQDN #include //getFQDN #ifndef _WIN32 + + #if defined(ADIOS2_HAVE_DATAMAN) || defined(ADIOS2_HAVE_TABLE) #include #include +#include // gethostname #include //AvailableIpAddresses() inet_ntoa #include //AvailableIpAddresses() struct if_nameindex #include //AvailableIpAddresses() struct sockaddr_in From 79f44635b8c474cf64ea0cfe20f09d7598faf16f Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 21 Nov 2023 11:51:46 -0500 Subject: [PATCH 52/58] format --- source/adios2/helper/adiosNetwork.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/adios2/helper/adiosNetwork.cpp b/source/adios2/helper/adiosNetwork.cpp index c458f80f05..d4554289a2 100644 --- a/source/adios2/helper/adiosNetwork.cpp +++ b/source/adios2/helper/adiosNetwork.cpp @@ -20,17 +20,16 @@ #ifndef _WIN32 - #if defined(ADIOS2_HAVE_DATAMAN) || defined(ADIOS2_HAVE_TABLE) #include #include -#include // gethostname #include //AvailableIpAddresses() inet_ntoa #include //AvailableIpAddresses() struct if_nameindex #include //AvailableIpAddresses() struct sockaddr_in #include //AvailableIpAddresses() ioctl +#include // gethostname #include From 17f4398be1f2645efe0f8c700b5746fbae8a03eb Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 21 Nov 2023 16:23:09 -0500 Subject: [PATCH 53/58] attempt to fix windows build --- source/adios2/helper/adiosNetwork.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/source/adios2/helper/adiosNetwork.cpp b/source/adios2/helper/adiosNetwork.cpp index d4554289a2..bdb2e76c61 100644 --- a/source/adios2/helper/adiosNetwork.cpp +++ b/source/adios2/helper/adiosNetwork.cpp @@ -14,12 +14,12 @@ #include // memcpy +#ifndef _WIN32 + #include //getFQDN #include //getFQDN #include //getFQDN -#ifndef _WIN32 - #if defined(ADIOS2_HAVE_DATAMAN) || defined(ADIOS2_HAVE_TABLE) #include @@ -34,7 +34,11 @@ #include #endif // ADIOS2_HAVE_DATAMAN || ADIOS2_HAVE_TABLE -#endif // _WIN32 + +#else // _WIN32 +#include +#include // GetComputerName +#endif // _WIN32 namespace adios2 { @@ -50,6 +54,7 @@ std::string GetFQDN() noexcept memset(hostname, 0, sizeof(hostname)); if (GetComputerName(infoBuf, &bufCharCount)) { + int i; for (i = 0; i < sizeof(hostname); i++) { hostname[i] = infoBuf[i]; From 92f5bd668e2c2a2becbd98b15f2e4a4583d3daa1 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 21 Nov 2023 16:52:47 -0500 Subject: [PATCH 54/58] attempt to fix linux build --- source/adios2/helper/adiosNetwork.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/adios2/helper/adiosNetwork.cpp b/source/adios2/helper/adiosNetwork.cpp index bdb2e76c61..2ae22a2b07 100644 --- a/source/adios2/helper/adiosNetwork.cpp +++ b/source/adios2/helper/adiosNetwork.cpp @@ -19,6 +19,7 @@ #include //getFQDN #include //getFQDN #include //getFQDN +#include // gethostname #if defined(ADIOS2_HAVE_DATAMAN) || defined(ADIOS2_HAVE_TABLE) @@ -29,7 +30,6 @@ #include //AvailableIpAddresses() struct if_nameindex #include //AvailableIpAddresses() struct sockaddr_in #include //AvailableIpAddresses() ioctl -#include // gethostname #include From b14d43e45458e314c594f50ee152c3eea5688e1b Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 12 Dec 2023 16:18:55 -0500 Subject: [PATCH 55/58] Compile CampaignManager.cpp only if ADIOS2_HAVE_CAMPAIGN, otherwise use empty functions for campaign recorder. --- source/adios2/CMakeLists.txt | 5 ++--- source/adios2/engine/campaign/CampaignManager.h | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index 225f4105a5..d02c6eeab4 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -65,8 +65,6 @@ add_library(adios2_core engine/inline/InlineReader.cpp engine/inline/InlineReader.tcc engine/inline/InlineWriter.cpp engine/inline/InlineWriter.tcc - engine/campaign/CampaignManager.cpp - engine/null/NullWriter.cpp engine/null/NullWriter.tcc engine/null/NullReader.cpp engine/null/NullReader.tcc @@ -210,7 +208,8 @@ if(ADIOS2_HAVE_Campaign) target_sources(adios2_core PRIVATE engine/campaign/CampaignReader.cpp engine/campaign/CampaignReader.tcc - engine/campaign/CampaignData.cpp) + engine/campaign/CampaignData.cpp + engine/campaign/CampaignManager.cpp) target_link_libraries(adios2_core PRIVATE SQLite::SQLite3 ZLIB::ZLIB) endif() diff --git a/source/adios2/engine/campaign/CampaignManager.h b/source/adios2/engine/campaign/CampaignManager.h index a39c448606..7a313f5627 100644 --- a/source/adios2/engine/campaign/CampaignManager.h +++ b/source/adios2/engine/campaign/CampaignManager.h @@ -15,11 +15,14 @@ #ifndef ADIOS2_ENGINE_CAMPAIGNMANAGER_H_ #define ADIOS2_ENGINE_CAMPAIGNMANAGER_H_ -#include "CampaignRecord.h" #include "adios2/common/ADIOSConfig.h" #include "adios2/helper/adiosComm.h" +#ifdef ADIOS2_HAVE_CAMPAIGN +#include "CampaignRecord.h" + #include +#endif /* ADIOS2_HAVE_CAMPAIGN */ namespace adios2 { @@ -31,7 +34,7 @@ namespace engine class CampaignManager { - +#ifdef ADIOS2_HAVE_CAMPAIGN public: CampaignManager(helper::Comm &comm); ~CampaignManager(); @@ -48,6 +51,16 @@ class CampaignManager CampaignRecordMap cmap; std::ofstream m_Output; const std::string m_CampaignDir = "adios-campaign"; + +#else +public: + CampaignManager(helper::Comm &comm){}; + ~CampaignManager() = default; + void Open(const std::string &name){}; + void Record(const std::string &name, const size_t step, const double time){}; + void Close(){}; + +#endif /* ADIOS2_HAVE_CAMPAIGN */ }; } // end namespace engine From 358b121d9f7d5d273e3fc9e447427764ce0da27e Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 12 Dec 2023 16:19:37 -0500 Subject: [PATCH 56/58] add the Campaign writer as example. This produces several files that can be added to a single campaign. It also supports restarting the application and adding more steps, so that a campaign archive can be updated later. --- examples/CMakeLists.txt | 1 + examples/campaign/CMakeLists.txt | 4 ++-- examples/campaign/campaign_write.cpp | 17 +++++++++++++++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 316e77723e..a255e411ed 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -8,3 +8,4 @@ add_subdirectory(hello) add_subdirectory(plugins) add_subdirectory(simulations) add_subdirectory(useCases) +add_subdirectory(campaign) diff --git a/examples/campaign/CMakeLists.txt b/examples/campaign/CMakeLists.txt index ad7b3aed0d..0ec67ac102 100644 --- a/examples/campaign/CMakeLists.txt +++ b/examples/campaign/CMakeLists.txt @@ -4,6 +4,6 @@ #------------------------------------------------------------------------------# if(ADIOS2_HAVE_MPI) - add_executable(campaign_write campaign_write.cpp) - target_link_libraries(campaign_write adios2::cxx11_mpi adios2_core MPI::MPI_C) + add_executable(adios2_campaign_write_mpi campaign_write.cpp) + target_link_libraries(adios2_campaign_write_mpi adios2::cxx11_mpi adios2_core MPI::MPI_C) endif() diff --git a/examples/campaign/campaign_write.cpp b/examples/campaign/campaign_write.cpp index 29c397a72c..00490bebf0 100644 --- a/examples/campaign/campaign_write.cpp +++ b/examples/campaign/campaign_write.cpp @@ -11,6 +11,9 @@ * other new outputs are created. * * Campaign management handles BP4/BP5 outputs + * + * Campaign management needs to be turned on explicitly at configuration time + * -DADIOS2_USE_Campaign=ON * * Outputs: * dataAll.bp: written by every process @@ -19,8 +22,18 @@ * dataStep{N}.bp a file series every step (written by all) * dataAnother.h5: written by rank 1..N from another ADIOS object * dataNew{N}.bp : a new file when restarting from step N - * dataFinal.bp: written by rank 1 from a third ADIOS object at the end - + * dataFinal.bp: written by rank 1 from a third ADIOS object at the end + * + * After running this example, the adios2_campaign_manager scripts must be used + * to create/update/delete a campaign archive from this run + * Then, the Campaign engine can be used in reading codes to use the campaign + * archive to read the content. + * + * E.g. + * adios2_campaign_manager -n example_campaign-write_101 -c ~/.campaign-store create + * adios2_campaign_manager -n example_campaign-write_101 -c ~/.campaign-store info + * bpls -E campaign -P "cachepath=/tmp/campaign" ~/.campaign-store/example_campaign-write_101.aca -la + * * Created on: May 17, 2023 * Author: Norbert Podhorszki */ From 265203544d8e470347ec7fadcfdc85eb97705ba3 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 12 Dec 2023 16:25:37 -0500 Subject: [PATCH 57/58] fix formatting --- examples/campaign/campaign_write.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/examples/campaign/campaign_write.cpp b/examples/campaign/campaign_write.cpp index 00490bebf0..aed73cc22a 100644 --- a/examples/campaign/campaign_write.cpp +++ b/examples/campaign/campaign_write.cpp @@ -11,7 +11,7 @@ * other new outputs are created. * * Campaign management handles BP4/BP5 outputs - * + * * Campaign management needs to be turned on explicitly at configuration time * -DADIOS2_USE_Campaign=ON * @@ -22,17 +22,18 @@ * dataStep{N}.bp a file series every step (written by all) * dataAnother.h5: written by rank 1..N from another ADIOS object * dataNew{N}.bp : a new file when restarting from step N - * dataFinal.bp: written by rank 1 from a third ADIOS object at the end + * dataFinal.bp: written by rank 1 from a third ADIOS object at the end * * After running this example, the adios2_campaign_manager scripts must be used * to create/update/delete a campaign archive from this run * Then, the Campaign engine can be used in reading codes to use the campaign - * archive to read the content. - * + * archive to read the content. + * * E.g. * adios2_campaign_manager -n example_campaign-write_101 -c ~/.campaign-store create * adios2_campaign_manager -n example_campaign-write_101 -c ~/.campaign-store info - * bpls -E campaign -P "cachepath=/tmp/campaign" ~/.campaign-store/example_campaign-write_101.aca -la + * bpls -E campaign -P "cachepath=/tmp/campaign" \ + * ~/.campaign-store/example_campaign-write_101.aca -la * * Created on: May 17, 2023 * Author: Norbert Podhorszki From 4f7367a414df36d1ff9f7abfa01b5bf626d13713 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Tue, 12 Dec 2023 17:16:30 -0500 Subject: [PATCH 58/58] remove unused variable to avoid warning in nvhpc build --- source/adios2/helper/adiosNetwork.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/adios2/helper/adiosNetwork.cpp b/source/adios2/helper/adiosNetwork.cpp index 2ae22a2b07..a29abd462b 100644 --- a/source/adios2/helper/adiosNetwork.cpp +++ b/source/adios2/helper/adiosNetwork.cpp @@ -66,7 +66,6 @@ std::string GetFQDN() noexcept } #else struct addrinfo hints, *info, *p; - int gai_result; hostname[1023] = '\0'; gethostname(hostname, 1023); @@ -76,7 +75,7 @@ std::string GetFQDN() noexcept hints.ai_socktype = SOCK_STREAM; hints.ai_flags = AI_CANONNAME; - if ((gai_result = getaddrinfo(hostname, NULL, &hints, &info)) == 0) + if (getaddrinfo(hostname, NULL, &hints, &info) == 0) { for (p = info; p != NULL; p = p->ai_next) {