Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

WIP: Alternative path to struct (BP)Info #2565

Closed
wants to merge 6 commits into from
Closed

WIP: Alternative path to struct (BP)Info #2565

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f986814
Rename struct Info to struct BPInfo
eisenhauer Nov 29, 2020
a496130
MinInfo
eisenhauer Jan 21, 2021
02637f3
Kill unref
eisenhauer Jan 21, 2021
ad0a757
Manage memory, default to MinBlocks
eisenhauer Jan 21, 2021
dee3585
blocks info on demand for SST/FFS-CP
eisenhauer Jan 26, 2021
a88495d
Delete generated core blocksinfo
eisenhauer Jan 26, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 67 additions & 4 deletions bindings/CXX11/adios2/cxx11/Engine.tcc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,14 @@ namespace
template <class T>
static std::vector<typename Variable<T>::Info>
ToBlocksInfo(const std::vector<typename core::Variable<
typename TypeInfo<T>::IOType>::Info> &coreBlocksInfo)
typename TypeInfo<T>::IOType>::BPInfo> &coreBlocksInfo)
{
using IOType = typename TypeInfo<T>::IOType;

std::vector<typename Variable<T>::Info> blocksInfo;
blocksInfo.reserve(coreBlocksInfo.size());

for (const typename core::Variable<IOType>::Info &coreBlockInfo :
for (const typename core::Variable<IOType>::BPInfo &coreBlockInfo :
coreBlocksInfo)
{
typename Variable<T>::Info blockInfo;
Expand All @@ -59,6 +59,58 @@ ToBlocksInfo(const std::vector<typename core::Variable<

return blocksInfo;
}
template <class T>
static std::vector<typename Variable<T>::Info>
ToBlocksInfo(const core::Engine::MinVarInfo *coreVarInfo)
{
using IOType = typename TypeInfo<T>::IOType;

auto coreBlocksInfo = coreVarInfo->BlocksInfo;

std::vector<typename Variable<T>::Info> blocksInfo;
blocksInfo.reserve(coreBlocksInfo.size());

for (auto &coreBlockInfo : coreBlocksInfo)
{
typename Variable<T>::Info blockInfo;

if (coreVarInfo->Shape)
{
blockInfo.Start.reserve(coreVarInfo->Dims);
blockInfo.Count.reserve(coreVarInfo->Dims);
for (int i = 0; i < coreVarInfo->Dims; i++)
{
blockInfo.Start.push_back(coreBlockInfo.Start[i]);
blockInfo.Count.push_back(coreBlockInfo.Count[i]);
}
}
else
{
blockInfo.Count.reserve(coreVarInfo->Dims);
for (int i = 0; i < coreVarInfo->Dims; i++)
{
blockInfo.Count.push_back(coreBlockInfo.Count[i]);
}
}
blockInfo.WriterID = coreBlockInfo.WriterID;

blockInfo.IsValue = coreVarInfo->IsValue;
blockInfo.IsReverseDims = coreVarInfo->IsReverseDims;
if (blockInfo.IsValue)
{
blockInfo.Value = *((T *)coreBlockInfo.BufferP);
}
else
{
blockInfo.Min = *((T *)&coreBlockInfo.MinUnion);
blockInfo.Max = *((T *)&coreBlockInfo.MaxUnion);
}
blockInfo.BlockID = coreBlockInfo.BlockID;
blocksInfo.push_back(blockInfo);
}

return blocksInfo;
}
} // end empty namespace

template <class T>
Expand Down Expand Up @@ -294,7 +346,7 @@ Engine::AllStepsBlocksInfo(const Variable<T> variable) const
variable.m_Variable,
"for variable in call to Engine::AllStepsBlocksInfo");

const std::map<size_t, std::vector<typename core::Variable<IOType>::Info>>
const std::map<size_t, std::vector<typename core::Variable<IOType>::BPInfo>>
coreAllStepsBlockInfo =
m_Engine->AllStepsBlocksInfo(*variable.m_Variable);

Expand All @@ -304,7 +356,7 @@ Engine::AllStepsBlocksInfo(const Variable<T> variable) const
for (const auto &pair : coreAllStepsBlockInfo)
{
const size_t step = pair.first;
const std::vector<typename core::Variable<IOType>::Info>
const std::vector<typename core::Variable<IOType>::BPInfo>
&coreBlocksInfo = pair.second;

allStepsBlocksInfo[step] = ToBlocksInfo<T>(coreBlocksInfo);
Expand Down Expand Up @@ -333,6 +385,17 @@ Engine::BlocksInfo(const Variable<T> variable, const size_t step) const
adios2::helper::CheckForNullptr(
variable.m_Variable, "for variable in call to Engine::BlocksInfo");

const auto minBlocksInfo =
m_Engine->MinBlocksInfo(*variable.m_Variable, step);

if (minBlocksInfo)
{
std::vector<typename Variable<T>::Info> Ret =
ToBlocksInfo<T>(minBlocksInfo);
delete minBlocksInfo;
return Ret;
}

const auto blocksInfo =
m_Engine->BlocksInfo<IOType>(*variable.m_Variable, step);
return ToBlocksInfo<T>(blocksInfo);
Expand Down
4 changes: 2 additions & 2 deletions bindings/CXX11/adios2/cxx11/Variable.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,8 +213,8 @@ namespace adios2
template <> \
const T *Variable<T>::Info::Data() const \
{ \
const core::Variable<T>::Info *coreInfo = \
reinterpret_cast<const core::Variable<T>::Info *>(m_Info); \
const core::Variable<T>::BPInfo *coreInfo = \
reinterpret_cast<const core::Variable<T>::BPInfo *>(m_Info); \
\
return m_Info ? (coreInfo->BufferP ? coreInfo->BufferP \
: coreInfo->BufferV.data()) \
Expand Down
6 changes: 3 additions & 3 deletions bindings/CXX11/adios2/cxx11/Variable.tcc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,14 @@ namespace
template <class T>
std::vector<typename Variable<T>::Info>
ToBlocksInfo(const std::vector<typename core::Variable<
typename TypeInfo<T>::IOType>::Info> &coreBlocksInfo)
typename TypeInfo<T>::IOType>::BPInfo> &coreBlocksInfo)
{
using IOType = typename TypeInfo<T>::IOType;

std::vector<typename Variable<T>::Info> blocksInfo;
blocksInfo.reserve(coreBlocksInfo.size());

for (const typename core::Variable<IOType>::Info &coreBlockInfo :
for (const typename core::Variable<IOType>::BPInfo &coreBlockInfo :
coreBlocksInfo)
{
typename Variable<T>::Info blockInfo;
Expand Down Expand Up @@ -69,7 +69,7 @@ Variable<T>::DoAllStepsBlocksInfo()
"in call to Variable<T>::AllStepsBlocksInfo");

// PRIVATE INPUT
const std::vector<std::vector<typename core::Variable<IOType>::Info>>
const std::vector<std::vector<typename core::Variable<IOType>::BPInfo>>
coreAllStepsBlocksInfo = m_Variable->AllStepsBlocksInfo();

// PUBLIC OUTPUT
Expand Down
30 changes: 15 additions & 15 deletions source/adios2/core/Engine.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,11 +126,11 @@ ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
{ \
ThrowUp("DoGetDeferred"); \
} \
typename Variable<T>::Info *Engine::DoGetBlockSync(Variable<T> &v) \
typename Variable<T>::BPInfo *Engine::DoGetBlockSync(Variable<T> &v) \
{ \
return nullptr; \
} \
typename Variable<T>::Info *Engine::DoGetBlockDeferred(Variable<T> &v) \
typename Variable<T>::BPInfo *Engine::DoGetBlockDeferred(Variable<T> &v) \
{ \
return nullptr; \
}
Expand All @@ -139,25 +139,25 @@ ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
#undef declare_type

#define declare_type(T) \
std::map<size_t, std::vector<typename Variable<T>::Info>> \
std::map<size_t, std::vector<typename Variable<T>::BPInfo>> \
Engine::DoAllStepsBlocksInfo(const Variable<T> &variable) const \
{ \
ThrowUp("DoAllStepsBlocksInfo"); \
return std::map<size_t, std::vector<typename Variable<T>::Info>>(); \
return std::map<size_t, std::vector<typename Variable<T>::BPInfo>>(); \
} \
\
std::vector<std::vector<typename Variable<T>::Info>> \
std::vector<std::vector<typename Variable<T>::BPInfo>> \
Engine::DoAllRelativeStepsBlocksInfo(const Variable<T> &variable) const \
{ \
ThrowUp("DoAllRelativeStepsBlocksInfo"); \
return std::vector<std::vector<typename Variable<T>::Info>>(); \
return std::vector<std::vector<typename Variable<T>::BPInfo>>(); \
} \
\
std::vector<typename Variable<T>::Info> Engine::DoBlocksInfo( \
std::vector<typename Variable<T>::BPInfo> Engine::DoBlocksInfo( \
const Variable<T> &variable, const size_t step) const \
{ \
ThrowUp("DoBlocksInfo"); \
return std::vector<typename Variable<T>::Info>(); \
return std::vector<typename Variable<T>::BPInfo>(); \
} \
std::vector<size_t> Engine::DoGetAbsoluteSteps( \
const Variable<T> &variable) const \
Expand Down Expand Up @@ -223,21 +223,21 @@ void Engine::CheckOpenModes(const std::set<Mode> &modes,
template void Engine::Get<T>(const std::string &, std::vector<T> &, \
const Mode); \
\
template typename Variable<T>::Info *Engine::Get<T>(Variable<T> &, \
const Mode); \
template typename Variable<T>::Info *Engine::Get<T>(const std::string &, \
const Mode); \
template typename Variable<T>::BPInfo *Engine::Get<T>(Variable<T> &, \
const Mode); \
template typename Variable<T>::BPInfo *Engine::Get<T>(const std::string &, \
const Mode); \
\
template Variable<T> &Engine::FindVariable( \
const std::string &variableName, const std::string hint); \
\
template std::map<size_t, std::vector<typename Variable<T>::Info>> \
template std::map<size_t, std::vector<typename Variable<T>::BPInfo>> \
Engine::AllStepsBlocksInfo(const Variable<T> &) const; \
\
template std::vector<std::vector<typename Variable<T>::Info>> \
template std::vector<std::vector<typename Variable<T>::BPInfo>> \
Engine::AllRelativeStepsBlocksInfo(const Variable<T> &) const; \
\
template std::vector<typename Variable<T>::Info> Engine::BlocksInfo( \
template std::vector<typename Variable<T>::BPInfo> Engine::BlocksInfo( \
const Variable<T> &, const size_t) const; \
template std::vector<size_t> Engine::GetAbsoluteSteps(const Variable<T> &) \
const;
Expand Down
82 changes: 64 additions & 18 deletions source/adios2/core/Engine.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -314,8 +314,8 @@ class Engine
* </pre>
*/
template <class T>
typename Variable<T>::Info *Get(Variable<T> &variable,
const Mode launch = Mode::Deferred);
typename Variable<T>::BPInfo *Get(Variable<T> &variable,
const Mode launch = Mode::Deferred);

/**
* @brief Get version for block selection that accepts a variableName as
Expand All @@ -341,8 +341,8 @@ class Engine
* </pre>
*/
template <class T>
typename Variable<T>::Info *Get(const std::string &variableName,
const Mode launch = Mode::Deferred);
typename Variable<T>::BPInfo *Get(const std::string &variableName,
const Mode launch = Mode::Deferred);

template <class T>
void Get(core::Variable<T> &, T **) const;
Expand Down Expand Up @@ -390,7 +390,7 @@ class Engine
* </pre>
*/
template <class T>
std::map<size_t, std::vector<typename Variable<T>::Info>>
std::map<size_t, std::vector<typename Variable<T>::BPInfo>>
AllStepsBlocksInfo(const Variable<T> &variable) const;

/**
Expand All @@ -400,7 +400,7 @@ class Engine
* @return
*/
template <class T>
std::vector<std::vector<typename Variable<T>::Info>>
std::vector<std::vector<typename Variable<T>::BPInfo>>
AllRelativeStepsBlocksInfo(const Variable<T> &variable) const;

/**
Expand All @@ -413,7 +413,7 @@ class Engine
* found it returns an empty vector
*/
template <class T>
std::vector<typename Variable<T>::Info>
std::vector<typename Variable<T>::BPInfo>
BlocksInfo(const Variable<T> &variable, const size_t step) const;

/**
Expand Down Expand Up @@ -450,6 +450,51 @@ class Engine
/* for adios2 internal testing */
virtual size_t DebugGetDataBufferSize() const;

union PrimitiveStdtypeUnion
{
int8_t int8;
int16_t int16;
int32_t int32;
int64_t int64;
uint8_t uint8;
uint16_t uint16;
uint32_t uint32;
uint64_t uint64;
float f;
double d;
long double ld;
};

struct MinBlockInfo
{
int WriterID = 0;
size_t BlockID = 0;
size_t *Start;
size_t *Count;
union PrimitiveStdtypeUnion MinUnion;
union PrimitiveStdtypeUnion MaxUnion;
void *BufferP = NULL;
};
struct MinVarInfo
{
int Dims;
size_t *Shape;
bool IsValue = false;
bool IsReverseDims = false;
std::vector<struct MinBlockInfo> BlocksInfo;
MinVarInfo(int D, size_t *S)
: Dims(D), Shape(S), IsValue(false), IsReverseDims(false),
BlocksInfo({})
{
}
};

virtual MinVarInfo *MinBlocksInfo(const VariableBase &,
const size_t Step) const
{
return nullptr;
}

protected:
/** from ADIOS class passed to Engine created with Open
* if no communicator is passed */
Expand Down Expand Up @@ -494,8 +539,8 @@ class Engine
#define declare_type(T) \
virtual void DoGetSync(Variable<T> &, T *); \
virtual void DoGetDeferred(Variable<T> &, T *); \
virtual typename Variable<T>::Info *DoGetBlockSync(Variable<T> &); \
virtual typename Variable<T>::Info *DoGetBlockDeferred(Variable<T> &);
virtual typename Variable<T>::BPInfo *DoGetBlockSync(Variable<T> &); \
virtual typename Variable<T>::BPInfo *DoGetBlockDeferred(Variable<T> &);
ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
#undef declare_type

Expand All @@ -512,13 +557,13 @@ class Engine
const std::string hint);

#define declare_type(T) \
virtual std::map<size_t, std::vector<typename Variable<T>::Info>> \
virtual std::map<size_t, std::vector<typename Variable<T>::BPInfo>> \
DoAllStepsBlocksInfo(const Variable<T> &variable) const; \
\
virtual std::vector<std::vector<typename Variable<T>::Info>> \
virtual std::vector<std::vector<typename Variable<T>::BPInfo>> \
DoAllRelativeStepsBlocksInfo(const Variable<T> &variable) const; \
\
virtual std::vector<typename Variable<T>::Info> DoBlocksInfo( \
virtual std::vector<typename Variable<T>::BPInfo> DoBlocksInfo( \
const Variable<T> &variable, const size_t step) const; \
virtual std::vector<size_t> DoGetAbsoluteSteps( \
const Variable<T> &variable) const;
Expand Down Expand Up @@ -592,21 +637,22 @@ class Engine
extern template void Engine::Get<T>(const std::string &, std::vector<T> &, \
const Mode); \
\
extern template typename Variable<T>::Info *Engine::Get<T>(Variable<T> &, \
const Mode); \
extern template typename Variable<T>::Info *Engine::Get<T>( \
extern template typename Variable<T>::BPInfo *Engine::Get<T>( \
Variable<T> &, const Mode); \
extern template typename Variable<T>::BPInfo *Engine::Get<T>( \
const std::string &, const Mode); \
\
extern template Variable<T> &Engine::FindVariable( \
const std::string &variableName, const std::string hint); \
\
extern template std::map<size_t, std::vector<typename Variable<T>::Info>> \
extern template std::map<size_t, \
std::vector<typename Variable<T>::BPInfo>> \
Engine::AllStepsBlocksInfo(const Variable<T> &) const; \
\
extern template std::vector<std::vector<typename Variable<T>::Info>> \
extern template std::vector<std::vector<typename Variable<T>::BPInfo>> \
Engine::AllRelativeStepsBlocksInfo(const Variable<T> &) const; \
\
extern template std::vector<typename Variable<T>::Info> \
extern template std::vector<typename Variable<T>::BPInfo> \
Engine::BlocksInfo(const Variable<T> &, const size_t) const; \
\
extern template std::vector<size_t> Engine::GetAbsoluteSteps( \
Expand Down
Loading