Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Modify data file name functions in BP3 and BP4 #1787

Merged
merged 1 commit into from
Sep 26, 2019
Merged

Modify data file name functions in BP3 and BP4 #1787

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion source/adios2/engine/bp3/BP3Reader.tcc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ void BP3Reader::ReadVariableBlocks(Variable<T> &variable)
const std::string subFileName =
m_BP3Deserializer.GetBPSubFileName(
m_Name, subStreamBoxInfo.SubStreamID,
m_BP3Deserializer.m_Minifooter.HasSubFiles);
m_BP3Deserializer.m_Minifooter.HasSubFiles, true);

m_SubFileManager.OpenFileID(
subFileName, subStreamBoxInfo.SubStreamID, Mode::Read,
Expand Down
2 changes: 1 addition & 1 deletion source/adios2/engine/bp4/BP4Reader.tcc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ void BP4Reader::ReadVariableBlocks(Variable<T> &variable)
const std::string subFileName =
m_BP4Deserializer.GetBPSubFileName(
m_Name, subStreamBoxInfo.SubStreamID,
m_BP4Deserializer.m_Minifooter.HasSubFiles);
m_BP4Deserializer.m_Minifooter.HasSubFiles, true);

m_DataFileManager.OpenFileID(
subFileName, subStreamBoxInfo.SubStreamID, Mode::Read,
Expand Down
6 changes: 5 additions & 1 deletion source/adios2/toolkit/format/bp/BPBase.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,11 @@ void BPBase::Init(const Params &parameters, const std::string hint)
{
subStreams = m_SizeMPI;
}
m_Aggregator.Init(subStreams, m_Comm);

if (subStreams < m_SizeMPI)
{
m_Aggregator.Init(subStreams, m_Comm);
}
}
else if (key == "node-local")
{
Expand Down
12 changes: 7 additions & 5 deletions source/adios2/toolkit/format/bp/bp3/BP3Base.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,10 @@ BP3Base::GetBPSubStreamNames(const std::vector<std::string> &names) const

std::string BP3Base::GetBPSubFileName(const std::string &name,
const size_t subFileIndex,
const bool hasSubFiles) const noexcept
const bool hasSubFiles,
const bool isReader) const noexcept
{
return GetBPSubStreamName(name, subFileIndex, hasSubFiles);
return GetBPSubStreamName(name, subFileIndex, hasSubFiles, isReader);
}

size_t BP3Base::GetBPIndexSizeInData(const std::string &variableName,
Expand Down Expand Up @@ -142,8 +143,8 @@ BP3Base::ReadElementIndexHeader(const std::vector<char> &buffer,

// PRIVATE
std::string BP3Base::GetBPSubStreamName(const std::string &name,
const size_t rank,
const bool hasSubFiles) const noexcept
const size_t id, const bool hasSubFiles,
const bool isReader) const noexcept
{
if (!hasSubFiles)
{
Expand All @@ -162,7 +163,8 @@ std::string BP3Base::GetBPSubStreamName(const std::string &name,
}

const size_t index =
m_Aggregator.m_IsActive ? m_Aggregator.m_SubStreamIndex : rank;
isReader ? id
: m_Aggregator.m_IsActive ? m_Aggregator.m_SubStreamIndex : id;

const std::string bpRankName(bpName + ".dir" + PathSeparator + bpRoot +
"." + std::to_string(index));
Expand Down
9 changes: 5 additions & 4 deletions source/adios2/toolkit/format/bp/bp3/BP3Base.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,8 @@ class BP3Base : virtual public BPBase

std::string GetBPSubFileName(const std::string &name,
const size_t subFileIndex,
const bool hasSubFiles = true) const noexcept;
const bool hasSubFiles = true,
const bool isReader = false) const noexcept;

/**
* Returns the estimated variable index size. Used by ResizeBuffer public
Expand All @@ -100,9 +101,9 @@ class BP3Base : virtual public BPBase
noexcept final;

private:
std::string GetBPSubStreamName(const std::string &name, const size_t rank,
const bool hasSubFiles = true) const
noexcept;
std::string GetBPSubStreamName(const std::string &name, const size_t id,
const bool hasSubFiles = true,
const bool isReader = false) const noexcept;
};

} // end namespace format
Expand Down
12 changes: 7 additions & 5 deletions source/adios2/toolkit/format/bp/bp4/BP4Base.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,9 +113,10 @@ BP4Base::GetBPSubStreamNames(const std::vector<std::string> &names) const

std::string BP4Base::GetBPSubFileName(const std::string &name,
const size_t subFileIndex,
const bool hasSubFiles) const noexcept
const bool hasSubFiles,
const bool isReader) const noexcept
{
return GetBPSubStreamName(name, subFileIndex, hasSubFiles);
return GetBPSubStreamName(name, subFileIndex, hasSubFiles, isReader);
}

size_t BP4Base::GetBPIndexSizeInData(const std::string &variableName,
Expand Down Expand Up @@ -188,8 +189,8 @@ BP4Base::ReadElementIndexHeader(const std::vector<char> &buffer,

// PRIVATE
std::string BP4Base::GetBPSubStreamName(const std::string &name,
const size_t rank,
const bool hasSubFiles) const noexcept
const size_t id, const bool hasSubFiles,
const bool isReader) const noexcept
{
if (!hasSubFiles)
{
Expand All @@ -199,7 +200,8 @@ std::string BP4Base::GetBPSubStreamName(const std::string &name,
const std::string bpName = RemoveTrailingSlash(name);

const size_t index =
m_Aggregator.m_IsActive ? m_Aggregator.m_SubStreamIndex : rank;
isReader ? id
: m_Aggregator.m_IsActive ? m_Aggregator.m_SubStreamIndex : id;

/* the name of a data file starts with "data." */
const std::string bpRankName(bpName + PathSeparator + "data." +
Expand Down
9 changes: 5 additions & 4 deletions source/adios2/toolkit/format/bp/bp4/BP4Base.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,8 @@ class BP4Base : virtual public BPBase

std::string GetBPSubFileName(const std::string &name,
const size_t subFileIndex,
const bool hasSubFiles = true) const noexcept;
const bool hasSubFiles = true,
const bool isReader = false) const noexcept;

/**
* Returns the estimated variable index size. Used by ResizeBuffer public
Expand Down Expand Up @@ -139,9 +140,9 @@ class BP4Base : virtual public BPBase
noexcept final;

private:
std::string GetBPSubStreamName(const std::string &name, const size_t rank,
const bool hasSubFiles = true) const
noexcept;
std::string GetBPSubStreamName(const std::string &name, const size_t id,
const bool hasSubFiles = true,
const bool isReader = false) const noexcept;
};

} // end namespace format
Expand Down
33 changes: 16 additions & 17 deletions source/adios2/toolkit/format/bp/bp4/BP4Serializer.tcc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -436,23 +436,22 @@ size_t BP4Serializer::PutVariableMetadataInData(
position);

// here align pointer for span
if (span != nullptr)
{
const size_t padLengthPosition = position;
uint8_t zero = 0;
// skip 1 for paddingLength and 4 for VMD] ending
helper::CopyToBuffer(buffer, position, &zero, 5);
// here check for the next aligned pointer
const size_t extraBytes = m_Data.Align<T>();
const std::string pad = std::string(extraBytes, '\0') + "VMD]";

size_t backPosition = padLengthPosition;
const uint8_t padLength = static_cast<uint8_t>(pad.size());
helper::CopyToBuffer(buffer, backPosition, &padLength);
helper::CopyToBuffer(buffer, backPosition, pad.c_str(), pad.size());

position += extraBytes;
}

const size_t padLengthPosition = position;
uint8_t zero = 0;
// skip 1 for paddingLength and 4 for VMD] ending
helper::CopyToBuffer(buffer, position, &zero, 5);
// here check for the next aligned pointer
const size_t extraBytes = m_Data.Align<T>();
const std::string pad =
span == nullptr ? "VMD]" : std::string(extraBytes, '\0') + "VMD]";

size_t backPosition = padLengthPosition;
const uint8_t padLength = static_cast<uint8_t>(pad.size());
helper::CopyToBuffer(buffer, backPosition, &padLength);
helper::CopyToBuffer(buffer, backPosition, pad.c_str(), pad.size());

position += extraBytes;

absolutePosition += position - mdBeginPosition;
return varLengthPosition;
Expand Down
9 changes: 3 additions & 6 deletions testing/adios2/engine/bp/TestBPWriteAggregateRead.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,7 @@ void WriteAggRead1D8(const std::string substreams)
{
// Each process would write a 1x8 array and all processes would
// form a mpiSize * Nx 1D array
const std::string fname("ADIOS2BPWriteAggregateRead1D8_" + substreams +
".bp");
const std::string fname("BPWriteAggregateRead1D8_" + substreams + ".bp");

int mpiRank = 0, mpiSize = 1;
// Number of rows
Expand Down Expand Up @@ -322,8 +321,7 @@ void WriteAggRead2D4x2(const std::string substreams)
{
// Each process would write a 2x4 array and all processes would
// form a 2D 2 * (numberOfProcess*Nx) matrix where Nx is 4 here
const std::string fname("ADIOS2BPWriteAggregateRead2D2x4_" + substreams +
".bp");
const std::string fname("BPWriteAggregateRead2D2x4_" + substreams + ".bp");

int mpiRank = 0, mpiSize = 1;
// Number of rows
Expand Down Expand Up @@ -634,8 +632,7 @@ void WriteAggRead2D2x4(const std::string substreams)
{
// Each process would write a 4x2 array and all processes would
// form a 2D 4 * (NumberOfProcess * Nx) matrix where Nx is 2 here
const std::string fname("ADIOS2BPWriteAggregateRead2D4x2_" + substreams +
".bp");
const std::string fname("BPWriteAggregateRead2D4x2_" + substreams + ".bp");

int mpiRank = 0, mpiSize = 1;
// Number of rows
Expand Down