Skip to content

Commit

Permalink
Merge pull request #2634 from JasonRuonanWang/dataman
Browse files Browse the repository at this point in the history 
added mgard into dataman
  • Loading branch information
@JasonRuonanWang
JasonRuonanWang authored Feb 21, 2021
2 parents 649d293 + 0d451dd commit 19982f8
Show file tree
Hide file tree
Showing 3 changed files with 125 additions and 36 deletions.
4 changes: 4 additions & 0 deletions source/adios2/toolkit/format/dataman/DataManSerializer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,10 @@ bool DataManSerializer::IsCompressionAvailable(const std::string &method,
return true;
}
}
else if (method == "mgard")
{
return true;
}
return false;
}

Expand Down
10 changes: 7 additions & 3 deletions source/adios2/toolkit/format/dataman/DataManSerializer.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,15 +157,19 @@ class DataManSerializer

private:
template <class T>
bool PutZfp(nlohmann::json &metaj, size_t &datasize, const T *inputData,
void PutZfp(nlohmann::json &metaj, size_t &datasize, const T *inputData,
const Dims &varCount, const Params &params);

template <class T>
bool PutSz(nlohmann::json &metaj, size_t &datasize, const T *inputData,
void PutSz(nlohmann::json &metaj, size_t &datasize, const T *inputData,
const Dims &varCount, const Params &params);

template <class T>
bool PutBZip2(nlohmann::json &metaj, size_t &datasize, const T *inputData,
void PutBZip2(nlohmann::json &metaj, size_t &datasize, const T *inputData,
const Dims &varCount, const Params &params);

template <class T>
void PutMgard(nlohmann::json &metaj, size_t &datasize, const T *inputData,
const Dims &varCount, const Params &params);

template <class T>
Expand Down
147 changes: 114 additions & 33 deletions source/adios2/toolkit/format/dataman/DataManSerializer.tcc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@
#ifdef ADIOS2_HAVE_BZIP2
#include "adios2/operator/compress/CompressBZIP2.h"
#endif
#ifdef ADIOS2_HAVE_MGARD
#include "adios2/operator/compress/CompressMGARD.h"
#endif

#include "adios2/helper/adiosFunctions.h"

Expand Down Expand Up @@ -150,26 +153,67 @@ void DataManSerializer::PutData(
if (IsCompressionAvailable(compressionMethod,
helper::GetDataType<T>(), varCount))
{
compressed = PutZfp<T>(metaj, datasize, inputData, varCount,
ops[0].Parameters);
try
{
PutZfp<T>(metaj, datasize, inputData, varCount,
ops[0].Parameters);
compressed = true;
}
catch (std::exception &e)
{
std::cout << e.what() << std::endl;
}
}
}
else if (compressionMethod == "sz")
{
if (IsCompressionAvailable(compressionMethod,
helper::GetDataType<T>(), varCount))
{
compressed = PutSz<T>(metaj, datasize, inputData, varCount,
ops[0].Parameters);
try
{
PutSz<T>(metaj, datasize, inputData, varCount,
ops[0].Parameters);
compressed = true;
}
catch (std::exception &e)
{
std::cout << e.what() << std::endl;
}
}
}
else if (compressionMethod == "bzip2")
{
if (IsCompressionAvailable(compressionMethod,
helper::GetDataType<T>(), varCount))
{
compressed = PutBZip2<T>(metaj, datasize, inputData, varCount,
ops[0].Parameters);
try
{
PutBZip2<T>(metaj, datasize, inputData, varCount,
ops[0].Parameters);
compressed = true;
}
catch (std::exception &e)
{
std::cout << e.what() << std::endl;
}
}
}
else if (compressionMethod == "mgard")
{
if (IsCompressionAvailable(compressionMethod,
helper::GetDataType<T>(), varCount))
{
try
{
PutMgard<T>(metaj, datasize, inputData, varCount,
ops[0].Parameters);
compressed = true;
}
catch (std::exception &e)
{
std::cout << e.what() << std::endl;
}
}
}
else
Expand Down Expand Up @@ -295,16 +339,13 @@ int DataManSerializer::GetData(T *outputData, const std::string &varName,
}
catch (std::exception &e)
{
std::cout << "[DataManDeserializer::Get] Zfp "
"decompression failed with exception: "
<< e.what() << std::endl;
std::cout << e.what() << std::endl;
return -4; // decompression failed
}

input_data = decompressBuffer.data();
#else
throw std::runtime_error("ADIOS2 does not have ZFP");
return -101; // zfp library not found
#endif
}
else if (j.compression == "sz")
Expand All @@ -325,15 +366,12 @@ int DataManSerializer::GetData(T *outputData, const std::string &varName,
}
catch (std::exception &e)
{
std::cout << "[DataManDeserializer::Get] Zfp "
"decompression failed with exception: "
<< e.what() << std::endl;
std::cout << e.what() << std::endl;
return -4; // decompression failed
}
input_data = decompressBuffer.data();
#else
throw std::runtime_error("ADIOS2 does not have SZ");
return -102; // sz library not found
#endif
}
else if (j.compression == "bzip2")
Expand All @@ -355,15 +393,39 @@ int DataManSerializer::GetData(T *outputData, const std::string &varName,
}
catch (std::exception &e)
{
std::cout << "[DataManDeserializer::Get] Zfp "
"decompression failed with exception: "
<< e.what() << std::endl;
std::cout << e.what() << std::endl;
return -4; // decompression failed
}
input_data = decompressBuffer.data();
#else
throw std::runtime_error("ADIOS2 does not have Bzip2");
return -103; // bzip2 library not found
#endif
}
else if (j.compression == "mgard")
{
#ifdef ADIOS2_HAVE_MGARD
core::compress::CompressMGARD decompressor(j.params);
size_t datasize =
std::accumulate(j.count.begin(), j.count.end(), sizeof(T),
std::multiplies<size_t>());

decompressBuffer.reserve(datasize);
try
{
decompressor.Decompress(j.buffer->data() + j.position,
j.size, decompressBuffer.data(),
j.count, j.type, j.params);
decompressed = true;
}
catch (std::exception &e)
{
std::cout << e.what() << std::endl;
return -4; // decompression failed
}

input_data = decompressBuffer.data();
#else
throw std::runtime_error("ADIOS2 does not have MGARD");
#endif
}

Expand Down Expand Up @@ -404,7 +466,7 @@ int DataManSerializer::GetData(T *outputData, const std::string &varName,
}

template <class T>
bool DataManSerializer::PutZfp(nlohmann::json &metaj, size_t &datasize,
void DataManSerializer::PutZfp(nlohmann::json &metaj, size_t &datasize,
const T *inputData, const Dims &varCount,
const Params &params)
{
Expand All @@ -420,22 +482,19 @@ bool DataManSerializer::PutZfp(nlohmann::json &metaj, size_t &datasize,
datasize = compressor.Compress(inputData, varCount, sizeof(T),
helper::GetDataType<T>(),
m_CompressBuffer.data(), params, info);
return true;
}
catch (std::exception &e)
{
std::cout << "Got exception " << e.what()
<< " from ZFP. Turned off compression." << std::endl;
throw(e);
}
#else
throw(std::invalid_argument(
"ZFP compression used but ZFP library is not linked to ADIOS2"));
#endif
return false;
}

template <class T>
bool DataManSerializer::PutSz(nlohmann::json &metaj, size_t &datasize,
void DataManSerializer::PutSz(nlohmann::json &metaj, size_t &datasize,
const T *inputData, const Dims &varCount,
const Params &params)
{
Expand All @@ -451,22 +510,19 @@ bool DataManSerializer::PutSz(nlohmann::json &metaj, size_t &datasize,
datasize = compressor.Compress(inputData, varCount, sizeof(T),
helper::GetDataType<T>(),
m_CompressBuffer.data(), params, info);
return true;
}
catch (std::exception &e)
{
std::cout << "Got exception " << e.what()
<< " from SZ. Turned off compression." << std::endl;
throw(e);
}
#else
throw(std::invalid_argument(
"SZ compression used but SZ library is not linked to ADIOS2"));
#endif
return false;
}

template <class T>
bool DataManSerializer::PutBZip2(nlohmann::json &metaj, size_t &datasize,
void DataManSerializer::PutBZip2(nlohmann::json &metaj, size_t &datasize,
const T *inputData, const Dims &varCount,
const Params &params)
{
Expand All @@ -482,18 +538,43 @@ bool DataManSerializer::PutBZip2(nlohmann::json &metaj, size_t &datasize,
datasize = compressor.Compress(inputData, varCount, sizeof(T),
helper::GetDataType<T>(),
m_CompressBuffer.data(), params, info);
return true;
}
catch (std::exception &e)
{
std::cout << "Got exception " << e.what()
<< " from BZip2. Turned off compression." << std::endl;
throw(e);
}
#else
throw(std::invalid_argument(
"BZip2 compression used but BZip2 library is not linked to ADIOS2"));
#endif
return false;
}

template <class T>
void DataManSerializer::PutMgard(nlohmann::json &metaj, size_t &datasize,
const T *inputData, const Dims &varCount,
const Params &params)
{
TAU_SCOPED_TIMER_FUNC();
#ifdef ADIOS2_HAVE_MGARD
core::compress::CompressMGARD compressor(params);
m_CompressBuffer.reserve(std::accumulate(varCount.begin(), varCount.end(),
sizeof(T),
std::multiplies<size_t>()));
try
{
Params info;
datasize = compressor.Compress(inputData, varCount, sizeof(T),
helper::GetDataType<T>(),
m_CompressBuffer.data(), params, info);
}
catch (std::exception &e)
{
throw(e);
}
#else
throw(std::invalid_argument(
"MGARD compression used but MGARD library is not linked to ADIOS2"));
#endif
}

template <class T>
Expand Down

0 comments on commit 19982f8

Please sign in to comment.