From 0cebe966ea78fa667fd2975abe177abe1584b81d Mon Sep 17 00:00:00 2001 From: Caitlin Ross Date: Sat, 16 May 2020 14:30:50 -0400 Subject: [PATCH] adding inline engine heat transfer example/test --- examples/heatTransfer/CMakeLists.txt | 1 + examples/heatTransfer/heat_inline.xml | 38 ++++ examples/heatTransfer/inline/CMakeLists.txt | 14 ++ examples/heatTransfer/inline/InlineIO.cpp | 94 +++++++++ examples/heatTransfer/inline/InlineIO.h | 43 ++++ examples/heatTransfer/inline/main.cpp | 191 ++++++++++++++++++ testing/examples/heatTransfer/CMakeLists.txt | 2 + .../examples/heatTransfer/TestInlineMxM.cmake | 31 +++ 8 files changed, 414 insertions(+) create mode 100644 examples/heatTransfer/heat_inline.xml create mode 100644 examples/heatTransfer/inline/CMakeLists.txt create mode 100644 examples/heatTransfer/inline/InlineIO.cpp create mode 100644 examples/heatTransfer/inline/InlineIO.h create mode 100644 examples/heatTransfer/inline/main.cpp create mode 100644 testing/examples/heatTransfer/TestInlineMxM.cmake diff --git a/examples/heatTransfer/CMakeLists.txt b/examples/heatTransfer/CMakeLists.txt index 52f9cab1af..48a07f6fd2 100644 --- a/examples/heatTransfer/CMakeLists.txt +++ b/examples/heatTransfer/CMakeLists.txt @@ -6,3 +6,4 @@ add_subdirectory(write) add_subdirectory(read) add_subdirectory(read_fileonly) +add_subdirectory(inline) diff --git a/examples/heatTransfer/heat_inline.xml b/examples/heatTransfer/heat_inline.xml new file mode 100644 index 0000000000..7de2cb351b --- /dev/null +++ b/examples/heatTransfer/heat_inline.xml @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/heatTransfer/inline/CMakeLists.txt b/examples/heatTransfer/inline/CMakeLists.txt new file mode 100644 index 0000000000..54fd8de038 --- /dev/null +++ b/examples/heatTransfer/inline/CMakeLists.txt @@ -0,0 +1,14 @@ +#------------------------------------------------------------------------------# +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +#------------------------------------------------------------------------------# + +add_executable(heatTransfer_inline_adios2 + main.cpp + InlineIO.cpp + ../write/HeatTransfer.cpp + ../write/Settings.cpp +) +target_link_libraries(heatTransfer_inline_adios2 + adios2::cxx11_mpi MPI::MPI_C ${CMAKE_THREAD_LIBS_INIT} +) diff --git a/examples/heatTransfer/inline/InlineIO.cpp b/examples/heatTransfer/inline/InlineIO.cpp new file mode 100644 index 0000000000..61bfb50156 --- /dev/null +++ b/examples/heatTransfer/inline/InlineIO.cpp @@ -0,0 +1,94 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * InlineIO.cpp + * + * Created on: May 2020 + * Author: Caitlin Ross + */ + +#include "InlineIO.h" + +InlineIO::InlineIO(const Settings &s, MPI_Comm comm) +{ + std::string writerName = "writer"; + std::string readerName = "reader"; + + ad = adios2::ADIOS(s.configfile, comm); + inlineIO = ad.DeclareIO("inlineIO"); + + auto params = inlineIO.Parameters(); + if (params.find("writerID") != params.end()) + { + writerName = params["writerID"]; + } + if (params.find("readerID") != params.end()) + { + readerName = params["readerID"]; + } + + if (!inlineIO.InConfigFile()) + { + inlineIO.SetEngine("Inline"); + inlineIO.SetParameters({{"writerID", writerName}, {"readerID", readerName}}); + } + + // define T as 2D global array + varT = inlineIO.DefineVariable( + "T", + // Global dimensions + {s.gndx, s.gndy}, + // starting offset of the local array in the global space + {s.offsx, s.offsy}, + // local size, could be defined later using SetSelection() + {s.ndx, s.ndy}); + + inlineWriter = inlineIO.Open(writerName, adios2::Mode::Write, comm); + inlineReader = inlineIO.Open(readerName, adios2::Mode::Read, comm); + + // Promise that we are not going to change the variable sizes nor add new + // variables + inlineWriter.LockWriterDefinitions(); + inlineReader.LockReaderSelections(); +} + +InlineIO::~InlineIO() +{ + inlineWriter.Close(); + inlineReader.Close(); +} + +void InlineIO::write(const HeatTransfer& ht) +{ + inlineWriter.BeginStep(); + v = ht.data_noghost(); + inlineWriter.Put(varT, v.data()); + inlineWriter.EndStep(); +} + +const double* InlineIO::read(bool firstStep) +{ + adios2::StepStatus status = + inlineReader.BeginStep(adios2::StepMode::Read); + if (status != adios2::StepStatus::OK) + { + return nullptr; + } + + auto blocksInfo = inlineReader.BlocksInfo(varT, inlineReader.CurrentStep()); + // in this example we're only expecting one block + if (blocksInfo.size() != 1) + { + throw std::runtime_error("InlineIO::read found incorrect number of blocks"); + } + + auto& info = blocksInfo[0]; + varT.SetBlockSelection(info.BlockID); + inlineReader.Get(varT, info); + inlineReader.EndStep(); + + // now we can get the pointer with info.Data() + return info.Data(); +} + diff --git a/examples/heatTransfer/inline/InlineIO.h b/examples/heatTransfer/inline/InlineIO.h new file mode 100644 index 0000000000..8a4fb89ae6 --- /dev/null +++ b/examples/heatTransfer/inline/InlineIO.h @@ -0,0 +1,43 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * InlineIO.h + * + * Created on: May 2020 + * Author: Caitlin Ross + */ + +#ifndef InlineIO_H_ +#define InlineIO_H_ + +#include "../write/HeatTransfer.h" +#include "../write/Settings.h" + +#include + +#include "adios2.h" + +class InlineIO +{ +public: + InlineIO(const Settings &s, MPI_Comm comm); + ~InlineIO(); + void write(const HeatTransfer &ht); + const double* read(bool firstStep); + +private: + adios2::ADIOS ad; + adios2::IO inlineIO; + adios2::Engine inlineWriter; + adios2::Engine inlineReader; + + adios2::Variable varT; + + // need this so data on the write side doesn't go out + // of scope before the reader can use it + std::vector v; + +}; + +#endif /* InlineIO_H_ */ diff --git a/examples/heatTransfer/inline/main.cpp b/examples/heatTransfer/inline/main.cpp new file mode 100644 index 0000000000..413dfad0de --- /dev/null +++ b/examples/heatTransfer/inline/main.cpp @@ -0,0 +1,191 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * main.cpp + * + * Recreates heat_transfer.f90 (Fortran) ADIOS tutorial example in C++ + * This version is adapted from heatTransfer/write/main.cpp and + * heatTransfer/read/heatRead.cpp for use with the inline engine, + * which requires the writer and reader to be created in the same process + * and the same ADIOS IO object. + * + * Created on: May 2020 + * Author: Norbert Podhorszki + * Caitlin Ross + * + */ +#include + +#include "adios2.h" + +#include +#include +#include +#include + +#include "InlineIO.h" +#include "../write/HeatTransfer.h" +#include "../write/Settings.h" + +void printUsage() +{ + std::cout << "Usage: heatTransfer config output N M nx ny steps " + "iterations\n" + << " config: XML config file to use\n" + << " output: name of output data file/stream\n" + << " N: number of processes in X dimension\n" + << " M: number of processes in Y dimension\n" + << " nx: local array size in X dimension per processor\n" + << " ny: local array size in Y dimension per processor\n" + << " steps: the total number of steps to output\n" + << " iterations: one step consist of this many iterations\n\n"; +} + +void Compute(const double* Tin, std::vector &Tout, + std::vector &dT, bool firstStep) +{ + /* Compute dT and + * copy Tin into Tout as it will be used for calculating dT in the + * next step + */ + if (firstStep) + { + for (size_t i = 0; i < dT.size(); ++i) + { + dT[i] = 0; + Tout[i] = Tin[i]; + } + } + else + { + for (size_t i = 0; i < dT.size(); ++i) + { + dT[i] = Tout[i] - Tin[i]; + Tout[i] = Tin[i]; + } + } +} + +std::vector Tout; +std::vector dT; +adios2::Variable vTout; +adios2::Variable vdT; +adios2::IO outIO; +adios2::ADIOS ad; +adios2::Engine writer; + +void writeOutput() +{ + writer.BeginStep(); + if (vTout) + writer.Put(vTout, Tout.data()); + if (vdT) + writer.Put(vdT, dT.data()); + writer.EndStep(); +} + +void setupOutputIO(const Settings& s, MPI_Comm comm) +{ + ad = adios2::ADIOS(s.configfile, comm); + outIO = ad.DeclareIO("readerOutput"); + Tout.resize(s.ndx * s.ndy); + dT.resize(s.ndx * s.ndy); + + /* Create output variables and open output stream */ + // For inline engine, there's no exchange of data between processes, + // so the shape of variables to be written out for validation + // is the same as the writer + vTout = outIO.DefineVariable( + "T", {s.gndx, s.gndy}, {s.offsx, s.offsy}, {s.ndx, s.ndy}); + vdT = outIO.DefineVariable( + "dT", {s.gndx, s.gndy}, {s.offsx, s.offsy}, {s.ndx, s.ndy}); + writer = outIO.Open(s.outputfile, adios2::Mode::Write, + comm); +} + +int main(int argc, char *argv[]) +{ + MPI_Init(&argc, &argv); + + /* When writer and reader is launched together with a single mpirun command, + the world comm spans all applications. We have to split and create the + local 'world' communicator mpiHeatTransferComm for the writer only. + When writer and reader is launched separately, the mpiHeatTransferComm + communicator will just equal the MPI_COMM_WORLD. + */ + + int wrank, wnproc; + MPI_Comm_rank(MPI_COMM_WORLD, &wrank); + MPI_Comm_size(MPI_COMM_WORLD, &wnproc); + + const unsigned int color = 1; + MPI_Comm mpiHeatTransferComm; + MPI_Comm_split(MPI_COMM_WORLD, color, wrank, &mpiHeatTransferComm); + + int rank, nproc; + MPI_Comm_rank(mpiHeatTransferComm, &rank); + MPI_Comm_size(mpiHeatTransferComm, &nproc); + + try + { + double timeStart = MPI_Wtime(); + Settings settings(argc, argv, rank, nproc); + HeatTransfer ht(settings); + InlineIO io(settings, mpiHeatTransferComm); + setupOutputIO(settings, mpiHeatTransferComm); + + ht.init(false); + // ht.printT("Initialized T:", mpiHeatTransferComm); + ht.heatEdges(); + ht.exchange(mpiHeatTransferComm); + // ht.printT("Heated T:", mpiHeatTransferComm); + + io.write(ht); + const double* Tin = io.read(true); + Compute(Tin, Tout, dT, true); + writeOutput(); + + for (unsigned int t = 1; t <= settings.steps; ++t) + { + if (rank == 0) + std::cout << "Step " << t << ":\n"; + for (unsigned int iter = 1; iter <= settings.iterations; ++iter) + { + ht.iterate(); + ht.exchange(mpiHeatTransferComm); + ht.heatEdges(); + } + + io.write(ht); + Tin = io.read(false); + Compute(Tin, Tout, dT, false); + writeOutput(); + } + MPI_Barrier(mpiHeatTransferComm); + + double timeEnd = MPI_Wtime(); + if (rank == 0) + std::cout << "Total runtime = " << timeEnd - timeStart << "s\n"; + + writer.Close(); + } + catch (std::invalid_argument &e) // command-line argument errors + { + std::cout << e.what() << std::endl; + printUsage(); + } + catch (std::ios_base::failure &e) // I/O failure (e.g. file not found) + { + std::cout << "I/O base exception caught\n"; + std::cout << e.what() << std::endl; + } + catch (std::exception &e) // All other exceptions + { + std::cout << "Exception caught\n"; + std::cout << e.what() << std::endl; + } + + MPI_Finalize(); + return 0; +} diff --git a/testing/examples/heatTransfer/CMakeLists.txt b/testing/examples/heatTransfer/CMakeLists.txt index 5e48178299..44b7d864af 100644 --- a/testing/examples/heatTransfer/CMakeLists.txt +++ b/testing/examples/heatTransfer/CMakeLists.txt @@ -7,6 +7,8 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/TestBPFileMx1.cmake) include(${CMAKE_CURRENT_SOURCE_DIR}/TestBPFileMxM.cmake) include(${CMAKE_CURRENT_SOURCE_DIR}/TestBPFileMxN.cmake) +include(${CMAKE_CURRENT_SOURCE_DIR}/TestInlineMxM.cmake) + if(ADIOS2_HAVE_ZFP) #include(${CMAKE_CURRENT_SOURCE_DIR}/TestBPFileMxM_zfp.cmake) #include(${CMAKE_CURRENT_SOURCE_DIR}/TestBPFileMxN_zfp.cmake) diff --git a/testing/examples/heatTransfer/TestInlineMxM.cmake b/testing/examples/heatTransfer/TestInlineMxM.cmake new file mode 100644 index 0000000000..b14102f196 --- /dev/null +++ b/testing/examples/heatTransfer/TestInlineMxM.cmake @@ -0,0 +1,31 @@ +#------------------------------------------------------------------------------# +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +#------------------------------------------------------------------------------# + +include(ADIOSFunctions) + +add_test(NAME HeatTransfer.Inline.MxM + COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_EXTRA_FLAGS} + ${MPIEXEC_NUMPROC_FLAG} 4 + $ + ${PROJECT_SOURCE_DIR}/examples/heatTransfer/heat_inline.xml + Read.bp 2 2 10 10 10 10 +) +set_tests_properties(HeatTransfer.Inline.MxM PROPERTIES PROCESSORS 8) + +add_test(NAME HeatTransfer.Inline.MxM.Dump + COMMAND ${CMAKE_COMMAND} + -DARG1=-d + -DINPUT_FILE=Read.bp + -DOUTPUT_FILE=Dump.txt + -P "${PROJECT_BINARY_DIR}/$/bpls.cmake" +) + +add_test(NAME HeatTransfer.Inline.MxM.Validate + COMMAND ${DIFF_COMMAND} -u -w + ${CMAKE_CURRENT_SOURCE_DIR}/HeatTransfer.Dump.txt + Dump.txt +) + +SetupTestPipeline(HeatTransfer.Inline.MxM ";Dump;Validate" TRUE)