From 4bd2f76ac196a0b35340d27cff0b92658fe429a5 Mon Sep 17 00:00:00 2001 From: Sigfried Haering Date: Wed, 26 Jun 2024 13:35:11 -0700 Subject: [PATCH] support for specifying a restart directory --- src/M2ulPhyS2Boltzmann.cpp | 16 ++++++++-------- src/chemistry.cpp | 2 +- src/chemistry.hpp | 2 +- src/gpu_constructor.cpp | 4 ++-- src/gpu_constructor.hpp | 4 ++-- src/io.cpp | 16 ++++++++++++---- src/io.hpp | 5 +++-- src/loMachIO.cpp | 3 ++- src/mesh_base.cpp | 14 +++++++++++--- src/reaction.cpp | 4 ++-- src/reaction.hpp | 4 ++-- src/tps.cpp | 3 +-- src/tps2Boltzmann.cpp | 2 -- src/utils.cpp | 7 +++++++ src/utils.hpp | 2 ++ 15 files changed, 56 insertions(+), 32 deletions(-) diff --git a/src/M2ulPhyS2Boltzmann.cpp b/src/M2ulPhyS2Boltzmann.cpp index a8aba27b6..8bdf6a5cc 100644 --- a/src/M2ulPhyS2Boltzmann.cpp +++ b/src/M2ulPhyS2Boltzmann.cpp @@ -90,12 +90,12 @@ void M2ulPhyS::fetch(TPS::Tps2Boltzmann &interface) { mfem::ParFiniteElementSpace *reaction_rates_fes(&(interface.NativeFes(TPS::Tps2Boltzmann::Index::ReactionRates))); externalReactionRates.reset(new mfem::ParGridFunction(reaction_rates_fes)); interface.interpolateToNativeFES(*externalReactionRates, TPS::Tps2Boltzmann::Index::ReactionRates); - #if defined(_CUDA_) || defined(_HIP_) - const double * data(externalReactionRates->Read() ); - int size(externalReactionRates->FESpace()->GetNDofs() ); - assert(externalReactionRates->FESpace()->GetOrdering() == mfem::Ordering::byNODES); - gpu::deviceSetChemistryReactionData<<<1, 1>>>(data, size, chemistry_); - #else - chemistry_->setGridFunctionRates(*externalReactionRates); - #endif +#if defined(_CUDA_) || defined(_HIP_) + const double *data(externalReactionRates->Read()); + int size(externalReactionRates->FESpace()->GetNDofs()); + assert(externalReactionRates->FESpace()->GetOrdering() == mfem::Ordering::byNODES); + gpu::deviceSetChemistryReactionData<<<1, 1>>>(data, size, chemistry_); +#else + chemistry_->setGridFunctionRates(*externalReactionRates); +#endif } diff --git a/src/chemistry.cpp b/src/chemistry.cpp index fed7512d3..5960b463e 100644 --- a/src/chemistry.cpp +++ b/src/chemistry.cpp @@ -109,7 +109,7 @@ MFEM_HOST_DEVICE Chemistry::~Chemistry() { } } -MFEM_HOST_DEVICE void Chemistry::setRates(const double * data, int size) { +MFEM_HOST_DEVICE void Chemistry::setRates(const double *data, int size) { for (int r = 0; r < numReactions_; r++) { if (reactions_[r]->reactionModel == GRIDFUNCTION_RXN) { GridFunctionReaction *rx = (GridFunctionReaction *)(reactions_[r]); diff --git a/src/chemistry.hpp b/src/chemistry.hpp index 906d249af..79af5a885 100644 --- a/src/chemistry.hpp +++ b/src/chemistry.hpp @@ -97,7 +97,7 @@ class Chemistry { // Set the grid function rates for GRIDFUNCTION_RXN reaction types void setGridFunctionRates(mfem::GridFunction &f); - MFEM_HOST_DEVICE void setRates(const double * data, int size); + MFEM_HOST_DEVICE void setRates(const double *data, int size); // return Vector of reaction rate coefficients, with the size of numReaction_. // WARNING(marc) I have removed "virtual" qualifier here assuming these functions will not diff --git a/src/gpu_constructor.cpp b/src/gpu_constructor.cpp index bf58b88a4..62c5d6f1b 100644 --- a/src/gpu_constructor.cpp +++ b/src/gpu_constructor.cpp @@ -124,11 +124,11 @@ __global__ void freeDeviceRadiation(Radiation *radiation) { //--------------------------------------------------- // And finally device setters //--------------------------------------------------- -__global__ void deviceSetGridFunctionReactionData(const double * data, int size, GridFunctionReaction * reaction) { +__global__ void deviceSetGridFunctionReactionData(const double *data, int size, GridFunctionReaction *reaction) { reaction->setData(data, size); } -__global__ void deviceSetChemistryReactionData(const double * data, int size, Chemistry * chem) { +__global__ void deviceSetChemistryReactionData(const double *data, int size, Chemistry *chem) { chem->setRates(data, size); } diff --git a/src/gpu_constructor.hpp b/src/gpu_constructor.hpp index 64e98d34d..18d516dd2 100644 --- a/src/gpu_constructor.hpp +++ b/src/gpu_constructor.hpp @@ -161,8 +161,8 @@ __global__ void freeDeviceChemistry(Chemistry *chem); __global__ void freeDeviceRadiation(Radiation *radiation); //! Set the data to a GridFunctionReaction -__global__ void deviceSetGridFunctionReactionData(const double * data, int size, GridFunctionReaction * reaction); -__global__ void deviceSetChemistryReactionData(const double * data, int size, Chemistry * chem); +__global__ void deviceSetGridFunctionReactionData(const double *data, int size, GridFunctionReaction *reaction); +__global__ void deviceSetChemistryReactionData(const double *data, int size, Chemistry *chem); #endif // cuda or hip } // namespace gpu diff --git a/src/io.cpp b/src/io.cpp index 4354c227b..638c93a47 100644 --- a/src/io.cpp +++ b/src/io.cpp @@ -35,6 +35,8 @@ #include "io.hpp" #include +#include +#include #include "M2ulPhyS.hpp" #include "utils.hpp" @@ -268,7 +270,8 @@ void M2ulPhyS::restart_files_hdf5(string mode, string inputFileName) { return; } -void partitioning_file_hdf5(std::string mode, MPI_Groups *groupsMPI, int nelemGlobal, Array &partitioning) { +void partitioning_file_hdf5(std::string mode, MPI_Groups *groupsMPI, int nelemGlobal, Array &partitioning, + std::string pathName) { MPI_Comm TPSCommWorld = groupsMPI->getTPSCommWorld(); const bool rank0 = groupsMPI->isWorldRoot(); const int nprocs = groupsMPI->getTPSWorldSize(); @@ -281,8 +284,10 @@ void partitioning_file_hdf5(std::string mode, MPI_Groups *groupsMPI, int nelemGl // hid_t file, dataspace, data_soln; hid_t file = -1, dataspace; herr_t status; - std::string fileName("partition"); - fileName += "." + std::to_string(nprocs) + "p.h5"; + std::string fileNameEnd("partition"); + fileNameEnd += "." + std::to_string(nprocs) + "p.h5"; + std::string fileName; + fileName = pathName + fileNameEnd; assert((mode == "read") || (mode == "write")); @@ -402,7 +407,7 @@ void read_variable_data_hdf5(hid_t file, string varName, size_t index, double *d H5Dclose(data_soln); } -IOOptions::IOOptions() : output_dir_("output"), restart_mode_("standard") {} +IOOptions::IOOptions() : output_dir_("output"), restart_dir_("./"), restart_mode_("standard") {} void IOOptions::read(TPS::Tps *tps, std::string prefix) { std::string basename; @@ -417,8 +422,11 @@ void IOOptions::read(TPS::Tps *tps, std::string prefix) { tps->getInput((basename + "/exitCheckFreq").c_str(), exit_check_frequency_, 500); assert(exit_check_frequency_ > 0); + tps->getInput((basename + "/restartBase").c_str(), restart_dir_, std::string("./")); tps->getInput((basename + "/restartMode").c_str(), restart_mode_, std::string("standard")); setRestartFlags(); + + int status = mkdir(restart_dir_.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); } void IOOptions::setRestartFlags() { diff --git a/src/io.hpp b/src/io.hpp index 70c4d449b..4685132ba 100644 --- a/src/io.hpp +++ b/src/io.hpp @@ -66,8 +66,8 @@ class IOOptions { int exit_check_frequency_ = 500; + std::string restart_dir_; std::string restart_mode_; - bool restart_variable_order_ = false; bool restart_serial_read_ = false; bool restart_serial_write_ = false; @@ -362,5 +362,6 @@ void write_variable_data_hdf5(hid_t group, std::string varName, hid_t dataspace, * @todo Refactor this function to make it more generic and fit better * into the IODataOrganizer paradigm. */ -void partitioning_file_hdf5(std::string mode, MPI_Groups *groupsMPI, int nelemGlobal, mfem::Array &partitioning); +void partitioning_file_hdf5(std::string mode, MPI_Groups *groupsMPI, int nelemGlobal, mfem::Array &partitioning, + std::string fileName = "./"); #endif // IO_HPP_ diff --git a/src/loMachIO.cpp b/src/loMachIO.cpp index e2c7997af..4e1701186 100644 --- a/src/loMachIO.cpp +++ b/src/loMachIO.cpp @@ -181,7 +181,8 @@ void LoMachSolver::restart_files_hdf5(string mode, string inputFileName) { } serialName = inputFileName; } else { - serialName = "restart_"; + serialName = loMach_opts_.io_opts_.restart_dir_; + serialName.append("/restart_"); serialName.append(loMach_opts_.io_opts_.output_dir_); serialName.append(".sol.h5"); } diff --git a/src/mesh_base.cpp b/src/mesh_base.cpp index 09e0477bd..122e45a85 100644 --- a/src/mesh_base.cpp +++ b/src/mesh_base.cpp @@ -174,13 +174,17 @@ void MeshBase::initializeMesh() { // nelemGlobal_ = mesh->GetNE(); if (rank0_) grvy_printf(ginfo, "Total # of mesh elements = %i\n", nelemGlobal_); + string restartDir; + restartDir = loMach_opts_->io_opts_.restart_dir_; + restartDir += "/"; + if (nprocs_ > 1) { if (loMach_opts_->io_opts_.restart_serial_read_) { assert(serial_mesh_->Conforming()); partitioning_ = Array(serial_mesh_->GeneratePartitioning(nprocs_, defaultPartMethod), nelemGlobal_); - partitioning_file_hdf5("write", groupsMPI, nelemGlobal_, partitioning_); + partitioning_file_hdf5("write", groupsMPI, nelemGlobal_, partitioning_, restartDir); } else { - partitioning_file_hdf5("read", groupsMPI, nelemGlobal_, partitioning_); + partitioning_file_hdf5("read", groupsMPI, nelemGlobal_, partitioning_, restartDir); } } @@ -203,12 +207,16 @@ void MeshBase::initializeMesh() { serial_mesh_->UniformRefinement(); } + string restartDir; + restartDir = loMach_opts_->io_opts_.restart_dir_; + restartDir += "/"; + // generate partitioning file (we assume conforming meshes) nelemGlobal_ = serial_mesh_->GetNE(); if (nprocs_ > 1) { assert(serial_mesh_->Conforming()); partitioning_ = Array(serial_mesh_->GeneratePartitioning(nprocs_, defaultPartMethod), nelemGlobal_); - if (rank0_) partitioning_file_hdf5("write", groupsMPI, nelemGlobal_, partitioning_); + if (rank0_) partitioning_file_hdf5("write", groupsMPI, nelemGlobal_, partitioning_, restartDir); } } diff --git a/src/reaction.cpp b/src/reaction.cpp index 18efb6072..54a02e478 100644 --- a/src/reaction.cpp +++ b/src/reaction.cpp @@ -84,12 +84,12 @@ MFEM_HOST_DEVICE GridFunctionReaction::GridFunctionReaction(int comp) MFEM_HOST_DEVICE GridFunctionReaction::~GridFunctionReaction() {} -MFEM_HOST_DEVICE void GridFunctionReaction::setData(const double * data, int size) { +MFEM_HOST_DEVICE void GridFunctionReaction::setData(const double *data, int size) { data_ = data + comp_ * size_; size_ = size; } -void GridFunctionReaction::setGridFunction(const mfem::GridFunction & f) { +void GridFunctionReaction::setGridFunction(const mfem::GridFunction &f) { size_ = f.FESpace()->GetNDofs(); assert(comp_ < f.FESpace()->GetVDim()); assert(f.FESpace()->GetOrdering() == mfem::Ordering::byNODES); diff --git a/src/reaction.hpp b/src/reaction.hpp index c563e0dd8..078c26faf 100644 --- a/src/reaction.hpp +++ b/src/reaction.hpp @@ -129,9 +129,9 @@ class GridFunctionReaction : public Reaction { MFEM_HOST_DEVICE virtual ~GridFunctionReaction(); - void setGridFunction(const mfem::GridFunction & f); + void setGridFunction(const mfem::GridFunction &f); - MFEM_HOST_DEVICE void setData(const double * data, int size); + MFEM_HOST_DEVICE void setData(const double *data, int size); MFEM_HOST_DEVICE virtual double computeRateCoefficient([[maybe_unused]] const double &T_h, [[maybe_unused]] const double &T_e, const int &dofindex, diff --git a/src/tps.cpp b/src/tps.cpp index b82b7caf3..7e0cd6c39 100644 --- a/src/tps.cpp +++ b/src/tps.cpp @@ -140,8 +140,7 @@ void Tps::parseCommandLineArgs(int argc, char *argv[]) { args.AddOption(&visualMode, "-visual", "--visualization", "", "--no-visualization", "Launch post-process visualization."); gpu_aware_mpi_ = false; - args.AddOption(&gpu_aware_mpi_, "-ga", "--gpu-aware-mpi", "", "--no-gpu-aware-mpi", - "Set GPU-aware MPI."); + args.AddOption(&gpu_aware_mpi_, "-ga", "--gpu-aware-mpi", "", "--no-gpu-aware-mpi", "Set GPU-aware MPI."); args.Parse(); if (!args.Good()) { diff --git a/src/tps2Boltzmann.cpp b/src/tps2Boltzmann.cpp index 72f9b0678..689fded06 100644 --- a/src/tps2Boltzmann.cpp +++ b/src/tps2Boltzmann.cpp @@ -94,12 +94,10 @@ Tps2Boltzmann::Tps2Boltzmann(Tps *tps) save_to_paraview_dc = tps->getInput("boltzmannInterface/save_to_paraview", false); - offsets.SetSize(NIndexes + 1); ncomps.SetSize(NIndexes + 1); } - int Tps2Boltzmann::_countBTEReactions() { int total_reactions(0); int bte_reactions(0); diff --git a/src/utils.cpp b/src/utils.cpp index 6af93458d..8dc7ab759 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -1156,6 +1156,13 @@ void Orthogonalize(Vector &v, MPI_Comm comm) { v -= global_sum / static_cast(global_size); } +bool copyFile(const char *SRC, const char *DEST) { + std::ifstream src(SRC, std::ios::binary); + std::ofstream dest(DEST, std::ios::binary); + dest << src.rdbuf(); + return src && dest; +} + namespace mfem { GradientVectorGridFunctionCoefficient::GradientVectorGridFunctionCoefficient(const GridFunction *gf) : MatrixCoefficient((gf) ? gf->VectorDim() : 0) { diff --git a/src/utils.hpp b/src/utils.hpp index 4fb5ef324..2cd4f0d86 100644 --- a/src/utils.hpp +++ b/src/utils.hpp @@ -174,6 +174,8 @@ void scalarGrad3D(ParGridFunction &u, ParGridFunction &gu); void vectorGrad3DV(FiniteElementSpace *fes, Vector u, Vector *gu, Vector *gv, Vector *gw); void scalarGrad3DV(FiniteElementSpace *fes, FiniteElementSpace *vfes, Vector u, Vector *gu); +bool copyFile(const char *SRC, const char *DEST); + /// Eliminate essential BCs in an Operator and apply to RHS. /// rename this to something sensible "ApplyEssentialBC" or something void EliminateRHS(Operator &A, ConstrainedOperator &constrainedA, const Array &ess_tdof_list, Vector &x, Vector &b,