diff --git a/Examples/BinaryBH/Main_BinaryBH.cpp b/Examples/BinaryBH/Main_BinaryBH.cpp index 2e9b65478..ad6c83c18 100644 --- a/Examples/BinaryBH/Main_BinaryBH.cpp +++ b/Examples/BinaryBH/Main_BinaryBH.cpp @@ -55,7 +55,8 @@ int runGRChombo(int argc, char *argv[]) int puncture_tracker_min_level = sim_params.max_level - 1; bh_amr.m_puncture_tracker.initial_setup( {sim_params.bh1_params.center, sim_params.bh2_params.center}, - sim_params.checkpoint_prefix, puncture_tracker_min_level); + sim_params.extraction_params.extraction_path, + puncture_tracker_min_level); } // The line below selects the problem that is simulated diff --git a/Examples/BinaryBH/params_very_cheap.txt b/Examples/BinaryBH/params_very_cheap.txt index 4e62738d8..a52d3cf5a 100644 --- a/Examples/BinaryBH/params_very_cheap.txt +++ b/Examples/BinaryBH/params_very_cheap.txt @@ -1,8 +1,23 @@ verbosity = 0 + +################################################# +# Filesystem parameters + +#output_path = "" # Main path for all files. Must exist! chk_prefix = BinaryBH_ plot_prefix = BinaryBHPlot_ #restart_file = BinaryBH_000360.3d.hdf5 +# subpaths - specify specific folders for hdf5, pout, extraction data +# (these are created at runtime) +hdf5_subpath = "hdf5" +pout_subpath = "pout" +extraction_subpath = "data" + +# change the name of output files +#pout_prefix = "pout" +################################################# + # Set up grid spacings and regrid params # NB - the N values need to be multiples of block_factor @@ -106,3 +121,4 @@ num_modes = 3 modes = 2 0 # l m for spherical harmonics 2 1 2 2 +#integral_file_prefix = "Weyl4_mode_" \ No newline at end of file diff --git a/Source/AMRInterpolator/SurfaceExtraction.hpp b/Source/AMRInterpolator/SurfaceExtraction.hpp index c46dbf636..13c46459d 100644 --- a/Source/AMRInterpolator/SurfaceExtraction.hpp +++ b/Source/AMRInterpolator/SurfaceExtraction.hpp @@ -50,6 +50,9 @@ template class SurfaceExtraction //!< extraction for each surface bool write_extraction; //!< whether or not to write the extracted data + std::string extraction_path, integral_file_prefix, + extraction_file_prefix; + int min_extraction_level() { return *(std::min_element(extraction_levels.begin(), diff --git a/Source/AMRInterpolator/SurfaceExtraction.impl.hpp b/Source/AMRInterpolator/SurfaceExtraction.impl.hpp index 16eb7e951..5d768845d 100644 --- a/Source/AMRInterpolator/SurfaceExtraction.impl.hpp +++ b/Source/AMRInterpolator/SurfaceExtraction.impl.hpp @@ -25,6 +25,9 @@ SurfaceExtraction::SurfaceExtraction( m_du(m_geom.du(m_params.num_points_u)), m_dv(m_geom.dv(m_params.num_points_v)), m_done_extraction(false) { + if (!GRParmParse::folder_exists(m_params.extraction_path)) + GRParmParse::mkdir_recursive(m_params.extraction_path); + // only interp points on rank 0 if (procID() == 0) { @@ -330,7 +333,8 @@ void SurfaceExtraction::write_extraction( CH_assert(m_done_extraction); if (procID() == 0) { - SmallDataIO extraction_file(a_file_prefix, m_dt, m_time, m_restart_time, + SmallDataIO extraction_file(m_params.extraction_path + a_file_prefix, + m_dt, m_time, m_restart_time, SmallDataIO::NEW, m_first_step); for (int isurface = 0; isurface < m_params.num_surfaces; ++isurface) @@ -415,8 +419,9 @@ void SurfaceExtraction::write_integrals( CH_assert(vect.size() == m_params.num_surfaces); } // open file for writing - SmallDataIO integral_file(a_filename, m_dt, m_time, m_restart_time, - SmallDataIO::APPEND, m_first_step); + SmallDataIO integral_file(m_params.extraction_path + a_filename, m_dt, + m_time, m_restart_time, SmallDataIO::APPEND, + m_first_step); // remove any duplicate data if this is a restart integral_file.remove_duplicate_time_data(); diff --git a/Source/BlackHoles/PunctureTracker.cpp b/Source/BlackHoles/PunctureTracker.cpp index 769af2996..ec713d66f 100644 --- a/Source/BlackHoles/PunctureTracker.cpp +++ b/Source/BlackHoles/PunctureTracker.cpp @@ -13,9 +13,12 @@ //! Set punctures post restart void PunctureTracker::initial_setup( const std::vector> &initial_puncture_coords, - const std::string &a_checkpoint_prefix, const int a_min_level) + const std::string &a_output_path, const int a_min_level) { - m_punctures_filename = a_checkpoint_prefix + "Punctures"; + if (!GRParmParse::folder_exists(a_output_path)) + GRParmParse::mkdir_recursive(a_output_path); + + m_punctures_filename = a_output_path + "punctures"; // first set the puncture data // m_num_punctures is only set later diff --git a/Source/BlackHoles/PunctureTracker.hpp b/Source/BlackHoles/PunctureTracker.hpp index ca5d175c1..bb7f45984 100644 --- a/Source/BlackHoles/PunctureTracker.hpp +++ b/Source/BlackHoles/PunctureTracker.hpp @@ -36,7 +36,7 @@ class PunctureTracker //! if the puncture locations are required for Tagging Criteria void initial_setup(const std::vector> &initial_puncture_coords, - const std::string &a_checkpoint_prefix, + const std::string &a_output_path, const int a_min_level = 0); //! set puncture locations on start (or restart) diff --git a/Source/GRChomboCore/ChomboParameters.hpp b/Source/GRChomboCore/ChomboParameters.hpp index e4dc7ff73..7d7a6c242 100644 --- a/Source/GRChomboCore/ChomboParameters.hpp +++ b/Source/GRChomboCore/ChomboParameters.hpp @@ -16,7 +16,7 @@ #include "GRParmParse.hpp" #include "UserVariables.hpp" #include "VariableType.hpp" -#include "unistd.h" +#include "unistd.h" // gives 'access' #include #include @@ -34,6 +34,9 @@ class ChomboParameters void read_params(GRParmParse &pp) { + // must be before any pout() in the code to setPoutBaseName + read_filesystem_params(pp); + pp.load("verbosity", verbosity, 0); // Grid setup pp.load("max_spatial_derivative_order", max_spatial_derivative_order, @@ -51,12 +54,6 @@ class ChomboParameters // L's, N's and center read_grid_params(pp); - // Misc - restart_from_checkpoint = pp.contains("restart_file"); - if (restart_from_checkpoint) - { - pp.load("restart_file", restart_file); - } pp.load("ignore_checkpoint_name_mismatch", ignore_checkpoint_name_mismatch, false); @@ -92,9 +89,7 @@ class ChomboParameters // time stepping outputs and regrid data pp.load("checkpoint_interval", checkpoint_interval, 1); - pp.load("chk_prefix", checkpoint_prefix); pp.load("plot_interval", plot_interval, 0); - pp.load("plot_prefix", plot_prefix); pp.load("stop_time", stop_time, 1.0); pp.load("max_steps", max_steps, 1000000); pp.load("write_plot_ghosts", write_plot_ghosts, false); @@ -126,6 +121,73 @@ class ChomboParameters just_check_params = true; } + void read_filesystem_params(GRParmParse &pp) + { + // In this function, cannot use default value - it may print a 'default + // message' to pout and a 'setPoutBaseName' must happen before + + restart_from_checkpoint = pp.contains("restart_file"); + if (restart_from_checkpoint) + { + pp.load("restart_file", restart_file); + } + + pp.load("chk_prefix", checkpoint_prefix); + pp.load("plot_prefix", plot_prefix); + + // Again, cannot use default value + if (pp.contains("pout_prefix")) + pp.load("pout_prefix", pout_prefix); + else + pout_prefix = "pout"; + + // folder structure ChomboParameters + std::string default_path = ""; + if (pp.contains("output_path")) + pp.load("output_path", output_path); + else + output_path = default_path; + + // user sets the 'folder', we transform it into the full path + if (pp.contains("pout_subpath")) + pp.load("pout_subpath", pout_path); + else + pout_path = default_path; + +#ifdef CH_USE_HDF5 + // user sets the 'folder', we transform it into the full path + if (pp.contains("hdf5_subpath")) + pp.load("hdf5_subpath", hdf5_path); + else + hdf5_path = default_path; +#endif + + // add backslash to paths + if (output_path != "" && output_path[output_path.size() - 1] != '/') + output_path += "/"; + if (pout_path != "" && pout_path[pout_path.size() - 1] != '/') + pout_path += "/"; +#ifdef CH_USE_HDF5 + if (hdf5_path != "" && hdf5_path[hdf5_path.size() - 1] != '/') + hdf5_path += "/"; +#endif + + if (output_path != "./" && output_path != "") + { +#ifdef CH_USE_HDF5 + hdf5_path = output_path + hdf5_path; +#endif + pout_path = output_path + pout_path; + } + + // change pout base name! + if (!GRParmParse::folder_exists(pout_path)) + GRParmParse::mkdir_recursive(pout_path); + setPoutBaseName(pout_path + pout_prefix); + + // only create hdf5 folder in setupAMRObject (when it becomes needed) + } + void read_grid_params(GRParmParse &pp) { // Grid N @@ -347,6 +409,14 @@ class ChomboParameters plot_interval <= 0 || plot_prefix != checkpoint_prefix, "should be different to checkpoint_prefix"); + check_parameter("output_path", output_path, + GRParmParse::folder_exists(output_path.c_str()), + "should be a valid folder"); + check_parameter("pout_path", pout_path, + GRParmParse::folder_exists(pout_path.c_str()), + "should be a valid folder"); + // can't check hdf5 directory yet - only created after + if (boundary_params.reflective_boundaries_exist) { for (int ivar = 0; ivar < NUM_VARS; ++ivar) @@ -398,7 +468,11 @@ class ChomboParameters int checkpoint_interval, plot_interval; // Steps between outputs int max_grid_size, block_factor; // max and min box sizes double fill_ratio; // determines how fussy the regridding is about tags - std::string checkpoint_prefix, plot_prefix; // naming of files + std::string checkpoint_prefix, plot_prefix, pout_prefix; // naming of files + std::string output_path, pout_path; // folder structure +#ifdef CH_USE_HDF5 + std::string hdf5_path; +#endif bool write_plot_ghosts; int num_plot_vars; std::vector> diff --git a/Source/GRChomboCore/SetupFunctions.hpp b/Source/GRChomboCore/SetupFunctions.hpp index 71c2cce05..393c9ec1b 100644 --- a/Source/GRChomboCore/SetupFunctions.hpp +++ b/Source/GRChomboCore/SetupFunctions.hpp @@ -73,12 +73,12 @@ void mainSetup(int argc, char *argv[]) if (rank == 0) { - pout() << " number_procs = " << number_procs << endl; + std::cout << " number_procs = " << number_procs << endl; #ifdef _OPENMP - pout() << " threads = " << omp_get_max_threads() << endl; + std::cout << " threads = " << omp_get_max_threads() << endl; #endif - pout() << " simd width (doubles) = " << simd_traits::simd_len - << endl; + std::cout << " simd width (doubles) = " << simd_traits::simd_len + << endl; } const int required_argc = 2; @@ -129,13 +129,16 @@ void setupAMRObject(GRAMR &gr_amr, AMRLevelFactory &a_factory) gr_amr.gridBufferSize(chombo_params.grid_buffer_size); // set checkpoint and plot intervals and prefixes +#ifdef CH_USE_HDF5 gr_amr.checkpointInterval(chombo_params.checkpoint_interval); - gr_amr.checkpointPrefix(chombo_params.checkpoint_prefix); + gr_amr.checkpointPrefix(chombo_params.hdf5_path + + chombo_params.checkpoint_prefix); if (chombo_params.plot_interval != 0) { gr_amr.plotInterval(chombo_params.plot_interval); - gr_amr.plotPrefix(chombo_params.plot_prefix); + gr_amr.plotPrefix(chombo_params.hdf5_path + chombo_params.plot_prefix); } +#endif // Number of coarse time steps from one regridding to the next gr_amr.regridIntervals(chombo_params.regrid_interval); @@ -159,12 +162,18 @@ void setupAMRObject(GRAMR &gr_amr, AMRLevelFactory &a_factory) // Set up input files if (!chombo_params.restart_from_checkpoint) { +#ifdef CH_USE_HDF5 + if (!GRParmParse::folder_exists(chombo_params.hdf5_path)) + GRParmParse::mkdir_recursive(chombo_params.hdf5_path); +#endif + gr_amr.setupForNewAMRRun(); } else { #ifdef CH_USE_HDF5 - HDF5Handle handle(chombo_params.restart_file, HDF5Handle::OPEN_RDONLY); + HDF5Handle handle(chombo_params.hdf5_path + chombo_params.restart_file, + HDF5Handle::OPEN_RDONLY); // read from checkpoint file gr_amr.setupForRestart(handle); handle.close(); diff --git a/Source/GRChomboCore/SimulationParametersBase.hpp b/Source/GRChomboCore/SimulationParametersBase.hpp index 8be50660d..3f377d27d 100644 --- a/Source/GRChomboCore/SimulationParametersBase.hpp +++ b/Source/GRChomboCore/SimulationParametersBase.hpp @@ -122,6 +122,23 @@ class SimulationParametersBase : public ChomboParameters } pp.load("write_extraction", extraction_params.write_extraction, false); + + std::string extraction_path; + pp.load("extraction_subpath", extraction_path, std::string("")); + if (extraction_path != "" && + extraction_path[extraction_path.size() - 1] != '/') + extraction_path += "/"; + if (output_path != "./" && output_path != "") + extraction_path = output_path + extraction_path; + + extraction_params.extraction_path = extraction_path; + + // default names to Weyl extraction + pp.load("extraction_file_prefix", + extraction_params.extraction_file_prefix, + std::string("Weyl4_extraction_")); + pp.load("integral_file_prefix", extraction_params.integral_file_prefix, + std::string("Weyl4_mode_")); } void check_params() diff --git a/Source/utils/GRParmParse.hpp b/Source/utils/GRParmParse.hpp index 66d22fa43..c3cc78313 100644 --- a/Source/utils/GRParmParse.hpp +++ b/Source/utils/GRParmParse.hpp @@ -12,8 +12,10 @@ // Other includes #include "ArrayTools.hpp" +#include "unistd.h" // gives 'mkdir' #include #include +#include // gives 'stat' and 'S_ISDIR' #include // Chombo namespace @@ -127,6 +129,65 @@ class GRParmParse : public ParmParse std::vector(num_comp, default_value)); } + static bool folder_exists(const std::string &path) + { + struct stat sb; + return (path == "") || + (stat(path.c_str(), &sb) == 0 && S_ISDIR(sb.st_mode)); + } + + static bool mkdir_recursive(const std::string &path, + MPI_Comm comm = Chombo_MPI::comm) + { +#ifdef CH_MPI + // all processes should get here at the same time + /* + e.g. if doing: + if (!folder_exists(path)) + mkdir_recursive(path); + We want to make sure the folder is not created until all + ranks have passed the 'folder_exists' + */ + MPI_Barrier(comm); +#endif + + bool success = true; + if (procID() == 0) + { + // Windows compatible with opposite backslash + static const std::string delimeters = "/\\"; + std::size_t found = path.find_first_of(delimeters); + + // NB: this would be very beautiful recursively, but let's not do it + // because the function involves MPI_Barrier's + while (success && found != std::string::npos) + { + std::string subpath; + subpath = path.substr(0, found); + + if (subpath != "." && subpath != "") + { + // success if created or if not created because it already + // exists + success &= + (mkdir(subpath.c_str(), 0700) == 0 || errno == EEXIST); + } + found = path.find_first_of(delimeters, found + 1); + } + // if path doesn't finish with "/", one more to do + if (found != path.size() - 1) + success &= (mkdir(path.c_str(), 0700) == 0 || errno == EEXIST); + } + +#ifdef CH_MPI + // all processes should wait to make sure folder structure is well set + // for everyone + MPI_Barrier(comm); +#endif + + return success; + } + protected: template , std::vector>> @@ -79,7 +77,7 @@ class WeylExtraction : public SphericalExtraction for (int imode = 0; imode < m_num_modes; ++imode) { const auto &mode = m_modes[imode]; - std::string integrals_filename = "Weyl_integral_" + + std::string integrals_filename = m_params.integral_file_prefix + std::to_string(mode.first) + std::to_string(mode.second); std::vector> integrals_for_writing = {