Improve folder structure of hdf5 / pouts / .dat

Fully backwards compatible.
The base folder where all files are outputted is now 'output_path', defaulted to "".
This folder must exist.

HDF5's are written to 'output_path + hdf5_subpath'.
pout files are written to 'output_path + pout_subpath' and can be renamed to 'pout_prefix'.
.dat files (punctures, extraction files) are written to 'output_path + extraction_subpath'.
All subpaths are defaulted to "". They are created at runtime, so need not exist before.

Examples/BinaryBH/Main_BinaryBH.cpp

@@ -55,7 +55,8 @@ int runGRChombo(int argc, char *argv[])
         int puncture_tracker_min_level = sim_params.max_level - 1;
         bh_amr.m_puncture_tracker.initial_setup(
             {sim_params.bh1_params.center, sim_params.bh2_params.center},
-            sim_params.checkpoint_prefix, puncture_tracker_min_level);
+            sim_params.extraction_params.extraction_path,
+            puncture_tracker_min_level);
     }
 
     // The line below selects the problem that is simulated

Examples/BinaryBH/params_very_cheap.txt

@@ -1,8 +1,23 @@
 verbosity = 0
+
+#################################################
+# Filesystem parameters
+
+#output_path = "" # Main path for all files. Must exist!
 chk_prefix = BinaryBH_
 plot_prefix = BinaryBHPlot_
 #restart_file = BinaryBH_000360.3d.hdf5
 
+# subpaths - specify specific folders for hdf5, pout, extraction data
+# (these are created at runtime)
+hdf5_subpath = "hdf5"
+pout_subpath = "pout"
+extraction_subpath = "data"
+
+# change the name of output files
+#pout_prefix = "pout"
+#################################################
+
 # Set up grid spacings and regrid params
 # NB - the N values need to be multiples of block_factor
 
@@ -106,3 +121,4 @@ num_modes = 3
 modes = 2 0 # l m for spherical harmonics
         2 1
         2 2
+#integral_file_prefix = "Weyl4_mode_"

Source/AMRInterpolator/SurfaceExtraction.hpp

@@ -50,6 +50,9 @@ template <class SurfaceGeometry> class SurfaceExtraction
                                             //!< extraction for each surface
         bool write_extraction; //!< whether or not to write the extracted data
 
+        std::string extraction_path, integral_file_prefix,
+            extraction_file_prefix;
+
         int min_extraction_level()
         {
             return *(std::min_element(extraction_levels.begin(),

Source/AMRInterpolator/SurfaceExtraction.impl.hpp

@@ -25,6 +25,9 @@ SurfaceExtraction<SurfaceGeometry>::SurfaceExtraction(
       m_du(m_geom.du(m_params.num_points_u)),
       m_dv(m_geom.dv(m_params.num_points_v)), m_done_extraction(false)
 {
+    if (!GRParmParse::folder_exists(m_params.extraction_path))
+        GRParmParse::mkdir_recursive(m_params.extraction_path);
+
     // only interp points on rank 0
     if (procID() == 0)
     {
@@ -330,7 +333,8 @@ void SurfaceExtraction<SurfaceGeometry>::write_extraction(
     CH_assert(m_done_extraction);
     if (procID() == 0)
     {
-        SmallDataIO extraction_file(a_file_prefix, m_dt, m_time, m_restart_time,
+        SmallDataIO extraction_file(m_params.extraction_path + a_file_prefix,
+                                    m_dt, m_time, m_restart_time,
                                     SmallDataIO::NEW, m_first_step);
 
         for (int isurface = 0; isurface < m_params.num_surfaces; ++isurface)
@@ -415,8 +419,9 @@ void SurfaceExtraction<SurfaceGeometry>::write_integrals(
             CH_assert(vect.size() == m_params.num_surfaces);
         }
         // open file for writing
-        SmallDataIO integral_file(a_filename, m_dt, m_time, m_restart_time,
-                                  SmallDataIO::APPEND, m_first_step);
+        SmallDataIO integral_file(m_params.extraction_path + a_filename, m_dt,
+                                  m_time, m_restart_time, SmallDataIO::APPEND,
+                                  m_first_step);
 
         // remove any duplicate data if this is a restart
         integral_file.remove_duplicate_time_data();

Source/BlackHoles/PunctureTracker.cpp

@@ -13,9 +13,12 @@
 //! Set punctures post restart
 void PunctureTracker::initial_setup(
     const std::vector<std::array<double, CH_SPACEDIM>> &initial_puncture_coords,
-    const std::string &a_checkpoint_prefix, const int a_min_level)
+    const std::string &a_output_path, const int a_min_level)
 {
-    m_punctures_filename = a_checkpoint_prefix + "Punctures";
+    if (!GRParmParse::folder_exists(a_output_path))
+        GRParmParse::mkdir_recursive(a_output_path);
+
+    m_punctures_filename = a_output_path + "punctures";
 
     // first set the puncture data
     // m_num_punctures is only set later

Source/BlackHoles/PunctureTracker.hpp

@@ -36,7 +36,7 @@ class PunctureTracker
     //! if the puncture locations are required for Tagging Criteria
     void initial_setup(const std::vector<std::array<double, CH_SPACEDIM>>
                            &initial_puncture_coords,
-                       const std::string &a_checkpoint_prefix,
+                       const std::string &a_output_path,
                        const int a_min_level = 0);
 
     //! set puncture locations on start (or restart)

Source/GRChomboCore/ChomboParameters.hpp

@@ -16,7 +16,7 @@
 #include "GRParmParse.hpp"
 #include "UserVariables.hpp"
 #include "VariableType.hpp"
-#include "unistd.h"
+#include "unistd.h" // gives 'access'
 #include <algorithm>
 #include <string>
 
@@ -34,6 +34,9 @@ class ChomboParameters
 
     void read_params(GRParmParse &pp)
     {
+        // must be before any pout() in the code to setPoutBaseName
+        read_filesystem_params(pp);
+
         pp.load("verbosity", verbosity, 0);
         // Grid setup
         pp.load("max_spatial_derivative_order", max_spatial_derivative_order,
@@ -51,12 +54,6 @@ class ChomboParameters
         // L's, N's and center
         read_grid_params(pp);
 
-        // Misc
-        restart_from_checkpoint = pp.contains("restart_file");
-        if (restart_from_checkpoint)
-        {
-            pp.load("restart_file", restart_file);
-        }
         pp.load("ignore_checkpoint_name_mismatch",
                 ignore_checkpoint_name_mismatch, false);
 
@@ -92,9 +89,7 @@ class ChomboParameters
 
         // time stepping outputs and regrid data
         pp.load("checkpoint_interval", checkpoint_interval, 1);
-        pp.load("chk_prefix", checkpoint_prefix);
         pp.load("plot_interval", plot_interval, 0);
-        pp.load("plot_prefix", plot_prefix);
         pp.load("stop_time", stop_time, 1.0);
         pp.load("max_steps", max_steps, 1000000);
         pp.load("write_plot_ghosts", write_plot_ghosts, false);
@@ -126,6 +121,73 @@ class ChomboParameters
             just_check_params = true;
     }
 
+    void read_filesystem_params(GRParmParse &pp)
+    {
+        // In this function, cannot use default value - it may print a 'default
+        // message' to pout and a 'setPoutBaseName' must happen before
+
+        restart_from_checkpoint = pp.contains("restart_file");
+        if (restart_from_checkpoint)
+        {
+            pp.load("restart_file", restart_file);
+        }
+
+        pp.load("chk_prefix", checkpoint_prefix);
+        pp.load("plot_prefix", plot_prefix);
+
+        // Again, cannot use default value
+        if (pp.contains("pout_prefix"))
+            pp.load("pout_prefix", pout_prefix);
+        else
+            pout_prefix = "pout";
+
+        // folder structure ChomboParameters
+        std::string default_path = "";
+        if (pp.contains("output_path"))
+            pp.load("output_path", output_path);
+        else
+            output_path = default_path;
+
+        // user sets the 'folder', we transform it into the full path
+        if (pp.contains("pout_subpath"))
+            pp.load("pout_subpath", pout_path);
+        else
+            pout_path = default_path;
+
+#ifdef CH_USE_HDF5
+        // user sets the 'folder', we transform it into the full path
+        if (pp.contains("hdf5_subpath"))
+            pp.load("hdf5_subpath", hdf5_path);
+        else
+            hdf5_path = default_path;
+#endif
+
+        // add backslash to paths
+        if (output_path != "" && output_path[output_path.size() - 1] != '/')
+            output_path += "/";
+        if (pout_path != "" && pout_path[pout_path.size() - 1] != '/')
+            pout_path += "/";
+#ifdef CH_USE_HDF5
+        if (hdf5_path != "" && hdf5_path[hdf5_path.size() - 1] != '/')
+            hdf5_path += "/";
+#endif
+
+        if (output_path != "./" && output_path != "")
+        {
+#ifdef CH_USE_HDF5
+            hdf5_path = output_path + hdf5_path;
+#endif
+            pout_path = output_path + pout_path;
+        }
+
+        // change pout base name!
+        if (!GRParmParse::folder_exists(pout_path))
+            GRParmParse::mkdir_recursive(pout_path);
+        setPoutBaseName(pout_path + pout_prefix);
+
+        // only create hdf5 folder in setupAMRObject (when it becomes needed)
+    }
+
     void read_grid_params(GRParmParse &pp)
     {
         // Grid N
@@ -347,6 +409,14 @@ class ChomboParameters
                         plot_interval <= 0 || plot_prefix != checkpoint_prefix,
                         "should be different to checkpoint_prefix");
 
+        check_parameter("output_path", output_path,
+                        GRParmParse::folder_exists(output_path.c_str()),
+                        "should be a valid folder");
+        check_parameter("pout_path", pout_path,
+                        GRParmParse::folder_exists(pout_path.c_str()),
+                        "should be a valid folder");
+        // can't check hdf5 directory yet - only created after
+
         if (boundary_params.reflective_boundaries_exist)
         {
             for (int ivar = 0; ivar < NUM_VARS; ++ivar)
@@ -398,7 +468,11 @@ class ChomboParameters
     int checkpoint_interval, plot_interval; // Steps between outputs
     int max_grid_size, block_factor;        // max and min box sizes
     double fill_ratio; // determines how fussy the regridding is about tags
-    std::string checkpoint_prefix, plot_prefix; // naming of files
+    std::string checkpoint_prefix, plot_prefix, pout_prefix; // naming of files
+    std::string output_path, pout_path;                      // folder structure
+#ifdef CH_USE_HDF5
+    std::string hdf5_path;
+#endif
     bool write_plot_ghosts;
     int num_plot_vars;
     std::vector<std::pair<int, VariableType>>

Source/GRChomboCore/SetupFunctions.hpp

@@ -73,12 +73,12 @@ void mainSetup(int argc, char *argv[])
 
     if (rank == 0)
     {
-        pout() << " number_procs = " << number_procs << endl;
+        std::cout << " number_procs = " << number_procs << endl;
 #ifdef _OPENMP
-        pout() << " threads = " << omp_get_max_threads() << endl;
+        std::cout << " threads = " << omp_get_max_threads() << endl;
 #endif
-        pout() << " simd width (doubles) = " << simd_traits<double>::simd_len
-               << endl;
+        std::cout << " simd width (doubles) = " << simd_traits<double>::simd_len
+                  << endl;
     }
 
     const int required_argc = 2;
@@ -129,13 +129,16 @@ void setupAMRObject(GRAMR &gr_amr, AMRLevelFactory &a_factory)
     gr_amr.gridBufferSize(chombo_params.grid_buffer_size);
 
     // set checkpoint and plot intervals and prefixes
+#ifdef CH_USE_HDF5
     gr_amr.checkpointInterval(chombo_params.checkpoint_interval);
-    gr_amr.checkpointPrefix(chombo_params.checkpoint_prefix);
+    gr_amr.checkpointPrefix(chombo_params.hdf5_path +
+                            chombo_params.checkpoint_prefix);
     if (chombo_params.plot_interval != 0)
     {
         gr_amr.plotInterval(chombo_params.plot_interval);
-        gr_amr.plotPrefix(chombo_params.plot_prefix);
+        gr_amr.plotPrefix(chombo_params.hdf5_path + chombo_params.plot_prefix);
     }
+#endif
 
     // Number of coarse time steps from one regridding to the next
     gr_amr.regridIntervals(chombo_params.regrid_interval);
@@ -159,12 +162,18 @@ void setupAMRObject(GRAMR &gr_amr, AMRLevelFactory &a_factory)
     // Set up input files
     if (!chombo_params.restart_from_checkpoint)
     {
+#ifdef CH_USE_HDF5
+        if (!GRParmParse::folder_exists(chombo_params.hdf5_path))
+            GRParmParse::mkdir_recursive(chombo_params.hdf5_path);
+#endif
+
         gr_amr.setupForNewAMRRun();
     }
     else
     {
 #ifdef CH_USE_HDF5
-        HDF5Handle handle(chombo_params.restart_file, HDF5Handle::OPEN_RDONLY);
+        HDF5Handle handle(chombo_params.hdf5_path + chombo_params.restart_file,
+                          HDF5Handle::OPEN_RDONLY);
         // read from checkpoint file
         gr_amr.setupForRestart(handle);
         handle.close();

Source/GRChomboCore/SimulationParametersBase.hpp

@@ -122,6 +122,23 @@ class SimulationParametersBase : public ChomboParameters
         }
 
         pp.load("write_extraction", extraction_params.write_extraction, false);
+
+        std::string extraction_path;
+        pp.load("extraction_subpath", extraction_path, std::string(""));
+        if (extraction_path != "" &&
+            extraction_path[extraction_path.size() - 1] != '/')
+            extraction_path += "/";
+        if (output_path != "./" && output_path != "")
+            extraction_path = output_path + extraction_path;
+
+        extraction_params.extraction_path = extraction_path;
+
+        // default names to Weyl extraction
+        pp.load("extraction_file_prefix",
+                extraction_params.extraction_file_prefix,
+                std::string("Weyl4_extraction_"));
+        pp.load("integral_file_prefix", extraction_params.integral_file_prefix,
+                std::string("Weyl4_mode_"));
     }
 
     void check_params()