Improve folder structure of hdf5 / pouts / .dat

Examples/BinaryBH/Main_BinaryBH.cpp

@@ -55,7 +55,7 @@ int runGRChombo(int argc, char *argv[])
         int puncture_tracker_min_level = sim_params.max_level - 1;
         bh_amr.m_puncture_tracker.initial_setup(
             {sim_params.bh1_params.center, sim_params.bh2_params.center},
-            sim_params.checkpoint_prefix, puncture_tracker_min_level);
+            "punctures", sim_params.data_path, puncture_tracker_min_level);
     }
 
     // The line below selects the problem that is simulated

Examples/BinaryBH/params_very_cheap.txt

@@ -1,8 +1,24 @@
 verbosity = 0
+
+#################################################
+# Filesystem parameters
+
+#output_path = "" # Main path for all files. Must exist!
 chk_prefix = BinaryBH_
 plot_prefix = BinaryBHPlot_
 #restart_file = BinaryBH_000360.3d.hdf5
 
+# subpaths - specific directories for hdf5, pout, extraction data
+# (these are created at runtime)
+hdf5_subpath = "hdf5"
+pout_subpath = "pout"
+data_subpath = "data"
+
+# change the name of output files
+#pout_prefix = "pout"
+print_progress_only_to_rank_0 = 1
+#################################################
+
 # Set up grid spacings and regrid params
 # NB - the N values need to be multiples of block_factor
 
@@ -106,3 +122,6 @@ num_modes = 3
 modes = 2 0 # l m for spherical harmonics
         2 1
         2 2
+write_extraction = 1
+#integral_file_prefix = "Weyl4_mode_"
+#extraction_subpath = "data/extraction" # directory for 'write_extraction = 1'

Examples/KerrBH/params.txt

@@ -4,6 +4,7 @@ verbosity = 0
 chk_prefix = KerrBH_
 plot_prefix = KerrBHp_
 #restart_file = KerrBH_000060.3d.hdf5
+print_progress_only_to_rank_0 = 1
 
 # 'N' is the number of subdivisions in each direction of a cubic box
 # 'L' is the length of the longest side of the box, dx_coarsest = L/N
@@ -12,7 +13,7 @@ plot_prefix = KerrBHp_
 # NB - if you have a non-cubic grid, you can specify 'N1' or 'N1_full',
 # 'N2' or 'N2_full' and 'N3' or 'N3_full' ( then dx_coarsest = L/N(max) )
 # NB - the N values need to be multiples of the block_factor
-N_full = 64
+N_full = 128
 L_full = 128
 
 # Spatial derivative order (only affects CCZ4 RHS)
@@ -36,7 +37,7 @@ max_level = 6 # There are (max_level+1) grids, so min is zero
 # Generally you do not need to regrid frequently on every level
 # Level Regridding:   0   1   2   3   4   5
 regrid_interval   =  50  25   5   5   5   5
-regrid_thresholds = 0.3 0.3 0.3 0.3 0.3 0.3
+regrid_thresholds = 0.1 0.1 0.1 0.1 0.1 0.1
 
 # Max box sizes
 max_grid_size = 16

Examples/KerrBH/params_cheap.txt

@@ -4,6 +4,7 @@ verbosity = 0
 chk_prefix = KerrBH_
 plot_prefix = KerrBHp_
 #restart_file = KerrBH_000060.3d.hdf5
+print_progress_only_to_rank_0 = 1
 
 # 'N' is the number of subdivisions in each direction of a cubic box
 # 'L' is the length of the longest side of the box, dx_coarsest = L/N
@@ -81,7 +82,7 @@ plot_interval = 0
 num_plot_vars = 0
 dt_multiplier = 0.25
 stop_time = 10.0
-max_steps = 2
+max_steps = 4
 
 #Lapse evolution
 lapse_power = 1.0

Source/AMRInterpolator/SurfaceExtraction.hpp

@@ -13,6 +13,7 @@
 // Other inclues
 #include "AMRInterpolator.hpp"
 #include "DimensionDefinitions.hpp"
+#include "FilesystemTools.hpp"
 #include "IntegrationMethod.hpp"
 #include "InterpolationQuery.hpp"
 #include "Lagrange.hpp"
@@ -50,6 +51,9 @@ template <class SurfaceGeometry> class SurfaceExtraction
                                             //!< extraction for each surface
         bool write_extraction; //!< whether or not to write the extracted data
 
+        std::string data_path, integral_file_prefix;
+        std::string extraction_path, extraction_file_prefix;
+
         int min_extraction_level()
         {
             return *(std::min_element(extraction_levels.begin(),

Source/AMRInterpolator/SurfaceExtraction.impl.hpp

@@ -25,6 +25,13 @@ SurfaceExtraction<SurfaceGeometry>::SurfaceExtraction(
       m_du(m_geom.du(m_params.num_points_u)),
       m_dv(m_geom.dv(m_params.num_points_v)), m_done_extraction(false)
 {
+    if (!FilesystemTools::directory_exists(m_params.data_path))
+        FilesystemTools::mkdir_recursive(m_params.data_path);
+
+    if (m_params.write_extraction &&
+        !FilesystemTools::directory_exists(m_params.extraction_path))
+        FilesystemTools::mkdir_recursive(m_params.extraction_path);
+
     // only interp points on rank 0
     if (procID() == 0)
     {
@@ -330,7 +337,8 @@ void SurfaceExtraction<SurfaceGeometry>::write_extraction(
     CH_assert(m_done_extraction);
     if (procID() == 0)
     {
-        SmallDataIO extraction_file(a_file_prefix, m_dt, m_time, m_restart_time,
+        SmallDataIO extraction_file(m_params.extraction_path + a_file_prefix,
+                                    m_dt, m_time, m_restart_time,
                                     SmallDataIO::NEW, m_first_step);
 
         for (int isurface = 0; isurface < m_params.num_surfaces; ++isurface)
@@ -415,8 +423,9 @@ void SurfaceExtraction<SurfaceGeometry>::write_integrals(
             CH_assert(vect.size() == m_params.num_surfaces);
         }
         // open file for writing
-        SmallDataIO integral_file(a_filename, m_dt, m_time, m_restart_time,
-                                  SmallDataIO::APPEND, m_first_step);
+        SmallDataIO integral_file(m_params.data_path + a_filename, m_dt, m_time,
+                                  m_restart_time, SmallDataIO::APPEND,
+                                  m_first_step);
 
         // remove any duplicate data if this is a restart
         integral_file.remove_duplicate_time_data();

Source/BlackHoles/PunctureTracker.cpp

@@ -13,9 +13,13 @@
 //! Set punctures post restart
 void PunctureTracker::initial_setup(
     const std::vector<std::array<double, CH_SPACEDIM>> &initial_puncture_coords,
-    const std::string &a_checkpoint_prefix, const int a_min_level)
+    const std::string &a_filename, const std::string &a_output_path,
+    const int a_min_level)
 {
-    m_punctures_filename = a_checkpoint_prefix + "Punctures";
+    if (!FilesystemTools::directory_exists(a_output_path))
+        FilesystemTools::mkdir_recursive(a_output_path);
+
+    m_punctures_filename = a_output_path + a_filename;
 
     // first set the puncture data
     // m_num_punctures is only set later

Source/BlackHoles/PunctureTracker.hpp

@@ -36,7 +36,8 @@ class PunctureTracker
     //! if the puncture locations are required for Tagging Criteria
     void initial_setup(const std::vector<std::array<double, CH_SPACEDIM>>
                            &initial_puncture_coords,
-                       const std::string &a_checkpoint_prefix,
+                       const std::string &a_filename = "punctures",
+                       const std::string &a_output_path = "",
                        const int a_min_level = 0);
 
     //! set puncture locations on start (or restart)

Source/GRChomboCore/ChomboParameters.hpp

@@ -13,10 +13,11 @@
 // General includes
 #include "ArrayTools.hpp"
 #include "BoundaryConditions.hpp"
+#include "FilesystemTools.hpp"
 #include "GRParmParse.hpp"
 #include "UserVariables.hpp"
 #include "VariableType.hpp"
-#include "unistd.h"
+#include "unistd.h" // gives 'access'
 #include <algorithm>
 #include <string>
 
@@ -34,6 +35,9 @@ class ChomboParameters
 
     void read_params(GRParmParse &pp)
     {
+        // must be before any pout() in the code to setPoutBaseName
+        read_filesystem_params(pp);
+
         pp.load("verbosity", verbosity, 0);
         // Grid setup
         pp.load("max_spatial_derivative_order", max_spatial_derivative_order,
@@ -51,12 +55,6 @@ class ChomboParameters
         // L's, N's and center
         read_grid_params(pp);
 
-        // Misc
-        restart_from_checkpoint = pp.contains("restart_file");
-        if (restart_from_checkpoint)
-        {
-            pp.load("restart_file", restart_file);
-        }
         pp.load("ignore_checkpoint_name_mismatch",
                 ignore_checkpoint_name_mismatch, false);
 
@@ -92,9 +90,7 @@ class ChomboParameters
 
         // time stepping outputs and regrid data
         pp.load("checkpoint_interval", checkpoint_interval, 1);
-        pp.load("chk_prefix", checkpoint_prefix);
         pp.load("plot_interval", plot_interval, 0);
-        pp.load("plot_prefix", plot_prefix);
         pp.load("stop_time", stop_time, 1.0);
         pp.load("max_steps", max_steps, 1000000);
         pp.load("write_plot_ghosts", write_plot_ghosts, false);
@@ -124,6 +120,85 @@ class ChomboParameters
 
         if (pp.contains("check_params"))
             just_check_params = true;
+
+        pp.load("print_progress_only_to_rank_0", print_progress_only_to_rank_0,
+                false);
+    }
+
+    void read_filesystem_params(GRParmParse &pp)
+    {
+        // In this function, cannot use default value - it may print a 'default
+        // message' to pout and a 'setPoutBaseName' must happen before
+
+        restart_from_checkpoint = pp.contains("restart_file");
+        if (restart_from_checkpoint)
+        {
+            pp.load("restart_file", restart_file);
+        }
+
+        pp.load("chk_prefix", checkpoint_prefix);
+        pp.load("plot_prefix", plot_prefix);
+
+#ifdef CH_MPI
+        // Again, cannot use default value
+        if (pp.contains("pout_prefix"))
+            pp.load("pout_prefix", pout_prefix);
+        else
+            pout_prefix = "pout";
+#endif
+
+        std::string default_path = "";
+        if (pp.contains("output_path"))
+            pp.load("output_path", output_path);
+        else
+            output_path = default_path;
+
+#ifdef CH_MPI
+        // user sets the 'subpath', we prepend 'output_path'
+        if (pp.contains("pout_subpath"))
+            pp.load("pout_subpath", pout_path);
+        else
+            pout_path = default_path;
+#endif
+
+#ifdef CH_USE_HDF5
+        // user sets the 'subpath', we prepend 'output_path'
+        if (pp.contains("hdf5_subpath"))
+            pp.load("hdf5_subpath", hdf5_path);
+        else
+            hdf5_path = default_path;
+#endif
+
+        // add backslash to paths
+        if (output_path != "" && output_path[output_path.size() - 1] != '/')
+            output_path += "/";
+#ifdef CH_MPI
+        if (pout_path != "" && pout_path[pout_path.size() - 1] != '/')
+            pout_path += "/";
+#endif
+#ifdef CH_USE_HDF5
+        if (hdf5_path != "" && hdf5_path[hdf5_path.size() - 1] != '/')
+            hdf5_path += "/";
+#endif
+
+        if (output_path != "./" && output_path != "")
+        {
+#ifdef CH_MPI
+            pout_path = output_path + pout_path;
+#endif
+#ifdef CH_USE_HDF5
+            hdf5_path = output_path + hdf5_path;
+#endif
+        }
+
+#ifdef CH_MPI
+        // change pout base name!
+        if (!FilesystemTools::directory_exists(pout_path))
+            FilesystemTools::mkdir_recursive(pout_path);
+        setPoutBaseName(pout_path + pout_prefix);
+#endif
+
+        // only create hdf5 directory in setupAMRObject (when it becomes needed)
     }
 
     void read_grid_params(GRParmParse &pp)
@@ -228,6 +303,7 @@ class ChomboParameters
             L = (L_full * max_N) / max_N_full;
 
         coarsest_dx = L / max_N;
+        coarsest_dt = coarsest_dx * dt_multiplier;
 
         // grid spacing params
         dx.fill(coarsest_dx);
@@ -347,6 +423,12 @@ class ChomboParameters
                         plot_interval <= 0 || plot_prefix != checkpoint_prefix,
                         "should be different to checkpoint_prefix");
 
+        check_parameter("output_path", output_path,
+                        FilesystemTools::directory_exists(output_path),
+                        "should be a valid directory");
+        // pout directory exists - we create it in read_filesystem_params()
+        // can't check hdf5 directory yet - only created after
+
         if (boundary_params.reflective_boundaries_exist)
         {
             for (int ivar = 0; ivar < NUM_VARS; ++ivar)
@@ -377,9 +459,10 @@ class ChomboParameters
     int verbosity;
     double L;                               // Physical sidelength of the grid
     std::array<double, CH_SPACEDIM> center; // grid center
-    IntVect ivN;        // The number of grid cells in each dimension
-    double coarsest_dx; // The coarsest resolution
-    int max_level;      // the max number of regriddings to do
+    IntVect ivN; // The number of grid cells in each dimension
+    double coarsest_dx,
+        coarsest_dt; // The coarsest resolution in space and time
+    int max_level;   // the max number of regriddings to do
     int max_spatial_derivative_order; // The maximum order of the spatial
                                       // derivatives - does nothing
                                       // in Chombo but can be used in examples
@@ -398,7 +481,14 @@ class ChomboParameters
     int checkpoint_interval, plot_interval; // Steps between outputs
     int max_grid_size, block_factor;        // max and min box sizes
     double fill_ratio; // determines how fussy the regridding is about tags
-    std::string checkpoint_prefix, plot_prefix; // naming of files
+    std::string checkpoint_prefix, plot_prefix, pout_prefix; // naming of files
+    std::string output_path; // base path to use for all files
+#ifdef CH_MPI
+    std::string pout_path; // base path for pout files
+#endif
+#ifdef CH_USE_HDF5
+    std::string hdf5_path; // base path for pout files
+#endif
     bool write_plot_ghosts;
     int num_plot_vars;
     std::vector<std::pair<int, VariableType>>
@@ -416,6 +506,7 @@ class ChomboParameters
     // For checking parameters and then exiting rather before instantiating
     // GRAMR (or child) object
     bool just_check_params = false;
+    bool print_progress_only_to_rank_0;
 
   protected:
     // the low and high corners of the domain taking into account reflective BCs