diff --git a/Examples/BinaryBH/BinaryBHLevel.cpp b/Examples/BinaryBH/BinaryBHLevel.cpp index 14acf7770..64d73c44a 100644 --- a/Examples/BinaryBH/BinaryBHLevel.cpp +++ b/Examples/BinaryBH/BinaryBHLevel.cpp @@ -178,9 +178,9 @@ void BinaryBHLevel::specificPostTimeStep() AMRReductions amr_reductions(m_gr_amr); double L2_Ham = amr_reductions.norm(c_Ham); double L2_Mom = amr_reductions.norm(Interval(c_Mom1, c_Mom3)); - SmallDataIO constraints_file("constraint_norms", m_dt, m_time, - m_restart_time, SmallDataIO::APPEND, - first_step); + SmallDataIO constraints_file(m_p.data_path + "constraint_norms", + m_dt, m_time, m_restart_time, + SmallDataIO::APPEND, first_step); constraints_file.remove_duplicate_time_data(); if (first_step) { diff --git a/Examples/BinaryBH/Main_BinaryBH.cpp b/Examples/BinaryBH/Main_BinaryBH.cpp index 2e9b65478..2080ad469 100644 --- a/Examples/BinaryBH/Main_BinaryBH.cpp +++ b/Examples/BinaryBH/Main_BinaryBH.cpp @@ -55,7 +55,7 @@ int runGRChombo(int argc, char *argv[]) int puncture_tracker_min_level = sim_params.max_level - 1; bh_amr.m_puncture_tracker.initial_setup( {sim_params.bh1_params.center, sim_params.bh2_params.center}, - sim_params.checkpoint_prefix, puncture_tracker_min_level); + "punctures", sim_params.data_path, puncture_tracker_min_level); } // The line below selects the problem that is simulated diff --git a/Examples/BinaryBH/SimulationParameters.hpp b/Examples/BinaryBH/SimulationParameters.hpp index 1ffd7165f..d3a701dc4 100644 --- a/Examples/BinaryBH/SimulationParameters.hpp +++ b/Examples/BinaryBH/SimulationParameters.hpp @@ -34,9 +34,7 @@ class SimulationParameters : public SimulationParametersBase /// Read shared parameters void read_shared_params(GRParmParse &pp) { - // Do we want Weyl extraction, puncture tracking and constraint norm - // calculation? - pp.load("activate_extraction", activate_extraction, false); + // Do we want puncture tracking and constraint norm calculation? pp.load("track_punctures", track_punctures, false); pp.load("puncture_tracking_level", puncture_tracking_level, max_level); pp.load("calculate_constraint_norms", calculate_constraint_norms, @@ -297,9 +295,9 @@ class SimulationParameters : public SimulationParametersBase "must be between 0 and max_level (inclusive)"); } - // Initial data - bool activate_extraction, track_punctures, calculate_constraint_norms; + bool track_punctures, calculate_constraint_norms; int puncture_tracking_level; + // Collection of parameters necessary for initial conditions // Set these even in the case of TwoPunctures as they are used elsewhere // e.g. for puncture tracking/tagging diff --git a/Examples/BinaryBH/params.txt b/Examples/BinaryBH/params.txt index b3d19cf48..5c8699efe 100644 --- a/Examples/BinaryBH/params.txt +++ b/Examples/BinaryBH/params.txt @@ -3,11 +3,56 @@ # and a guide to this specific example at # https://github.com/GRChombo/GRChombo/wiki/Running-the-BBH-example -# location / naming of output files (must give full path) +################################################# +# Filesystem parameters + verbosity = 0 + +# location / naming of output files +# output_path = "" # Main path for all files. Must exist! chk_prefix = BinaryBHChk_ plot_prefix = BinaryBHPlot_ -#restart_file = BinaryBHChk_000000.3d.hdf5 +# restart_file = BinaryBHChk_000000.3d.hdf5 + +# HDF5files are written every dt = L/N*dt_multiplier*checkpoint_interval +checkpoint_interval = 100 +# set to 0 to turn off plot files (except at t=0 and t=stop_time) +# set to -1 to never ever print plotfiles +plot_interval = 10 +num_plot_vars = 3 +plot_vars = chi Weyl4_Re Weyl4_Im + +# subpaths - specific directories for hdf5, pout, extraction data +# (these are created at runtime) +hdf5_subpath = "hdf5" +pout_subpath = "pout" +data_subpath = "data" + +# change the name of output files +# pout_prefix = "pout" +print_progress_only_to_rank_0 = 1 + +# ignore_checkpoint_name_mismatch = 0 +# write_plot_ghosts = 0 + +################################################# +# Initial Data parameters + +# provide 'offset' or 'center' + +massA = 0.48847892320123 +massB = 0.48847892320123 + +offsetA = 0.0 6.10679 0.0 +offsetB = 0.0 -6.10679 0.0 +# centerA = 256 232 256 +# centerB = 256 250 256 + +momentumA = -0.0841746 -0.000510846 0.0 +momentumB = 0.0841746 0.000510846 0.0 + +################################################# +# Grid parameters # 'N' is the number of subdivisions in each direction of a cubic box # 'L' is the length of the longest side of the box, dx_coarsest = L/N @@ -19,46 +64,29 @@ plot_prefix = BinaryBHPlot_ N_full = 64 L_full = 512 -# Spatial derivative order (only affects CCZ4 RHS) -max_spatial_derivative_order = 4 # can be 4 or 6 - -# Defaults to center of grid L/2 -# uncomment to change -#center = 256.0 256.0 256.0 - -# BH params, use puncture tracking to -# ensure horizons resolved -track_punctures = 1 -puncture_tracking_level = 5 -massA = 0.48847892320123 -offsetA = 0.0 6.10679 0.0 -momentumA = -0.0841746 -0.000510846 0.0 -massB = 0.48847892320123 -offsetB = 0.0 -6.10679 0.0 -momentumB = 0.0841746 0.000510846 0.0 +# Maximum number of times you can regrid above coarsest level +max_level = 9 # There are (max_level+1) grids, so min is zero -# regridding control, specify threshold (same on each level) +# Frequency of regridding at each level and thresholds on the tagging +# Need one for each level except the top one, ie max_level items +# Generally you do not need to regrid frequently on every level +# Level Regridding: 0 1 2 3 4 5 6 7 8 +regrid_interval = 0 0 0 64 64 64 64 64 64 regrid_threshold = 0.05 -max_level = 9 -# need max_level entries for regrid interval -regrid_interval = 0 0 0 64 64 64 64 64 64 # Max and min box sizes -max_grid_size = 16 -block_factor = 16 +max_box_size = 16 +min_box_size = 16 -# Set up time steps -# dt will be dx*dt_multiplier on each grid level -# HDF5files are written every dt = L/N*dt_multiplier*checkpoint_interval -checkpoint_interval = 100 -# set to zero to turn off plot files, comps defined in BinaryBHLevel.cpp -plot_interval = 10 -num_plot_vars = 3 -plot_vars = chi Weyl4_Re Weyl4_Im -dt_multiplier = 0.25 -stop_time = 2200.0 +# tag_buffer_size = 3 +# grid_buffer_size = 8 +# fill_ratio = 0.7 +# num_ghosts = 3 +# center = 256.0 256.0 256.0 # defaults to center of the grid + +################################################# +# Boundary Conditions parameters -# boundaries and periodicity of grid # Periodic directions - 0 = false, 1 = true isPeriodic = 0 0 0 # if not periodic, then specify the boundary type @@ -86,34 +114,62 @@ num_nonzero_asymptotic_vars = 5 nonzero_asymptotic_vars = chi h11 h22 h33 lapse nonzero_asymptotic_values = 1.0 1.0 1.0 1.0 1.0 +# if you are using extrapolating BC: +# extrapolation_order = 1 +# num_extrapolating_vars = -1 +# extrapolating_vars = + +################################################# +# Evolution parameters + +# dt will be dx*dt_multiplier on each grid level +dt_multiplier = 0.25 +stop_time = 2200.0 +# max_steps = 100 + +# Spatial derivative order (only affects CCZ4 RHS) +max_spatial_derivative_order = 4 # can be 4 or 6 + +nan_check = 1 + # Lapse evolution lapse_advec_coeff = 1.0 lapse_coeff = 2.0 lapse_power = 1.0 # Shift evolution -shift_advec_coeff = 0.0 +shift_advec_coeff = 0.0 # Usually no advection for beta shift_Gamma_coeff = 0.75 -eta = 1.0 +eta = 1.0 # eta of gamma driver, should be of order ~1/M_ADM of spacetime # CCZ4 parameters +formulation = 0 # 1 for BSSN, 0 for CCZ4 kappa1 = 0.1 -kappa2 = 0 -kappa3 = 1.0 +kappa2 = 0. +kappa3 = 1. covariantZ4 = 1 # 0: keep kappa1; 1 [default]: replace kappa1 -> kappa1/lapse # coefficient for KO numerical dissipation sigma = 1.0 -# extraction params -# default of extraction_center is center, uncomment to change -#extraction_center = 256 256 256 +track_punctures = 1 +puncture_tracking_level = 5 + +# calculate_constraint_norms = 0 + +# min_chi = 1.e-4 +# min_lapse = 1.e-4 + +################################################# +# Extraction parameters + +# extraction_center = 256 256 256 # defaults to center activate_extraction = 1 num_extraction_radii = 2 extraction_radii = 50.0 100.0 extraction_levels = 2 1 num_points_phi = 24 -num_points_theta = 36 +num_points_theta = 37 num_modes = 8 modes = 2 0 # l m for spherical harmonics 2 1 @@ -123,3 +179,12 @@ modes = 2 0 # l m for spherical harmonics 4 2 4 3 4 4 + +# integral_file_prefix = "Weyl4_mode_" + +# write_extraction = 0 +# extraction_subpath = "data/extraction" # directory for 'write_extraction = 1' +# extraction_file_prefix = "Weyl4_extraction_" + +################################################# + diff --git a/Examples/BinaryBH/params_expensive.txt b/Examples/BinaryBH/params_expensive.txt deleted file mode 100644 index 668fe14c6..000000000 --- a/Examples/BinaryBH/params_expensive.txt +++ /dev/null @@ -1,82 +0,0 @@ -verbosity = 0 -chk_prefix = BinaryBH_ -plot_prefix = BinaryBHPlot_ -#restart_file = BinaryBH_000360.3d.hdf5 - -# Set up grid spacings and regrid params -# NB - the N values need to be multiples of block_factor - -N1 = 256 -N2 = 256 -N3 = 256 -L = 192 - -# Spatial derivative order (only affects CCZ4 RHS) -max_spatial_derivative_order = 4 # can be 4 or 6 - -massA = 0.5 -offsetA = 3 0 0 - -massB = 0.5 -offsetB = -3 0 0 - -momentumA = 0. -0.1 0.0 -momentumB = 0. 0.1 0.0 - -regrid_threshold = 0.20 -max_level = 5 -regrid_interval = 512 256 32 16 8 4 2 2 2 -isPeriodic = 1 1 1 - -#Max and min box sizes -max_grid_size = 32 -block_factor = 16 -tag_buffer_size = 3 - -# Set up time steps -# dt will be dx*dt_multiplier on each grid level -# HDF5files are written every dt = L/N*dt_multiplier*checkpoint_interval - -checkpoint_interval = 4 -# set to zero to turn off plot files -plot_interval = 0 -num_plot_vars = 3 -plot_vars = chi Weyl4_Re Weyl4_Im -dt_multiplier = 0.25 -stop_time = 1.0 -max_steps = 2 - -nan_check = 1 - -#Lapse evolution -lapse_advec_coeff = 0.0 -lapse_coeff = 2.0 -lapse_power = 1.0 - -# Shift evolution -shift_advec_coeff = 0.0 -shift_Gamma_coeff = 0.75 -eta = 1.82 - -# CCZ4 parameters -kappa1 = 0.1 -kappa2 = 0 -kappa3 = 1. -covariantZ4 = 1 # 0: keep kappa1; 1 [default]: replace kappa1 -> kappa1/lapse - -#coefficient for KO numerical dissipation -sigma = 0.3 - -#extraction params -#default center to grid center, uncomment to change -#extraction_center = 64 64 64 -activate_extraction = 1 -num_extraction_radii = 3 -extraction_radii = 10. 30. 50. -extraction_levels = 2 1 1 -num_points_phi = 16 -num_points_theta = 24 -num_modes = 3 -modes = 2 0 # l m for spherical harmonics - 2 1 - 2 2 diff --git a/Examples/BinaryBH/params_two_punctures.txt b/Examples/BinaryBH/params_two_punctures.txt index 1f5f239bb..295bec8ea 100644 --- a/Examples/BinaryBH/params_two_punctures.txt +++ b/Examples/BinaryBH/params_two_punctures.txt @@ -1,11 +1,56 @@ # See the wiki page for an explanation of the params! # https://github.com/GRChombo/GRChombo/wiki/Guide-to-parameters -# location / naming of output files (must give full path) +################################################# +# Filesystem parameters + verbosity = 0 + +# location / naming of output files +# output_path = "" # Main path for all files. Must exist! chk_prefix = BinaryBHChk_ plot_prefix = BinaryBHPlot_ -#restart_file = BinaryBHChk_000000.3d.hdf5 +# restart_file = BinaryBHChk_000000.3d.hdf5 + +# HDF5files are written every dt = L/N*dt_multiplier*checkpoint_interval +checkpoint_interval = 100 +# set to 0 to turn off plot files (except at t=0 and t=stop_time) +# set to -1 to never ever print plotfiles +plot_interval = 10 +num_plot_vars = 3 +plot_vars = chi Weyl4_Re Weyl4_Im + +# subpaths - specific directories for hdf5, pout, extraction data +# (these are created at runtime) +hdf5_subpath = "hdf5" +pout_subpath = "pout" +data_subpath = "data" + +# change the name of output files +# pout_prefix = "pout" +print_progress_only_to_rank_0 = 1 + +# ignore_checkpoint_name_mismatch = 0 +# write_plot_ghosts = 0 + +################################################# +# Initial Data parameters + +# provide 'offset' or 'center' + +massA = 0.48847892320123 +massB = 0.48847892320123 + +offsetA = 0.0 6.10679 0.0 +offsetB = 0.0 -6.10679 0.0 +# centerA = 256 232 256 +# centerB = 256 250 256 + +momentumA = -0.0841746 -0.000510846 0.0 +momentumB = 0.0841746 0.000510846 0.0 + +################################################# +# Grid parameters # 'N' is the number of subdivisions in each direction of a cubic box # 'L' is the length of the longest side of the box, dx_coarsest = L/N @@ -17,37 +62,27 @@ plot_prefix = BinaryBHPlot_ N_full = 64 L_full = 512 -# Defaults to center of grid L/2 -# uncomment to change -#center = 256.0 256.0 256.0 - -# BH params, use puncture tracking to -# ensure horizons resolved -track_punctures = 1 -puncture_tracking_level = 5 +# Maximum number of times you can regrid above coarsest level +max_level = 9 # There are (max_level+1) grids, so min is zero -# regridding control, specify threshold (same on each level) -regrid_threshold = 0.05 -max_level = 9 # need max_level entries for regrid interval -regrid_interval = 0 0 0 64 64 64 64 64 64 +# Level Regridding: 0 1 2 3 4 5 6 7 8 +regrid_interval = 0 0 0 64 64 64 64 64 64 +regrid_threshold = 0.05 # Max and min box sizes -max_grid_size = 16 -block_factor = 16 +max_box_size = 16 +min_box_size = 16 -# Set up time steps -# dt will be dx*dt_multiplier on each grid level -# HDF5files are written every dt = L/N*dt_multiplier*checkpoint_interval -checkpoint_interval = 100 -# set to zero to turn off plot files, comps defined in BinaryBHLevel.cpp -plot_interval = 10 -num_plot_vars = 3 -plot_vars = chi Weyl4_Re Weyl4_Im -dt_multiplier = 0.25 -stop_time = 2200.0 +# tag_buffer_size = 3 +# grid_buffer_size = 8 +# fill_ratio = 0.7 +# num_ghosts = 3 +# center = 256.0 256.0 256.0 # defaults to center of the grid + +################################################# +# Boundary Conditions parameters -# boundaries and periodicity of grid # Periodic directions - 0 = false, 1 = true isPeriodic = 0 0 0 # if not periodic, then specify the boundary type @@ -75,34 +110,62 @@ num_nonzero_asymptotic_vars = 5 nonzero_asymptotic_vars = chi h11 h22 h33 lapse nonzero_asymptotic_values = 1.0 1.0 1.0 1.0 1.0 +# if you are using extrapolating BC: +# extrapolation_order = 1 +# num_extrapolating_vars = -1 +# extrapolating_vars = + +################################################# +# Evolution parameters + +# dt will be dx*dt_multiplier on each grid level +dt_multiplier = 0.25 +stop_time = 2200.0 +# max_steps = 100 + +# Spatial derivative order (only affects CCZ4 RHS) +max_spatial_derivative_order = 4 # can be 4 or 6 + +nan_check = 1 + # Lapse evolution lapse_advec_coeff = 1.0 lapse_coeff = 2.0 lapse_power = 1.0 # Shift evolution -shift_advec_coeff = 0.0 +shift_advec_coeff = 0.0 # Usually no advection for beta shift_Gamma_coeff = 0.75 -eta = 1.0 +eta = 1.0 # eta of gamma driver, should be of order ~1/M_ADM of spacetime # CCZ4 parameters +formulation = 0 # 1 for BSSN, 0 for CCZ4 kappa1 = 0.1 -kappa2 = 0 -kappa3 = 1.0 -covariantZ4 = 1 # 0: default. 1: dampk1 -> dampk1/lapse +kappa2 = 0. +kappa3 = 1. +covariantZ4 = 1 # 0: keep kappa1; 1 [default]: replace kappa1 -> kappa1/lapse # coefficient for KO numerical dissipation sigma = 1.0 -# extraction params -# default of extraction_center is center, uncomment to change -#extraction_center = 256 256 256 +track_punctures = 1 +puncture_tracking_level = 5 + +# calculate_constraint_norms = 0 + +# min_chi = 1.e-4 +# min_lapse = 1.e-4 + +################################################# +# Extraction parameters + +# extraction_center = 256 256 256 # defaults to center activate_extraction = 1 num_extraction_radii = 2 extraction_radii = 50.0 100.0 extraction_levels = 2 1 num_points_phi = 24 -num_points_theta = 36 +num_points_theta = 37 num_modes = 8 modes = 2 0 # l m for spherical harmonics 2 1 @@ -113,17 +176,23 @@ modes = 2 0 # l m for spherical harmonics 4 3 4 4 +# integral_file_prefix = "Weyl4_mode_" -### Two Punctures params +# write_extraction = 0 +# extraction_subpath = "data/extraction" # directory for 'write_extraction = 1' +# extraction_file_prefix = "Weyl4_extraction_" + +################################################# +# Two Punctures parameters # Main BH params # Either calculate target masses or set bare masses explicitly below TP_calculate_target_masses = true TP_target_mass_plus = 0.5 TP_target_mass_minus = 0.5 -#TP_adm_tol = 1e-10 -#TP_mass_plus = 0.9773790195003562 -#TP_mass_minus = 0.4758625198306865 +# TP_adm_tol = 1e-10 +# TP_mass_plus = 0.48847892320123 +# TP_mass_minus = 0.48847892320123 # offset in x direction (or z if TP_swap_xz set true) TP_offset_plus = 6.10679 TP_offset_minus = -6.10679 @@ -134,14 +203,14 @@ TP_spin_plus = 0.0 0.0 0.0 TP_spin_minus = 0.0 0.0 0.0 # Solver params -#TP_npoints_A = 30 -#TP_npoints_B = 30 -#TP_npoints_phi = 16; -#TP_Newton_tol = 1e-10 -#TP_Newton_maxit = 5 +# TP_npoints_A = 30 +# TP_npoints_B = 30 +# TP_npoints_phi = 16; +# TP_Newton_tol = 1e-10 +# TP_Newton_maxit = 5 TP_epsilon = 1e-6 -#TP_Tiny = 0.0 -#TP_Extend_Radius = 0.0 +# TP_Tiny = 0.0 +# TP_Extend_Radius = 0.0 # Initial data params TP_use_spectral_interpolation = true @@ -149,5 +218,7 @@ TP_initial_lapse = psi^n TP_initial_lapse_psi_exponent = -2.0 # Debug output -#TP_do_residuum_debug_output = false -#TP_do_initial_debug_output = false +# TP_do_residuum_debug_output = false +# TP_do_initial_debug_output = false + +################################################# diff --git a/Examples/BinaryBH/params_very_cheap.txt b/Examples/BinaryBH/params_very_cheap.txt index 4e62738d8..3d1123202 100644 --- a/Examples/BinaryBH/params_very_cheap.txt +++ b/Examples/BinaryBH/params_very_cheap.txt @@ -1,69 +1,88 @@ +# See the wiki page for an explanation of the params! +# https://github.com/GRChombo/GRChombo/wiki/Guide-to-parameters + +################################################# +# Filesystem parameters + verbosity = 0 + +# location / naming of output files +# output_path = "" # Main path for all files. Must exist! chk_prefix = BinaryBH_ plot_prefix = BinaryBHPlot_ -#restart_file = BinaryBH_000360.3d.hdf5 +# restart_file = BinaryBH_000360.3d.hdf5 -# Set up grid spacings and regrid params -# NB - the N values need to be multiples of block_factor - -# Spatial derivative order (only affects CCZ4 RHS) -max_spatial_derivative_order = 4 # can be 4 or 6 - -N_full = 32 -L_full = 16 +# HDF5files are written every dt = L/N*dt_multiplier*checkpoint_interval +checkpoint_interval = 1 +# set to 0 to turn off plot files (except at t=0 and t=stop_time) +# set to -1 to never ever print plotfiles +plot_interval = 1 +num_plot_vars = 4 +plot_vars = chi Ham Weyl4_Re Weyl4_Im -massA = 1 -offsetA = -2 0 0 +# subpaths - specific directories for hdf5, pout, extraction data +# (these are created at runtime) +hdf5_subpath = "hdf5" +pout_subpath = "pout" +data_subpath = "data" -massB = 0.82 -offsetB = 2 0 0 +# change the name of output files +# pout_prefix = "pout" +print_progress_only_to_rank_0 = 1 -momentumA = 0. -0.14025 0.0 -momentumB = 0. 0.14025 0.0 +# ignore_checkpoint_name_mismatch = 0 +# write_plot_ghosts = 0 -regrid_threshold = 0.3 -max_level = 1 -regrid_interval = 1 1 1 1 0 0 0 0 0 +################################################# +# Initial Data parameters -#Max and min box sizes -max_grid_size = 32 -block_factor = 8 -tag_buffer_size = 3 +# provide 'offset' or 'center' -# Set up time steps -# dt will be dx*dt_multiplier on each grid level -# HDF5files are written every dt = L/N*dt_multiplier*checkpoint_interval +massA = 0.5 +massB = 0.5 -checkpoint_interval = 1 -# set to zero to turn off plot files -plot_interval = 1 -num_plot_vars = 4 -plot_vars = chi Ham Weyl4_Re Weyl4_Im -dt_multiplier = 0.25 -stop_time = 10.0 +offsetA = -2 0 0 +offsetB = 2 0 0 +# centerA = 6 8 8 +# centerB = 10 8 8 + +momentumA = 0.0 -0.1 0.0 +momentumB = 0.0 0.1 0.0 + +################################################# +# Grid parameters + +# 'N' is the number of subdivisions in each direction of a cubic box +# 'L' is the length of the longest side of the box, dx_coarsest = L/N +# NB - If you use reflective BC and want to specify the subdivisions and side +# of the box were there are no symmetries, specify 'N_full' and 'L_full' instead +# NB - if you have a non-cubic grid, you can specify 'N1' or 'N1_full', +# 'N2' or 'N2_full' and 'N3' or 'N3_full' ( then dx_coarsest = L/N(max) ) +# NB - the N values need to be multiples of the block_factor +N_full = 32 +L_full = 16 -nan_check = 1 +# Maximum number of times you can regrid above coarsest level +max_level = 2 # There are (max_level+1) grids, so min is zero -#Lapse evolution -lapse_advec_coeff = 0.0 -lapse_coeff = 2.0 -lapse_power = 1.0 +# need max_level entries for regrid interval +# Level Regridding: 0 1 2 3 4 5 +regrid_interval = 1 1 1 1 0 0 +regrid_threshold = 0.3 -# Shift evolution -shift_advec_coeff = 0.0 -shift_Gamma_coeff = 0.75 -eta = 1.82 +# Max and min box sizes +max_box_size = 32 +min_box_size = 8 -# CCZ4 parameters -kappa1 = 0.1 -kappa2 = 0 -kappa3 = 1. -covariantZ4 = 1 # 0: keep kappa1; 1 [default]: replace kappa1 -> kappa1/lapse +# tag_buffer_size = 3 +# grid_buffer_size = 8 +# fill_ratio = 0.7 +# num_ghosts = 3 +# center = 8 8 8 # defaults to center of the grid -#coefficient for KO numerical dissipation -sigma = 0.3 +################################################# +# Boundary Conditions parameters -# boundaries and periodicity of grid # Periodic directions - 0 = false, 1 = true isPeriodic = 0 0 0 # if not periodic, then specify the boundary type @@ -76,7 +95,6 @@ lo_boundary = 1 1 2 # parity of all vars (in order given by UserVariables.hpp) # 0 = even # 1,2,3 = odd x, y, z -# 4,5,6 = odd xy, yz, xz # 7 = odd xyz vars_parity = 0 0 4 6 0 5 0 #chi and hij 0 0 4 6 0 5 0 #K and Aij @@ -91,18 +109,71 @@ num_nonzero_asymptotic_vars = 5 nonzero_asymptotic_vars = chi h11 h22 h33 lapse nonzero_asymptotic_values = 1.0 1.0 1.0 1.0 1.0 +# if you are using extrapolating BC: +# extrapolation_order = 1 +# num_extrapolating_vars = -1 +# extrapolating_vars = + +################################################# +# Evolution parameters + +# dt will be dx*dt_multiplier on each grid level +dt_multiplier = 0.25 +stop_time = 10.0 +# max_steps = 100 + +# Spatial derivative order (only affects CCZ4 RHS) +max_spatial_derivative_order = 4 # can be 4 or 6 + +nan_check = 1 + +# Lapse evolution +lapse_advec_coeff = 1.0 +lapse_coeff = 2.0 +lapse_power = 1.0 + +# Shift evolution +shift_advec_coeff = 0.0 # Usually no advection for beta +shift_Gamma_coeff = 0.75 +eta = 1.0 # eta of gamma driver, should be of order ~1/M_ADM of spacetime + +# CCZ4 parameters +formulation = 0 # 1 for BSSN, 0 for CCZ4 +kappa1 = 0.1 +kappa2 = 0. +kappa3 = 1. +covariantZ4 = 1 # 0: keep kappa1; 1 [default]: replace kappa1 -> kappa1/lapse + +# coefficient for KO numerical dissipation +sigma = 0.3 + track_punctures = 1 puncture_tracking_level = 1 -# extraction params -# default of extraction_center is center, uncomment to change +# calculate_constraint_norms = 0 + +# min_chi = 1.e-4 +# min_lapse = 1.e-4 + +################################################# +# Extraction parameters + +# extraction_center = 8 8 8 # defaults to center activate_extraction = 1 num_extraction_radii = 1 extraction_radii = 6.0 extraction_levels = 0 num_points_phi = 24 -num_points_theta = 36 +num_points_theta = 37 num_modes = 3 modes = 2 0 # l m for spherical harmonics 2 1 2 2 +# integral_file_prefix = "Weyl4_mode_" + +# write_extraction = 0 +# extraction_subpath = "data/extraction" # directory for 'write_extraction = 1' +# extraction_file_prefix = "Weyl4_extraction_" + +################################################# + diff --git a/Examples/KerrBH/params.txt b/Examples/KerrBH/params.txt index 81f66a6ca..1ce6a12d4 100644 --- a/Examples/KerrBH/params.txt +++ b/Examples/KerrBH/params.txt @@ -1,9 +1,47 @@ -#Params for runtime inputs +# See the wiki page for an explanation of the params! +# https://github.com/GRChombo/GRChombo/wiki/Guide-to-parameters + +################################################# +# Filesystem parameters verbosity = 0 + +# location / naming of output files +# output_path = "" # Main path for all files. Must exist! chk_prefix = KerrBH_ plot_prefix = KerrBHp_ -#restart_file = KerrBH_000060.3d.hdf5 +# restart_file = KerrBH_000060.3d.hdf5 + +# HDF5files are written every dt = L/N*dt_multiplier*checkpoint_interval +checkpoint_interval = 50 +# set to 0 to turn off plot files (except at t=0 and t=stop_time) +# set to -1 to never ever print plotfiles +plot_interval = 0 +num_plot_vars = 0 +# plot_vars = + +# subpaths - specific directories for hdf5, pout, extraction data +# (these are created at runtime) +hdf5_subpath = "hdf5" +pout_subpath = "pout" +data_subpath = "data" + +# change the name of output files +# pout_prefix = "pout" +print_progress_only_to_rank_0 = 1 + +# ignore_checkpoint_name_mismatch = 0 +# write_plot_ghosts = 0 + +################################################# +# Initial Data parameters + +kerr_mass = 1.0 +kerr_spin = 0.5 +# kerr_center = 64 64 64 # defauts to center + +################################################# +# Grid parameters # 'N' is the number of subdivisions in each direction of a cubic box # 'L' is the length of the longest side of the box, dx_coarsest = L/N @@ -12,22 +50,9 @@ plot_prefix = KerrBHp_ # NB - if you have a non-cubic grid, you can specify 'N1' or 'N1_full', # 'N2' or 'N2_full' and 'N3' or 'N3_full' ( then dx_coarsest = L/N(max) ) # NB - the N values need to be multiples of the block_factor -N_full = 64 +N_full = 128 L_full = 128 -# Spatial derivative order (only affects CCZ4 RHS) -max_spatial_derivative_order = 4 # can be 4 or 6 - -# Params for Kerr BH -kerr_mass = 1.0 -kerr_spin = 0.5 -# This is now defaulted to center of the grid so it does not need to be updated -# when changing L (unless one wants to put the BH somewhere off center) -#kerr_center = 64 64 64 - -# Regridding - -# Level data # Maximum number of times you can regrid above coarsest level max_level = 6 # There are (max_level+1) grids, so min is zero @@ -36,17 +61,22 @@ max_level = 6 # There are (max_level+1) grids, so min is zero # Generally you do not need to regrid frequently on every level # Level Regridding: 0 1 2 3 4 5 regrid_interval = 50 25 5 5 5 5 -regrid_thresholds = 0.3 0.3 0.3 0.3 0.3 0.3 +regrid_thresholds = 0.1 0.1 0.1 0.1 0.1 0.1 + +# Max and min box sizes +max_box_size = 16 +min_box_size = 16 + +# tag_buffer_size = 3 +# grid_buffer_size = 8 +# fill_ratio = 0.7 +# num_ghosts = 3 +# center = 256.0 256.0 256.0 # defaults to center of the grid -# Max box sizes -max_grid_size = 16 -# Min box size -block_factor = 16 -# Determines how exactly regridding tries to fit the tagging -fill_ratio = 0.75 +################################################# +# Boundary Conditions parameters -#boundaries and periodicity of grid -#Periodic directions - 0 = false, 1 = true +# Periodic directions - 0 = false, 1 = true isPeriodic = 0 0 0 # if not periodic, then specify the boundary type # 0 = static, 1 = sommerfeld, 2 = reflective @@ -72,41 +102,46 @@ num_nonzero_asymptotic_vars = 5 nonzero_asymptotic_vars = chi h11 h22 h33 lapse nonzero_asymptotic_values = 1.0 1.0 1.0 1.0 1.0 -# Set up time steps +# if you are using extrapolating BC: +# extrapolation_order = 1 +# num_extrapolating_vars = -1 +# extrapolating_vars = + +################################################# +# Evolution parameters + # dt will be dx*dt_multiplier on each grid level -# HDF5files are written every dt = L/N*dt_multiplier*checkpoint_interval -checkpoint_interval = 50 -plot_interval = 10 -num_plot_vars = 0 dt_multiplier = 0.25 stop_time = 20.0 -max_steps = 10000000 +# max_steps = 100 -#Lapse evolution -lapse_power = 1.0 +# Spatial derivative order (only affects CCZ4 RHS) +max_spatial_derivative_order = 4 # can be 4 or 6 + +nan_check = 1 + +# Lapse evolution +lapse_advec_coeff = 1.0 lapse_coeff = 2.0 -lapse_advec_coeff = 1 # 1 makes the lapse gauge 1+log slicing +lapse_power = 1.0 -# Shift evolution coefficients -shift_advec_coeff = 0 # Usually no advection for beta -shift_Gamma_coeff = 0.75 # -eta = 1.0 # This is gamma driver, usually of order 1/M_ADM of spacetime +# Shift evolution +shift_advec_coeff = 0.0 # Usually no advection for beta +shift_Gamma_coeff = 0.75 +eta = 1.0 # eta of gamma driver, should be of order ~1/M_ADM of spacetime # CCZ4 parameters -# if using BSSN the kappa values should be zero -formulation = 1 # 1 for BSSN, 0 for CCZ4 -kappa1 = 0 -kappa2 = 0 -kappa3 = 0 +formulation = 1 # 1 for BSSN, 0 for CCZ4 +kappa1 = 0. +kappa2 = 0. +kappa3 = 0. covariantZ4 = 1 # 0: keep kappa1; 1 [default]: replace kappa1 -> kappa1/lapse # coefficient for KO numerical dissipation sigma = 0.3 -#extraction params -activate_extraction = 0 -num_extraction_radii = 3 -extraction_radii = 30. 40. 50. -extraction_levels = 2 1 0 -num_points_phi = 50 -num_points_theta = 52 +# min_chi = 1.e-4 +# min_lapse = 1.e-4 + +################################################# + diff --git a/Examples/KerrBH/params_cheap.txt b/Examples/KerrBH/params_cheap.txt index 838137673..10135d106 100644 --- a/Examples/KerrBH/params_cheap.txt +++ b/Examples/KerrBH/params_cheap.txt @@ -1,9 +1,47 @@ -#Params for runtime inputs +# See the wiki page for an explanation of the params! +# https://github.com/GRChombo/GRChombo/wiki/Guide-to-parameters + +################################################# +# Filesystem parameters verbosity = 0 + +# location / naming of output files +# output_path = "" # Main path for all files. Must exist! chk_prefix = KerrBH_ plot_prefix = KerrBHp_ -#restart_file = KerrBH_000060.3d.hdf5 +# restart_file = KerrBH_000060.3d.hdf5 + +# HDF5files are written every dt = L/N*dt_multiplier*checkpoint_interval +checkpoint_interval = 1 +# set to 0 to turn off plot files (except at t=0 and t=stop_time) +# set to -1 to never ever print plotfiles +plot_interval = 0 +num_plot_vars = 0 +# plot_vars = + +# subpaths - specific directories for hdf5, pout, extraction data +# (these are created at runtime) +hdf5_subpath = "hdf5" +pout_subpath = "pout" +data_subpath = "data" + +# change the name of output files +# pout_prefix = "pout" +print_progress_only_to_rank_0 = 1 + +# ignore_checkpoint_name_mismatch = 0 +# write_plot_ghosts = 0 + +################################################# +# Initial Data parameters + +kerr_mass = 1.0 +kerr_spin = 0.5 +# kerr_center = 64 64 64 # defauts to center + +################################################# +# Grid parameters # 'N' is the number of subdivisions in each direction of a cubic box # 'L' is the length of the longest side of the box, dx_coarsest = L/N @@ -15,38 +53,30 @@ plot_prefix = KerrBHp_ N_full = 64 L_full = 128 -# Spatial derivative order (only affects CCZ4 RHS) -max_spatial_derivative_order = 4 # can be 4 or 6 - -# Params for Kerr BH -kerr_mass = 1.0 -kerr_spin = 0.5 -# This is now defaulted to center of the grid so it does not need to be updated -# when changing L (unless one wants to put the BH somewhere off center) -#kerr_center = 64 64 64 - -# Regridding - -# Level data # Maximum number of times you can regrid above coarsest level -max_level = 4 #4 There are (max_level+1) grids, so min is zero +max_level = 4 # There are (max_level+1) grids, so min is zero # Frequency of regridding at each level and thresholds on the tagging -# Need one for each level, ie max_level+1 items +# Need one for each level except the top one, ie max_level items # Generally you do not need to regrid frequently on every level -# Level Regridding: 0 1 2 3 4 5 6 -regrid_interval = 50 25 5 5 5 5 5 +# Level Regridding: 0 1 2 3 4 +regrid_interval = 50 25 5 5 5 regrid_thresholds = 0.3 0.3 0.3 0.3 0.3 -# Max box sizes -max_grid_size = 16 -# Min box size -block_factor = 16 -# Determines how exactly regridding tries to fit the tagging -fill_ratio = 0.75 +# Max and min box sizes +max_box_size = 16 +min_box_size = 16 + +# tag_buffer_size = 3 +# grid_buffer_size = 8 +# fill_ratio = 0.7 +# num_ghosts = 3 +# center = 256.0 256.0 256.0 # defaults to center of the grid + +################################################# +# Boundary Conditions parameters -#boundaries and periodicity of grid -#Periodic directions - 0 = false, 1 = true +# Periodic directions - 0 = false, 1 = true isPeriodic = 0 0 0 # if not periodic, then specify the boundary type # 0 = static, 1 = sommerfeld, 2 = reflective @@ -67,48 +97,50 @@ vars_parity = 0 0 4 6 0 5 0 #chi and hij vars_parity_diagnostic = 0 1 2 3 #Ham and Mom # if sommerfeld boundaries selected, must select -# asymptotic values (in order given by UserVariables.hpp) -vars_asymptotic_values = 1.0 1.0 0.0 0.0 1.0 0.0 1.0 #chi and hij - 0.0 0.0 0.0 0.0 0.0 0.0 0.0 #K and Aij - 0.0 0.0 0.0 0.0 #Theta and Gamma - 1.0 0.0 0.0 0.0 0.0 0.0 0.0 #lapse shift and B +# non zero asymptotic values +num_nonzero_asymptotic_vars = 5 +nonzero_asymptotic_vars = chi h11 h22 h33 lapse +nonzero_asymptotic_values = 1.0 1.0 1.0 1.0 1.0 + +# if you are using extrapolating BC: +# extrapolation_order = 1 +# num_extrapolating_vars = -1 +# extrapolating_vars = + +################################################# +# Evolution parameters -# Set up time steps # dt will be dx*dt_multiplier on each grid level -# HDF5files are written every dt = L/N*dt_multiplier*checkpoint_interval -checkpoint_interval = 1 -plot_interval = 0 -num_plot_vars = 0 dt_multiplier = 0.25 -stop_time = 10.0 -max_steps = 2 +# stop_time = 20.0 +max_steps = 4 -#Lapse evolution -lapse_power = 1.0 +# Spatial derivative order (only affects CCZ4 RHS) +max_spatial_derivative_order = 4 # can be 4 or 6 + +nan_check = 1 + +# Lapse evolution +lapse_advec_coeff = 1.0 lapse_coeff = 2.0 -lapse_advec_coeff = 1 # 1 makes the lapse gauge 1+log slicing +lapse_power = 1.0 -# Shift evolution coefficients -shift_advec_coeff = 0 # Usually no advection for beta -shift_Gamma_coeff = 0.75 # -eta = 1.0 # This is gamma driver, usually of order 1/M_ADM of spacetime +# Shift evolution +shift_advec_coeff = 0.0 # Usually no advection for beta +shift_Gamma_coeff = 0.75 +eta = 1.0 # eta of gamma driver, should be of order ~1/M_ADM of spacetime # CCZ4 parameters -# if using BSSN the kappa values should be zero -formulation = 1 # 1 for BSSN, 0 for CCZ4 -kappa1 = 0 -kappa2 = 0 -kappa3 = 0 +formulation = 1 # 1 for BSSN, 0 for CCZ4 +kappa1 = 0. +kappa2 = 0. +kappa3 = 0. covariantZ4 = 1 # 0: keep kappa1; 1 [default]: replace kappa1 -> kappa1/lapse # coefficient for KO numerical dissipation -# NB must be less than 0.5 for stability sigma = 0.3 -#extraction params -activate_extraction = 0 -num_extraction_radii = 3 -extraction_radii = 30. 40. 50. -extraction_levels = 1 1 0 -num_points_phi = 50 -num_points_theta = 52 +# min_chi = 1.e-4 +# min_lapse = 1.e-4 + +################################################# diff --git a/Examples/ScalarField/params.txt b/Examples/ScalarField/params.txt index 87057a296..450301bca 100644 --- a/Examples/ScalarField/params.txt +++ b/Examples/ScalarField/params.txt @@ -1,9 +1,59 @@ -#Params for runtime inputs +# See the wiki page for an explanation of the params! +# https://github.com/GRChombo/GRChombo/wiki/Guide-to-parameters + +################################################# +# Filesystem parameters verbosity = 0 + +# location / naming of output files +# output_path = "" # Main path for all files. Must exist! chk_prefix = ScalarField_ plot_prefix = ScalarFieldp_ -#restart_file = ScalarField_000000.3d.hdf5 +# restart_file = ScalarField_000000.3d.hdf5 + +# HDF5files are written every dt = L/N*dt_multiplier*checkpoint_interval +checkpoint_interval = 500 +# set to 0 to turn off plot files (except at t=0 and t=stop_time) +# set to -1 to never ever print plotfiles +plot_interval = 1 +num_plot_vars = 2 +plot_vars = chi phi + +# subpaths - specific directories for hdf5, pout, extraction data +# (these are created at runtime) +hdf5_subpath = "hdf5" +pout_subpath = "pout" +data_subpath = "data" + +# change the name of output files +# pout_prefix = "pout" +print_progress_only_to_rank_0 = 1 + +# ignore_checkpoint_name_mismatch = 0 +# write_plot_ghosts = 0 + +################################################# +# Initial Data parameters + +# Change the gravitational constant of the Universe! +# Default is 1.0, for standard geometric units +# Here we decouple the evolution so the scalar evolved on the +# metric background without backreaction (this avoids the need +# to solve the constaints) +G_Newton = 0.0 + +# Scalar field initial data +scalar_amplitude = 0.1 +scalar_width = 5.0 +scalar_mass = 0.2 + +# Kerr BH data +kerr_mass = 1.0 +kerr_spin = 0.0 + +################################################# +# Grid parameters # 'N' is the number of subdivisions in each direction of a cubic box # 'L' is the length of the longest side of the box, dx_coarsest = L/N @@ -15,24 +65,33 @@ plot_prefix = ScalarFieldp_ N_full = 128 L_full = 256 -# Spatial derivative order (only affects CCZ4 RHS) -max_spatial_derivative_order = 4 # can be 4 or 6 - -# Regridding - in this example use a fixed grid -tag_buffer_size = 0 - -# Level data # Maximum number of times you can regrid above coarsest level max_level = 6 # There are (max_level+1) grids, so min is zero -# Frequency of regridding at each level + +# Frequency of regridding at each level and thresholds on the tagging # Need one for each level except the top one, ie max_level items +# Generally you do not need to regrid frequently on every level # in this example turn off regridding on all levels +# Level Regridding: 0 1 2 3 4 5 +regrid_interval = 0 0 0 0 0 0 +# regrid_threshold = 0.5 + regrid_interval = 0 0 0 0 0 0 -# Max and min box size - for load balancing + +# Max and min box sizes max_box_size = 16 min_box_size = 16 -#boundaries and periodicity of grid +tag_buffer_size = 0 # this example uses a fixed grid + +# grid_buffer_size = 8 +# fill_ratio = 0.7 +# num_ghosts = 3 +# center = 256.0 256.0 256.0 # defaults to center of the grid + +################################################# +# Boundary Conditions parameters + #Periodic directions - 0 = false, 1 = true isPeriodic = 0 0 0 # if not periodic, then specify the boundary type @@ -61,54 +120,45 @@ num_nonzero_asymptotic_vars = 5 nonzero_asymptotic_vars = chi h11 h22 h33 lapse nonzero_asymptotic_values = 1.0 1.0 1.0 1.0 1.0 -# for extrapolating vars +# if you are using extrapolating BC: extrapolation_order = 1 num_extrapolating_vars = 2 extrapolating_vars = phi Pi -# Set up time steps +################################################# +# Evolution parameters + # dt will be dx*dt_multiplier on each grid level -# HDF5files are written every dt = L/N*dt_multiplier*checkpoint_interval -checkpoint_interval = 500 -plot_interval = 1 -num_plot_vars = 2 -plot_vars = chi phi dt_multiplier = 0.25 stop_time = 100.0 +# max_steps = 4 -#Lapse evolution -lapse_power = 1.0 +# Spatial derivative order (only affects CCZ4 RHS) +max_spatial_derivative_order = 4 # can be 4 or 6 + +nan_check = 1 + +# Lapse evolution +lapse_advec_coeff = 1.0 lapse_coeff = 2.0 -lapse_advec_coeff = 1 # 1 makes the lapse gauge 1+log slicing +lapse_power = 1.0 -# Shift evolution coefficients -shift_advec_coeff = 0 # Usually no advection for beta -shift_Gamma_coeff = 0.75 # -eta = 1.0 # This is gamma driver, usually of order 1/M_ADM of spacetime +# Shift evolution +shift_advec_coeff = 0.0 # Usually no advection for beta +shift_Gamma_coeff = 0.75 +eta = 1.0 # eta of gamma driver, should be of order ~1/M_ADM of spacetime # CCZ4 parameters -# if using BSSN the kappa values should be zero -formulation = 1 # 1 for BSSN, 0 for CCZ4 -kappa1 = 0 -kappa2 = 0 -kappa3 = 0 +formulation = 1 # 1 for BSSN, 0 for CCZ4 +kappa1 = 0. +kappa2 = 0. +kappa3 = 0. covariantZ4 = 1 # 0: keep kappa1; 1 [default]: replace kappa1 -> kappa1/lapse # coefficient for KO numerical dissipation sigma = 0.3 -# Change the gravitational constant of the Universe! -# Default is 1.0, for standard geometric units -# Here we decouple the evolution so the scalar evolved on the -# metric background without backreaction (this avoids the need -# to solve the constaints) -G_Newton = 0.0 +# min_chi = 1.e-4 +# min_lapse = 1.e-4 -# Scalar field initial data -scalar_amplitude = 0.1 -scalar_width = 5.0 -scalar_mass = 0.2 - -# Kerr data -kerr_mass = 1.0 -kerr_spin = 0.0 +################################################# diff --git a/Source/AMRInterpolator/SurfaceExtraction.hpp b/Source/AMRInterpolator/SurfaceExtraction.hpp index c46dbf636..e060225a7 100644 --- a/Source/AMRInterpolator/SurfaceExtraction.hpp +++ b/Source/AMRInterpolator/SurfaceExtraction.hpp @@ -13,6 +13,7 @@ // Other inclues #include "AMRInterpolator.hpp" #include "DimensionDefinitions.hpp" +#include "FilesystemTools.hpp" #include "IntegrationMethod.hpp" #include "InterpolationQuery.hpp" #include "Lagrange.hpp" @@ -50,6 +51,9 @@ template class SurfaceExtraction //!< extraction for each surface bool write_extraction; //!< whether or not to write the extracted data + std::string data_path, integral_file_prefix; + std::string extraction_path, extraction_file_prefix; + int min_extraction_level() { return *(std::min_element(extraction_levels.begin(), diff --git a/Source/AMRInterpolator/SurfaceExtraction.impl.hpp b/Source/AMRInterpolator/SurfaceExtraction.impl.hpp index 16eb7e951..5d6c75037 100644 --- a/Source/AMRInterpolator/SurfaceExtraction.impl.hpp +++ b/Source/AMRInterpolator/SurfaceExtraction.impl.hpp @@ -25,6 +25,18 @@ SurfaceExtraction::SurfaceExtraction( m_du(m_geom.du(m_params.num_points_u)), m_dv(m_geom.dv(m_params.num_points_v)), m_done_extraction(false) { + // check folders only in first two timesteps + // (or at m_first_step if this is not the first two timesteps) + if (m_time < m_restart_time + 1.5 * m_dt || m_first_step) + { + if (!FilesystemTools::directory_exists(m_params.data_path)) + FilesystemTools::mkdir_recursive(m_params.data_path); + + if (m_params.write_extraction && + !FilesystemTools::directory_exists(m_params.extraction_path)) + FilesystemTools::mkdir_recursive(m_params.extraction_path); + } + // only interp points on rank 0 if (procID() == 0) { @@ -330,7 +342,8 @@ void SurfaceExtraction::write_extraction( CH_assert(m_done_extraction); if (procID() == 0) { - SmallDataIO extraction_file(a_file_prefix, m_dt, m_time, m_restart_time, + SmallDataIO extraction_file(m_params.extraction_path + a_file_prefix, + m_dt, m_time, m_restart_time, SmallDataIO::NEW, m_first_step); for (int isurface = 0; isurface < m_params.num_surfaces; ++isurface) @@ -415,8 +428,9 @@ void SurfaceExtraction::write_integrals( CH_assert(vect.size() == m_params.num_surfaces); } // open file for writing - SmallDataIO integral_file(a_filename, m_dt, m_time, m_restart_time, - SmallDataIO::APPEND, m_first_step); + SmallDataIO integral_file(m_params.data_path + a_filename, m_dt, m_time, + m_restart_time, SmallDataIO::APPEND, + m_first_step); // remove any duplicate data if this is a restart integral_file.remove_duplicate_time_data(); diff --git a/Source/BlackHoles/PunctureTracker.cpp b/Source/BlackHoles/PunctureTracker.cpp index 769af2996..804c6a10f 100644 --- a/Source/BlackHoles/PunctureTracker.cpp +++ b/Source/BlackHoles/PunctureTracker.cpp @@ -13,9 +13,13 @@ //! Set punctures post restart void PunctureTracker::initial_setup( const std::vector> &initial_puncture_coords, - const std::string &a_checkpoint_prefix, const int a_min_level) + const std::string &a_filename, const std::string &a_output_path, + const int a_min_level) { - m_punctures_filename = a_checkpoint_prefix + "Punctures"; + if (!FilesystemTools::directory_exists(a_output_path)) + FilesystemTools::mkdir_recursive(a_output_path); + + m_punctures_filename = a_output_path + a_filename; // first set the puncture data // m_num_punctures is only set later diff --git a/Source/BlackHoles/PunctureTracker.hpp b/Source/BlackHoles/PunctureTracker.hpp index ca5d175c1..8d79b2c40 100644 --- a/Source/BlackHoles/PunctureTracker.hpp +++ b/Source/BlackHoles/PunctureTracker.hpp @@ -36,7 +36,8 @@ class PunctureTracker //! if the puncture locations are required for Tagging Criteria void initial_setup(const std::vector> &initial_puncture_coords, - const std::string &a_checkpoint_prefix, + const std::string &a_filename = "punctures", + const std::string &a_output_path = "", const int a_min_level = 0); //! set puncture locations on start (or restart) diff --git a/Source/GRChomboCore/ChomboParameters.hpp b/Source/GRChomboCore/ChomboParameters.hpp index e4dc7ff73..600884da2 100644 --- a/Source/GRChomboCore/ChomboParameters.hpp +++ b/Source/GRChomboCore/ChomboParameters.hpp @@ -13,10 +13,11 @@ // General includes #include "ArrayTools.hpp" #include "BoundaryConditions.hpp" +#include "FilesystemTools.hpp" #include "GRParmParse.hpp" #include "UserVariables.hpp" #include "VariableType.hpp" -#include "unistd.h" +#include "unistd.h" // gives 'access' #include #include @@ -34,6 +35,9 @@ class ChomboParameters void read_params(GRParmParse &pp) { + // must be before any pout() in the code to setPoutBaseName + read_filesystem_params(pp); + pp.load("verbosity", verbosity, 0); // Grid setup pp.load("max_spatial_derivative_order", max_spatial_derivative_order, @@ -51,12 +55,6 @@ class ChomboParameters // L's, N's and center read_grid_params(pp); - // Misc - restart_from_checkpoint = pp.contains("restart_file"); - if (restart_from_checkpoint) - { - pp.load("restart_file", restart_file); - } pp.load("ignore_checkpoint_name_mismatch", ignore_checkpoint_name_mismatch, false); @@ -92,9 +90,7 @@ class ChomboParameters // time stepping outputs and regrid data pp.load("checkpoint_interval", checkpoint_interval, 1); - pp.load("chk_prefix", checkpoint_prefix); pp.load("plot_interval", plot_interval, 0); - pp.load("plot_prefix", plot_prefix); pp.load("stop_time", stop_time, 1.0); pp.load("max_steps", max_steps, 1000000); pp.load("write_plot_ghosts", write_plot_ghosts, false); @@ -124,6 +120,85 @@ class ChomboParameters if (pp.contains("check_params")) just_check_params = true; + + pp.load("print_progress_only_to_rank_0", print_progress_only_to_rank_0, + false); + } + + void read_filesystem_params(GRParmParse &pp) + { + // In this function, cannot use default value - it may print a 'default + // message' to pout and a 'setPoutBaseName' must happen before + + restart_from_checkpoint = pp.contains("restart_file"); + if (restart_from_checkpoint) + { + pp.load("restart_file", restart_file); + } + + pp.load("chk_prefix", checkpoint_prefix); + pp.load("plot_prefix", plot_prefix); + +#ifdef CH_MPI + // Again, cannot use default value + if (pp.contains("pout_prefix")) + pp.load("pout_prefix", pout_prefix); + else + pout_prefix = "pout"; +#endif + + std::string default_path = ""; + if (pp.contains("output_path")) + pp.load("output_path", output_path); + else + output_path = default_path; + +#ifdef CH_MPI + // user sets the 'subpath', we prepend 'output_path' + if (pp.contains("pout_subpath")) + pp.load("pout_subpath", pout_path); + else + pout_path = default_path; +#endif + +#ifdef CH_USE_HDF5 + // user sets the 'subpath', we prepend 'output_path' + if (pp.contains("hdf5_subpath")) + pp.load("hdf5_subpath", hdf5_path); + else + hdf5_path = default_path; +#endif + + // add backslash to paths + if (!output_path.empty() && output_path.back() != '/') + output_path += "/"; +#ifdef CH_MPI + if (!pout_path.empty() && pout_path.back() != '/') + pout_path += "/"; +#endif +#ifdef CH_USE_HDF5 + if (!hdf5_path.empty() && hdf5_path.back() != '/') + hdf5_path += "/"; +#endif + + if (output_path != "./" && !output_path.empty()) + { +#ifdef CH_MPI + pout_path = output_path + pout_path; +#endif +#ifdef CH_USE_HDF5 + hdf5_path = output_path + hdf5_path; +#endif + } + +#ifdef CH_MPI + // change pout base name! + if (!FilesystemTools::directory_exists(pout_path)) + FilesystemTools::mkdir_recursive(pout_path); + setPoutBaseName(pout_path + pout_prefix); +#endif + + // only create hdf5 directory in setupAMRObject (when it becomes needed) } void read_grid_params(GRParmParse &pp) @@ -228,6 +303,7 @@ class ChomboParameters L = (L_full * max_N) / max_N_full; coarsest_dx = L / max_N; + coarsest_dt = coarsest_dx * dt_multiplier; // grid spacing params dx.fill(coarsest_dx); @@ -347,6 +423,12 @@ class ChomboParameters plot_interval <= 0 || plot_prefix != checkpoint_prefix, "should be different to checkpoint_prefix"); + check_parameter("output_path", output_path, + FilesystemTools::directory_exists(output_path), + "should be a valid directory"); + // pout directory exists - we create it in read_filesystem_params() + // can't check hdf5 directory yet - only created after + if (boundary_params.reflective_boundaries_exist) { for (int ivar = 0; ivar < NUM_VARS; ++ivar) @@ -377,9 +459,10 @@ class ChomboParameters int verbosity; double L; // Physical sidelength of the grid std::array center; // grid center - IntVect ivN; // The number of grid cells in each dimension - double coarsest_dx; // The coarsest resolution - int max_level; // the max number of regriddings to do + IntVect ivN; // The number of grid cells in each dimension + double coarsest_dx, + coarsest_dt; // The coarsest resolution in space and time + int max_level; // the max number of regriddings to do int max_spatial_derivative_order; // The maximum order of the spatial // derivatives - does nothing // in Chombo but can be used in examples @@ -398,7 +481,14 @@ class ChomboParameters int checkpoint_interval, plot_interval; // Steps between outputs int max_grid_size, block_factor; // max and min box sizes double fill_ratio; // determines how fussy the regridding is about tags - std::string checkpoint_prefix, plot_prefix; // naming of files + std::string checkpoint_prefix, plot_prefix, pout_prefix; // naming of files + std::string output_path; // base path to use for all files +#ifdef CH_MPI + std::string pout_path; // base path for pout files +#endif +#ifdef CH_USE_HDF5 + std::string hdf5_path; // base path for pout files +#endif bool write_plot_ghosts; int num_plot_vars; std::vector> @@ -416,6 +506,7 @@ class ChomboParameters // For checking parameters and then exiting rather before instantiating // GRAMR (or child) object bool just_check_params = false; + bool print_progress_only_to_rank_0; protected: // the low and high corners of the domain taking into account reflective BCs diff --git a/Source/GRChomboCore/GRAMRLevel.cpp b/Source/GRChomboCore/GRAMRLevel.cpp index 5f160d664..c0c061784 100644 --- a/Source/GRChomboCore/GRAMRLevel.cpp +++ b/Source/GRChomboCore/GRAMRLevel.cpp @@ -100,17 +100,11 @@ Real GRAMRLevel::advance() { CH_TIME("GRAMRLevel::advance"); - // Work out roughly how fast the evolution is going since restart - double speed = (m_time - m_restart_time) / m_gr_amr.get_walltime(); - - // Get information on number of boxes on this level (helps with better load - // balancing) - const DisjointBoxLayout &level_domain = m_state_new.disjointBoxLayout(); - int nbox = level_domain.dataIterator().size(); - int total_nbox = level_domain.size(); - pout() << "GRAMRLevel::advance level " << m_level << " at time " << m_time - << " (" << speed << " M/hr)" - << ". Boxes on this rank: " << nbox << " / " << total_nbox << endl; + // if 'print_progress_only_to_rank_0', still print if it's level 0 or + // t=restart_time + if (!m_p.print_progress_only_to_rank_0 || (procID() == 0) || + m_time == m_restart_time) + printProgress("GRAMRLevel::advance"); // copy soln to old state to save it m_state_new.copyTo(m_state_new.interval(), m_state_old, @@ -345,6 +339,12 @@ void GRAMRLevel::regrid(const Vector &a_new_grids) m_state_diagnostics.define(level_domain, NUM_DIAGNOSTIC_VARS, iv_ghosts); } + + // if 'print_progress_only_to_rank_0', print progress only on regrids + // (except for rank 0, which kept doing prints) + // print here instead of 'postRegrid' to avoid prints in reverse level order + if (m_p.print_progress_only_to_rank_0 && (procID() != 0)) + printProgress("GRAMRLevel::regrid"); } /// things to do after regridding @@ -989,15 +989,14 @@ double GRAMRLevel::get_dx() const { return m_dx; } bool GRAMRLevel::at_level_timestep_multiple(int a_level) const { - const double coarsest_dt = m_p.coarsest_dx * m_p.dt_multiplier; - double target_dt = coarsest_dt; + double target_dt = m_p.coarsest_dt; for (int ilevel = 0; ilevel < a_level; ++ilevel) { target_dt /= m_p.ref_ratios[ilevel]; } // get difference to nearest multiple of target_dt const double time_remainder = remainder(m_time, target_dt); - return (abs(time_remainder) < m_gr_amr.timeEps() * coarsest_dt); + return (abs(time_remainder) < m_gr_amr.timeEps() * m_p.coarsest_dt); } void GRAMRLevel::fillAllGhosts(const VariableType var_type, @@ -1102,3 +1101,19 @@ void GRAMRLevel::defineExchangeCopier(const DisjointBoxLayout &a_level_grids) IntVect iv_ghosts = m_num_ghosts * IntVect::Unit; m_exchange_copier.exchangeDefine(m_grown_grids, iv_ghosts); } + +void GRAMRLevel::printProgress(const std::string &from) const +{ + // Work out roughly how fast the evolution is going since restart + double speed = (m_time - m_restart_time) / m_gr_amr.get_walltime(); + + // Get information on number of boxes on this level (helps with better + // load balancing) + const DisjointBoxLayout &level_domain = m_state_new.disjointBoxLayout(); + int nbox = level_domain.dataIterator().size(); + int total_nbox = level_domain.size(); + + pout() << from << " level " << m_level << " at time " << m_time << " (" + << speed << " M/hr)" + << ". Boxes on this rank: " << nbox << " / " << total_nbox << endl; +} \ No newline at end of file diff --git a/Source/GRChomboCore/GRAMRLevel.hpp b/Source/GRChomboCore/GRAMRLevel.hpp index 9764ff3ee..cc99d1d9c 100644 --- a/Source/GRChomboCore/GRAMRLevel.hpp +++ b/Source/GRChomboCore/GRAMRLevel.hpp @@ -206,6 +206,8 @@ class GRAMRLevel : public AMRLevel, public InterpSource /// copying ghost cells between boxes virtual void defineExchangeCopier(const DisjointBoxLayout &a_level_domain); + void printProgress(const std::string &from) const; + BoundaryConditions m_boundaries; // the class for implementing BCs GRLevelData m_state_old; //!< the solution at the old time diff --git a/Source/GRChomboCore/SetupFunctions.hpp b/Source/GRChomboCore/SetupFunctions.hpp index 71c2cce05..6ec37cf4b 100644 --- a/Source/GRChomboCore/SetupFunctions.hpp +++ b/Source/GRChomboCore/SetupFunctions.hpp @@ -18,6 +18,7 @@ using std::cerr; using std::endl; #include "ChomboParameters.hpp" #include "DerivativeSetup.hpp" +#include "FilesystemTools.hpp" #include "GRAMR.hpp" #include "GRParmParse.hpp" #include "IntegrationMethodSetup.hpp" @@ -73,12 +74,12 @@ void mainSetup(int argc, char *argv[]) if (rank == 0) { - pout() << " number_procs = " << number_procs << endl; + std::cout << " number_procs = " << number_procs << endl; #ifdef _OPENMP - pout() << " threads = " << omp_get_max_threads() << endl; + std::cout << " threads = " << omp_get_max_threads() << endl; #endif - pout() << " simd width (doubles) = " << simd_traits::simd_len - << endl; + std::cout << " simd width (doubles) = " << simd_traits::simd_len + << endl; } const int required_argc = 2; @@ -129,13 +130,16 @@ void setupAMRObject(GRAMR &gr_amr, AMRLevelFactory &a_factory) gr_amr.gridBufferSize(chombo_params.grid_buffer_size); // set checkpoint and plot intervals and prefixes +#ifdef CH_USE_HDF5 gr_amr.checkpointInterval(chombo_params.checkpoint_interval); - gr_amr.checkpointPrefix(chombo_params.checkpoint_prefix); + gr_amr.checkpointPrefix(chombo_params.hdf5_path + + chombo_params.checkpoint_prefix); if (chombo_params.plot_interval != 0) { gr_amr.plotInterval(chombo_params.plot_interval); - gr_amr.plotPrefix(chombo_params.plot_prefix); + gr_amr.plotPrefix(chombo_params.hdf5_path + chombo_params.plot_prefix); } +#endif // Number of coarse time steps from one regridding to the next gr_amr.regridIntervals(chombo_params.regrid_interval); @@ -159,12 +163,18 @@ void setupAMRObject(GRAMR &gr_amr, AMRLevelFactory &a_factory) // Set up input files if (!chombo_params.restart_from_checkpoint) { +#ifdef CH_USE_HDF5 + if (!FilesystemTools::directory_exists(chombo_params.hdf5_path)) + FilesystemTools::mkdir_recursive(chombo_params.hdf5_path); +#endif + gr_amr.setupForNewAMRRun(); } else { #ifdef CH_USE_HDF5 - HDF5Handle handle(chombo_params.restart_file, HDF5Handle::OPEN_RDONLY); + HDF5Handle handle(chombo_params.hdf5_path + chombo_params.restart_file, + HDF5Handle::OPEN_RDONLY); // read from checkpoint file gr_amr.setupForRestart(handle); handle.close(); diff --git a/Source/GRChomboCore/SimulationParametersBase.hpp b/Source/GRChomboCore/SimulationParametersBase.hpp index 8be50660d..a2889683c 100644 --- a/Source/GRChomboCore/SimulationParametersBase.hpp +++ b/Source/GRChomboCore/SimulationParametersBase.hpp @@ -58,70 +58,106 @@ class SimulationParametersBase : public ChomboParameters pp.load("min_chi", min_chi, 1e-4); pp.load("min_lapse", min_lapse, 1e-4); + // directory to store data (extraction files, puncture data, constraint + // norms) + pp.load("data_subpath", data_path, std::string("")); + if (!data_path.empty() && data_path.back() != '/') + data_path += "/"; + if (output_path != "./" && !output_path.empty()) + data_path = output_path + data_path; + // Extraction params - pp.load("num_extraction_radii", extraction_params.num_extraction_radii, - 1); - // Check for multiple extraction radii, otherwise load single - // radius/level (for backwards compatibility). - if (pp.contains("extraction_levels")) - { - pp.load("extraction_levels", extraction_params.extraction_levels, - extraction_params.num_extraction_radii); - } - else - { - pp.load("extraction_level", extraction_params.extraction_levels, 1, - 0); - } - if (pp.contains("extraction_radii")) - { - pp.load("extraction_radii", extraction_params.extraction_radii, - extraction_params.num_extraction_radii); - } - else - { - pp.load("extraction_radius", extraction_params.extraction_radii, 1, - 0.1); - } + pp.load("activate_extraction", activate_extraction, false); - pp.load("num_points_phi", extraction_params.num_points_phi, 2); - pp.load("num_points_theta", extraction_params.num_points_theta, 5); - if (extraction_params.num_points_theta % 2 == 0) + if (activate_extraction) { - extraction_params.num_points_theta += 1; - pout() << "Parameter: num_points_theta incompatible with Simpson's " - << "rule so increased by 1.\n"; - } - pp.load("extraction_center", extraction_params.center, center); + pp.load("num_extraction_radii", + extraction_params.num_extraction_radii, 1); - if (pp.contains("modes")) - { - pp.load("num_modes", extraction_params.num_modes); - std::vector extraction_modes_vect(2 * - extraction_params.num_modes); - pp.load("modes", extraction_modes_vect, - 2 * extraction_params.num_modes); - extraction_params.modes.resize(extraction_params.num_modes); - for (int i = 0; i < extraction_params.num_modes; ++i) + // Check for multiple extraction radii, otherwise load single + // radius/level (for backwards compatibility). + if (pp.contains("extraction_levels")) { - extraction_params.modes[i].first = extraction_modes_vect[2 * i]; - extraction_params.modes[i].second = - extraction_modes_vect[2 * i + 1]; + pp.load("extraction_levels", + extraction_params.extraction_levels, + extraction_params.num_extraction_radii); } - } - else - { - // by default extraction (l,m) = (2,0), (2,1) and (2,2) - extraction_params.num_modes = 3; - extraction_params.modes.resize(3); - for (int i = 0; i < 3; ++i) + else { - extraction_params.modes[i].first = 2; - extraction_params.modes[i].second = i; + pp.load("extraction_level", extraction_params.extraction_levels, + 1, 0); } - } + if (pp.contains("extraction_radii")) + { + pp.load("extraction_radii", extraction_params.extraction_radii, + extraction_params.num_extraction_radii); + } + else + { + pp.load("extraction_radius", extraction_params.extraction_radii, + 1, 0.1); + } + + pp.load("num_points_phi", extraction_params.num_points_phi, 2); + pp.load("num_points_theta", extraction_params.num_points_theta, 5); + if (extraction_params.num_points_theta % 2 == 0) + { + extraction_params.num_points_theta += 1; + pout() << "Parameter: num_points_theta incompatible with " + "Simpson's " + << "rule so increased by 1.\n"; + } + pp.load("extraction_center", extraction_params.center, center); + + if (pp.contains("modes")) + { + pp.load("num_modes", extraction_params.num_modes); + std::vector extraction_modes_vect( + 2 * extraction_params.num_modes); + pp.load("modes", extraction_modes_vect, + 2 * extraction_params.num_modes); + extraction_params.modes.resize(extraction_params.num_modes); + for (int i = 0; i < extraction_params.num_modes; ++i) + { + extraction_params.modes[i].first = + extraction_modes_vect[2 * i]; + extraction_params.modes[i].second = + extraction_modes_vect[2 * i + 1]; + } + } + else + { + // by default extraction (l,m) = (2,0), (2,1) and (2,2) + extraction_params.num_modes = 3; + extraction_params.modes.resize(3); + for (int i = 0; i < 3; ++i) + { + extraction_params.modes[i].first = 2; + extraction_params.modes[i].second = i; + } + } + + pp.load("write_extraction", extraction_params.write_extraction, + false); + + std::string extraction_path; + pp.load("extraction_subpath", extraction_path, data_path); + if (!extraction_path.empty() && extraction_path.back() != '/') + extraction_path += "/"; + if (output_path != "./" && !output_path.empty()) + extraction_path = output_path + extraction_path; - pp.load("write_extraction", extraction_params.write_extraction, false); + extraction_params.data_path = data_path; + extraction_params.extraction_path = extraction_path; + + // default names to Weyl extraction + pp.load("extraction_file_prefix", + extraction_params.extraction_file_prefix, + std::string("Weyl4_extraction_")); + pp.load("integral_file_prefix", + extraction_params.integral_file_prefix, + std::string("Weyl4_mode_")); + } } void check_params() @@ -204,43 +240,57 @@ class SimulationParametersBase : public ChomboParameters "usually O(1/M_ADM) so typically O(1) in code units"); // Now extraction parameters - FOR1(idir) + if (activate_extraction) { - std::string center_name = - "extraction_center[" + std::to_string(idir) + "]"; - double center_in_dir = extraction_params.center[idir]; - check_parameter(center_name, center_in_dir, - (center_in_dir >= reflective_domain_lo[idir]) && - (center_in_dir <= reflective_domain_hi[idir]), - "must be in the computational domain after " - "applying reflective symmetry"); - for (int iradius = 0; - iradius < extraction_params.num_extraction_radii; ++iradius) + check_parameter( + "num_extraction_radii", extraction_params.num_extraction_radii, + extraction_params.num_extraction_radii > 0, + "must be bigger than 0 when activate_extraction = 1"); + + FOR1(idir) { - std::string radius_name = - "extraction_radii[" + std::to_string(iradius) + "]"; - double radius = extraction_params.extraction_radii[iradius]; - if (idir == 0) - check_parameter(radius_name, radius, radius >= 0.0, - "must be >= 0.0"); + std::string center_name = + "extraction_center[" + std::to_string(idir) + "]"; + double center_in_dir = extraction_params.center[idir]; check_parameter( - radius_name, radius, - (center_in_dir - radius >= reflective_domain_lo[idir]) && - (center_in_dir + radius <= reflective_domain_hi[idir]), - "extraction sphere must lie within the computational " - "domain after applying reflective symmetry"); + center_name, center_in_dir, + (center_in_dir >= reflective_domain_lo[idir]) && + (center_in_dir <= reflective_domain_hi[idir]), + "must be in the computational domain after " + "applying reflective symmetry"); + for (int iradius = 0; + iradius < extraction_params.num_extraction_radii; + ++iradius) + { + std::string radius_name = + "extraction_radii[" + std::to_string(iradius) + "]"; + double radius = extraction_params.extraction_radii[iradius]; + if (idir == 0) + check_parameter(radius_name, radius, radius >= 0.0, + "must be >= 0.0"); + check_parameter( + radius_name, radius, + (center_in_dir - radius >= + reflective_domain_lo[idir]) && + (center_in_dir + radius <= + reflective_domain_hi[idir]), + "extraction sphere must lie within the computational " + "domain after applying reflective symmetry"); + } + } + for (int imode = 0; imode < extraction_params.num_modes; ++imode) + { + auto &mode = extraction_params.modes[imode]; + int l = mode.first; + int m = mode.second; + std::string mode_name = "modes[" + std::to_string(imode) + "]"; + std::string value_str = "(" + std::to_string(mode.first) + + ", " + std::to_string(mode.second) + + ")"; + check_parameter( + mode_name, value_str, (l >= 2) && (abs(m) <= l), + "l must be >= 2 and m must satisfy -l <= m <= l"); } - } - for (int imode = 0; imode < extraction_params.num_modes; ++imode) - { - auto &mode = extraction_params.modes[imode]; - int l = mode.first; - int m = mode.second; - std::string mode_name = "modes[" + std::to_string(imode) + "]"; - std::string value_str = "(" + std::to_string(mode.first) + ", " + - std::to_string(mode.second) + ")"; - check_parameter(mode_name, value_str, (l >= 2) && (abs(m) <= l), - "l must be >= 2 and m must satisfy -l <= m <= l"); } } @@ -258,12 +308,16 @@ class SimulationParametersBase : public ChomboParameters int formulation; // Whether to use BSSN or CCZ4 - // Collection of parameters necessary for the CCZ4 RHS and extraction + // Collection of parameters necessary for the CCZ4 RHS // Note the gauge parameters are specific to MovingPunctureGauge // If you are using a different gauge, you need to load your parameters // in your own SimulationParameters class. CCZ4_params_t<> ccz4_params; + + bool activate_extraction; SphericalExtraction::params_t extraction_params; + + std::string data_path; }; #endif /* SIMULATIONPARAMETERSBASE_HPP_ */ diff --git a/Source/TaggingCriteria/FixedGridsTaggingCriterion.hpp b/Source/TaggingCriteria/FixedGridsTaggingCriterion.hpp index 90b94f3cf..905c134ff 100644 --- a/Source/TaggingCriteria/FixedGridsTaggingCriterion.hpp +++ b/Source/TaggingCriteria/FixedGridsTaggingCriterion.hpp @@ -23,7 +23,7 @@ class FixedGridsTaggingCriterion FixedGridsTaggingCriterion(const double dx, const int a_level, const double a_L, const std::array a_center) - : m_dx(dx), m_level(a_level), m_L(a_L), m_center(a_center){}; + : m_dx(dx), m_L(a_L), m_level(a_level), m_center(a_center){}; template void compute(Cell current_cell) const { diff --git a/Source/utils/FilesystemTools.hpp b/Source/utils/FilesystemTools.hpp new file mode 100644 index 000000000..86a3f5ca1 --- /dev/null +++ b/Source/utils/FilesystemTools.hpp @@ -0,0 +1,84 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef FILESYSTEMTOOLS_HPP_ +#define FILESYSTEMTOOLS_HPP_ + +// Chombo includes +#include "SPMD.H" // gives procID() + +// Other includes +#include // gives 'stat' and 'S_ISDIR' +#include // gives 'mkdir' + +// Chombo namespace +#include "UsingNamespace.H" + +// Some filesystem useful functions + +// This should be changed in the future by the actual C++17 filesystem library +// when we decide to only support from C++17 + +namespace FilesystemTools +{ + +static bool directory_exists(const std::string &path) +{ + struct stat stat_struct; + return (path == "") || (stat(path.c_str(), &stat_struct) == 0 && + S_ISDIR(stat_struct.st_mode)); +} + +static bool mkdir_recursive(const std::string &path) +{ +#ifdef CH_MPI + // all processes should get here at the same time + // e.g. if doing: + // if (!directory_exists(path)) + // mkdir_recursive(path); + // We want to make sure the directory is not created until all + // ranks have passed the 'directory_exists' + MPI_Barrier(Chombo_MPI::comm); +#endif + + bool success = true; + if (procID() == 0) + { + // For Windows backslash, use delimeters = "/\\" + static const std::string delimeter = "/"; + std::size_t found = path.find_first_of(delimeter); + + // NB: this would be very beautiful recursively, but let's not do it + // because the function involves MPI_Barrier's + while (success && found != std::string::npos) + { + std::string subpath; + subpath = path.substr(0, found); + + if (subpath != "." && subpath != "") + { + // success if created or if not created because it already + // exists + success &= + (mkdir(subpath.c_str(), 0700) == 0 || errno == EEXIST); + } + found = path.find_first_of(delimeter, found + 1); + } + // if path doesn't finish with "/", one more to do + if (found != path.size() - 1) + success &= (mkdir(path.c_str(), 0700) == 0 || errno == EEXIST); + } + +#ifdef CH_MPI + // all processes should wait to make sure directory structure is well set + // for everyone + MPI_Barrier(Chombo_MPI::comm); +#endif + + return success; +} +} // namespace FilesystemTools + +#endif /* FILESYSTEMTOOLS_HPP_ */ diff --git a/Source/utils/WeylExtraction.hpp b/Source/utils/WeylExtraction.hpp index 64b809e69..670effcae 100644 --- a/Source/utils/WeylExtraction.hpp +++ b/Source/utils/WeylExtraction.hpp @@ -45,9 +45,7 @@ class WeylExtraction : public SphericalExtraction extract(a_interpolator); if (m_params.write_extraction) - { - write_extraction("Weyl4ExtractionOut_"); - } + write_extraction(m_params.extraction_file_prefix); // now calculate and write the requested spherical harmonic modes std::vector, std::vector>> @@ -79,7 +77,7 @@ class WeylExtraction : public SphericalExtraction for (int imode = 0; imode < m_num_modes; ++imode) { const auto &mode = m_modes[imode]; - std::string integrals_filename = "Weyl_integral_" + + std::string integrals_filename = m_params.integral_file_prefix + std::to_string(mode.first) + std::to_string(mode.second); std::vector> integrals_for_writing = {