From 07d4882c58429889d9dfb841a6cc8feb9c1faffd Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Tue, 19 Jan 2021 10:15:35 +0100 Subject: [PATCH 01/92] New AHFinder for public code + 2D GRChombo + improved Interpolator classes --- Examples/BinaryBH/BinaryBHLevel.cpp | 13 +- Examples/BinaryBH/GNUmakefile | 1 + Examples/BinaryBH/Main_BinaryBH.cpp | 14 + Examples/BinaryBH/SimulationParameters.hpp | 12 + Examples/BinaryBH/params.txt | 30 + Examples/BinaryBH/params_expensive.txt | 79 - Examples/BinaryBH/params_very_cheap.txt | 13 +- Examples/KerrBH/GNUmakefile | 4 +- Examples/KerrBH/KerrBHLevel.cpp | 34 +- Examples/KerrBH/KerrBHLevel.hpp | 5 + Examples/KerrBH/Main_KerrBH.cpp | 41 +- Examples/KerrBH/SimulationParameters.hpp | 17 + Examples/KerrBH/UserVariables.hpp | 8 + Examples/KerrBH/params.txt | 18 + Examples/KerrBH/params_cheap.txt | 39 +- Examples/ScalarField/GNUmakefile | 4 +- Examples/ScalarField/Main_ScalarField.cpp | 40 +- Examples/ScalarField/ScalarFieldLevel.cpp | 12 +- Examples/ScalarField/ScalarFieldLevel.hpp | 6 + Examples/ScalarField/SimulationParameters.hpp | 10 + Examples/ScalarField/params.txt | 19 + Examples/SingleBH/DiagnosticVariables.hpp | 29 + Examples/SingleBH/GNUmakefile | 26 + Examples/SingleBH/Main_SingleBH.cpp | 82 + Examples/SingleBH/SimulationParameters.hpp | 55 + Examples/SingleBH/SingleBHLevel.cpp | 88 + Examples/SingleBH/SingleBHLevel.hpp | 49 + Examples/SingleBH/UserVariables.hpp | 29 + Examples/SingleBH/params.txt | 106 + .../Cartesius.Make.defs.local | 1 - .../MakeDefsLocalExamples/MN4.Make.defs.local | 1 - .../athena.Make.defs.local | 2 +- .../cosmos.Make.defs.local | 4 +- .../docker_hpc-base-stretch.Make.defs.local | 4 +- .../gwdg.Make.defs.local | 1 - .../nesi-pan.Make.defs.local | 1 - .../ubuntu-18.04.Make.defs.local | 2 +- Source/AMRInterpolator/AMRInterpolator.hpp | 7 +- .../AMRInterpolator/AMRInterpolator.impl.hpp | 67 +- .../AMRInterpolator/CylindricalGeometry.hpp | 38 +- Source/AMRInterpolator/Derivative.hpp | 18 +- Source/AMRInterpolator/DerivativeSetup.hpp | 8 +- Source/AMRInterpolator/InterpSource.hpp | 5 +- Source/AMRInterpolator/Lagrange.hpp | 28 +- Source/AMRInterpolator/Lagrange.impl.hpp | 79 +- Source/AMRInterpolator/QuinticConvolution.hpp | 23 +- .../QuinticConvolution.impl.hpp | 49 +- Source/AMRInterpolator/SimpleArrayBox.hpp | 75 + Source/AMRInterpolator/SimpleInterpSource.hpp | 46 + Source/AMRInterpolator/SphericalGeometry.hpp | 64 +- Source/ApparentHorizonFinder/AHData.hpp | 88 + Source/ApparentHorizonFinder/AHDeriv.hpp | 48 + Source/ApparentHorizonFinder/AHFinder.cpp | 540 ++++++ Source/ApparentHorizonFinder/AHFinder.hpp | 233 +++ .../AHFunctionDefault.hpp | 56 + Source/ApparentHorizonFinder/AHFunctions.hpp | 327 ++++ .../ApparentHorizonFinder/AHGeometryData.hpp | 51 + .../ApparentHorizonFinder/AHInterpolation.hpp | 138 ++ .../AHInterpolation.impl.hpp | 517 +++++ .../AHSphericalGeometry.hpp | 138 ++ .../AHStringGeometry.hpp | 109 ++ .../ApparentHorizonFinder/ApparentHorizon.hpp | 296 +++ .../ApparentHorizon.impl.hpp | 1712 +++++++++++++++++ .../ApparentHorizon_petsc.impl.hpp | 959 +++++++++ Source/BlackHoles/BHAMR.hpp | 17 + Source/BlackHoles/PunctureTracker.cpp | 4 + Source/BoxUtils/BoxLoops.impl.hpp | 9 +- Source/BoxUtils/BoxPointers.hpp | 12 +- Source/BoxUtils/NanCheck.hpp | 28 +- Source/CCZ4/ADMConformalVars.hpp | 38 + Source/CCZ4/BSSNVars.hpp | 19 + Source/GRChomboCore/BoundaryConditions.cpp | 27 +- Source/GRChomboCore/GRAMR.cpp | 2 + Source/GRChomboCore/GRAMR.hpp | 3 + Source/GRChomboCore/GRAMRLevel.cpp | 39 +- Source/GRChomboCore/GRAMRLevel.hpp | 2 +- .../GRChomboCore/SimulationParametersBase.hpp | 26 +- .../InitialConditions/BlackHoles/SingleBH.hpp | 50 + .../BlackHoles/SingleBH.impl.hpp | 71 + Source/utils/Coordinates.hpp | 22 +- Source/utils/SmallDataIO.cpp | 130 +- Source/utils/SmallDataIO.hpp | 8 +- Source/utils/TensorAlgebra.hpp | 78 +- Source/utils/VarsTools.hpp | 28 +- .../AMRInterpolatorTest.cpp | 4 +- .../AHTest2DFunction.hpp | 38 + .../ApparentHorizonTest2D.cpp | 127 ++ .../ApparentHorizonTest2D.inputs | 25 + .../ApparentHorizonTest2DLevel.hpp | 68 + Tests/ApparentHorizonFinderTest2D/GNUmakefile | 28 + .../SimulationParameters.hpp | 18 + .../UserVariables.hpp | 35 + .../ApparentHorizonTest3D.cpp | 127 ++ .../ApparentHorizonTest3D.inputs | 36 + .../ApparentHorizonTest3DLevel.hpp | 97 + Tests/ApparentHorizonFinderTest3D/GNUmakefile | 27 + .../SimulationParameters.hpp | 41 + .../UserVariables.hpp | 29 + Tests/InterpolatorTest/GNUmakefile | 20 + Tests/InterpolatorTest/InterpolatorTest.cpp | 115 ++ .../InterpolatorTest/InterpolatorTest.inputs | 1 + .../InterpolatorTest/SimulationParameters.hpp | 19 + Tests/InterpolatorTest/UserVariables.hpp | 26 + 103 files changed, 7739 insertions(+), 373 deletions(-) delete mode 100644 Examples/BinaryBH/params_expensive.txt create mode 100644 Examples/SingleBH/DiagnosticVariables.hpp create mode 100644 Examples/SingleBH/GNUmakefile create mode 100644 Examples/SingleBH/Main_SingleBH.cpp create mode 100644 Examples/SingleBH/SimulationParameters.hpp create mode 100644 Examples/SingleBH/SingleBHLevel.cpp create mode 100644 Examples/SingleBH/SingleBHLevel.hpp create mode 100644 Examples/SingleBH/UserVariables.hpp create mode 100644 Examples/SingleBH/params.txt mode change 100644 => 100755 Source/AMRInterpolator/DerivativeSetup.hpp create mode 100644 Source/AMRInterpolator/SimpleArrayBox.hpp create mode 100644 Source/AMRInterpolator/SimpleInterpSource.hpp create mode 100644 Source/ApparentHorizonFinder/AHData.hpp create mode 100644 Source/ApparentHorizonFinder/AHDeriv.hpp create mode 100644 Source/ApparentHorizonFinder/AHFinder.cpp create mode 100644 Source/ApparentHorizonFinder/AHFinder.hpp create mode 100644 Source/ApparentHorizonFinder/AHFunctionDefault.hpp create mode 100644 Source/ApparentHorizonFinder/AHFunctions.hpp create mode 100644 Source/ApparentHorizonFinder/AHGeometryData.hpp create mode 100644 Source/ApparentHorizonFinder/AHInterpolation.hpp create mode 100644 Source/ApparentHorizonFinder/AHInterpolation.impl.hpp create mode 100644 Source/ApparentHorizonFinder/AHSphericalGeometry.hpp create mode 100644 Source/ApparentHorizonFinder/AHStringGeometry.hpp create mode 100644 Source/ApparentHorizonFinder/ApparentHorizon.hpp create mode 100644 Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp create mode 100644 Source/ApparentHorizonFinder/ApparentHorizon_petsc.impl.hpp create mode 100644 Source/InitialConditions/BlackHoles/SingleBH.hpp create mode 100644 Source/InitialConditions/BlackHoles/SingleBH.impl.hpp create mode 100644 Tests/ApparentHorizonFinderTest2D/AHTest2DFunction.hpp create mode 100644 Tests/ApparentHorizonFinderTest2D/ApparentHorizonTest2D.cpp create mode 100644 Tests/ApparentHorizonFinderTest2D/ApparentHorizonTest2D.inputs create mode 100644 Tests/ApparentHorizonFinderTest2D/ApparentHorizonTest2DLevel.hpp create mode 100644 Tests/ApparentHorizonFinderTest2D/GNUmakefile create mode 100755 Tests/ApparentHorizonFinderTest2D/SimulationParameters.hpp create mode 100644 Tests/ApparentHorizonFinderTest2D/UserVariables.hpp create mode 100755 Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.cpp create mode 100755 Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.inputs create mode 100755 Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3DLevel.hpp create mode 100755 Tests/ApparentHorizonFinderTest3D/GNUmakefile create mode 100755 Tests/ApparentHorizonFinderTest3D/SimulationParameters.hpp create mode 100755 Tests/ApparentHorizonFinderTest3D/UserVariables.hpp create mode 100644 Tests/InterpolatorTest/GNUmakefile create mode 100644 Tests/InterpolatorTest/InterpolatorTest.cpp create mode 100644 Tests/InterpolatorTest/InterpolatorTest.inputs create mode 100644 Tests/InterpolatorTest/SimulationParameters.hpp create mode 100644 Tests/InterpolatorTest/UserVariables.hpp diff --git a/Examples/BinaryBH/BinaryBHLevel.cpp b/Examples/BinaryBH/BinaryBHLevel.cpp index ebbc16516..1e25b6e6d 100644 --- a/Examples/BinaryBH/BinaryBHLevel.cpp +++ b/Examples/BinaryBH/BinaryBHLevel.cpp @@ -30,8 +30,9 @@ void BinaryBHLevel::specificAdvance() // Check for nan's if (m_p.nan_check) - BoxLoops::loop(NanCheck("NaNCheck in specific Advance: "), m_state_new, - m_state_new, EXCLUDE_GHOST_CELLS, disable_simd()); + BoxLoops::loop( + NanCheck(m_dx, m_p.center, "NaNCheck in specific Advance"), + m_state_new, m_state_new, EXCLUDE_GHOST_CELLS, disable_simd()); } // This initial data uses an approximation for the metric which @@ -59,8 +60,7 @@ void BinaryBHLevel::specificEvalRHS(GRLevelData &a_soln, GRLevelData &a_rhs, BoxLoops::loop(make_compute_pack(TraceARemoval(), PositiveChiAndAlpha()), a_soln, a_soln, INCLUDE_GHOST_CELLS); - // Calculate CCZ4 right hand side and set constraints to zero to avoid - // undefined values + // Calculate CCZ4 right hand side BoxLoops::loop(CCZ4(m_p.ccz4_params, m_dx, m_p.sigma, m_p.formulation), a_soln, a_rhs, EXCLUDE_GHOST_CELLS); } @@ -161,6 +161,11 @@ void BinaryBHLevel::specificPostTimeStep() m_bh_amr.m_puncture_tracker.execute_tracking(m_time, m_restart_time, m_dt, write_punctures); } + +#ifdef USE_AHFINDER + if (m_p.AH_activate && m_level == m_p.AH_params.level_to_run) + m_bh_amr.m_ah_finder.solve(m_dt, m_time, m_restart_time); +#endif } // Things to do before a plot level - need to calculate the Weyl scalars diff --git a/Examples/BinaryBH/GNUmakefile b/Examples/BinaryBH/GNUmakefile index c6e70b9fd..2dc7a98e2 100644 --- a/Examples/BinaryBH/GNUmakefile +++ b/Examples/BinaryBH/GNUmakefile @@ -19,6 +19,7 @@ src_dirs := $(GRCHOMBO_SOURCE)/utils \ $(GRCHOMBO_SOURCE)/GRChomboCore \ $(GRCHOMBO_SOURCE)/AMRInterpolator \ $(GRCHOMBO_SOURCE)/TaggingCriteria \ + $(GRCHOMBO_SOURCE)/ApparentHorizonFinder \ $(GRCHOMBO_SOURCE)/InitialConditions/BlackHoles \ $(GRCHOMBO_SOURCE)/BlackHoles diff --git a/Examples/BinaryBH/Main_BinaryBH.cpp b/Examples/BinaryBH/Main_BinaryBH.cpp index 78737d2d8..eced1fd2e 100644 --- a/Examples/BinaryBH/Main_BinaryBH.cpp +++ b/Examples/BinaryBH/Main_BinaryBH.cpp @@ -62,6 +62,20 @@ int runGRChombo(int argc, char *argv[]) if (sim_params.track_punctures) bh_amr.m_puncture_tracker.restart_punctures(); +#ifdef USE_AHFINDER + if (sim_params.AH_activate) + { + AHSphericalGeometry sph1(sim_params.bh1_params.center); + AHSphericalGeometry sph2(sim_params.bh2_params.center); + + bh_amr.m_ah_finder.add_ah(sph1, sim_params.AH_1_initial_guess, + sim_params.AH_params); + bh_amr.m_ah_finder.add_ah(sph2, sim_params.AH_2_initial_guess, + sim_params.AH_params); + bh_amr.m_ah_finder.add_ah_merger(0, 1, sim_params.AH_params); + } +#endif + using Clock = std::chrono::steady_clock; using Minutes = std::chrono::duration>; diff --git a/Examples/BinaryBH/SimulationParameters.hpp b/Examples/BinaryBH/SimulationParameters.hpp index bb3ef7341..51b1a7010 100644 --- a/Examples/BinaryBH/SimulationParameters.hpp +++ b/Examples/BinaryBH/SimulationParameters.hpp @@ -54,6 +54,13 @@ class SimulationParameters : public SimulationParametersBase pp.load("puncture_tracking_level", puncture_tracking_level, max_level); pp.load("calculate_constraint_norms", calculate_constraint_norms, false); + +#ifdef USE_AHFINDER + pp.load("AH_1_initial_guess", AH_1_initial_guess, + 0.5 * bh1_params.mass); + pp.load("AH_2_initial_guess", AH_2_initial_guess, + 0.5 * bh2_params.mass); +#endif } void check_params() @@ -99,6 +106,11 @@ class SimulationParameters : public SimulationParametersBase // Collection of parameters necessary for initial conditions BoostedBH::params_t bh2_params; BoostedBH::params_t bh1_params; + +#ifdef USE_AHFINDER + double AH_1_initial_guess; + double AH_2_initial_guess; +#endif }; #endif /* SIMULATIONPARAMETERS_HPP */ diff --git a/Examples/BinaryBH/params.txt b/Examples/BinaryBH/params.txt index 4b1d78148..2fc612b61 100644 --- a/Examples/BinaryBH/params.txt +++ b/Examples/BinaryBH/params.txt @@ -120,3 +120,33 @@ modes = 2 0 # l m for spherical harmonics 4 2 4 3 4 4 + + +#Apparent Horizon finder +AH_activate = 1 +AH_num_ranks = 20 +AH_num_points_u = 30 +AH_num_points_v = 50 +#AH_solve_interval = 1 +#AH_print_interval = 1 +#AH_track_center = true +#AH_predict_origin = true +#AH_merger_search_factor = 1. +#AH_merger_pre_factor = 1. +#AH_level_to_run = 0 +#AH_start_time = 0. +#AH_give_up_time = -1. +#AH_merger_search_factor = 1. +#AH_merger_pre_factor = 1. +#AH_allow_re_attempt = 0 +#AH_max_fails_after_lost = -1 +#AH_verbose = 1 +#AH_print_geometry_data = 0 +#AH_re_solve_at_restart = 0 +#AH_stop_if_max_fails = 0 + +#AH_1_initial_guess = 0.3 +#AH_2_initial_guess = 0.3 + +AH_num_extra_vars = 2 +AH_extra_vars = chi d1_Ham d2_A11 diff --git a/Examples/BinaryBH/params_expensive.txt b/Examples/BinaryBH/params_expensive.txt deleted file mode 100644 index 7bd1ae30d..000000000 --- a/Examples/BinaryBH/params_expensive.txt +++ /dev/null @@ -1,79 +0,0 @@ -verbosity = 0 -chk_prefix = BinaryBH_ -plot_prefix = BinaryBHPlot_ -#restart_file = BinaryBH_000360.3d.hdf5 - -# Set up grid spacings and regrid params -# NB - the N values need to be multiples of block_factor - -N1 = 256 -N2 = 256 -N3 = 256 -L = 192 - -massA = 0.5 -offsetA = 3 0 0 - -massB = 0.5 -offsetB = -3 0 0 - -momentumA = 0. -0.1 0.0 -momentumB = 0. 0.1 0.0 - -regrid_threshold = 0.20 -max_level = 5 -regrid_interval = 512 256 32 16 8 4 2 2 2 -isPeriodic = 1 1 1 - -#Max and min box sizes -max_grid_size = 32 -block_factor = 16 -tag_buffer_size = 3 - -# Set up time steps -# dt will be dx*dt_multiplier on each grid level -# HDF5files are written every dt = L/N*dt_multiplier*checkpoint_interval - -checkpoint_interval = 4 -# set to zero to turn off plot files -plot_interval = 0 -num_plot_vars = 3 -plot_vars = chi Weyl4_Re Weyl4_Im -dt_multiplier = 0.25 -stop_time = 1.0 -max_steps = 2 - -nan_check = 1 - -#Lapse evolution -lapse_advec_coeff = 0.0 -lapse_coeff = 2.0 -lapse_power = 1.0 - -# Shift evolution -shift_advec_coeff = 0.0 -shift_Gamma_coeff = 0.75 -eta = 1.82 - -# CCZ4 parameters -kappa1 = 0.1 -kappa2 = 0 -kappa3 = 1. -covariantZ4 = 1 # 0: default. 1: dampk1 -> dampk1/lapse - -#coefficient for KO numerical dissipation -sigma = 0.3 - -#extraction params -#default center to grid center, uncomment to change -#extraction_center = 64 64 64 -activate_extraction = 1 -num_extraction_radii = 3 -extraction_radii = 10. 30. 50. -extraction_levels = 2 1 1 -num_points_phi = 16 -num_points_theta = 24 -num_modes = 3 -modes = 2 0 # l m for spherical harmonics - 2 1 - 2 2 diff --git a/Examples/BinaryBH/params_very_cheap.txt b/Examples/BinaryBH/params_very_cheap.txt index c7cc88cf4..6cff37641 100644 --- a/Examples/BinaryBH/params_very_cheap.txt +++ b/Examples/BinaryBH/params_very_cheap.txt @@ -33,11 +33,11 @@ tag_buffer_size = 3 checkpoint_interval = 1 # set to zero to turn off plot files -plot_interval = 1 +plot_interval = 2 num_plot_vars = 4 plot_vars = chi Ham Weyl4_Re Weyl4_Im dt_multiplier = 0.25 -stop_time = 10.0 +stop_time = 2.0 nan_check = 1 @@ -104,3 +104,12 @@ modes = 2 0 # l m for spherical harmonics 2 1 2 2 +#Apparent Horizon finder +AH_activate = 1 +AH_num_ranks = 2 +AH_num_points_u = 11 +AH_num_points_v = 10 +AH_solve_interval = 2 +AH_print_interval = 2 +#AH_merger_search_factor = 1 + diff --git a/Examples/KerrBH/GNUmakefile b/Examples/KerrBH/GNUmakefile index d1176165c..226cef97d 100644 --- a/Examples/KerrBH/GNUmakefile +++ b/Examples/KerrBH/GNUmakefile @@ -18,7 +18,9 @@ src_dirs := $(GRCHOMBO_SOURCE)/utils \ $(GRCHOMBO_SOURCE)/BoxUtils \ $(GRCHOMBO_SOURCE)/GRChomboCore \ $(GRCHOMBO_SOURCE)/TaggingCriteria \ + $(GRCHOMBO_SOURCE)/InitialConditions/BlackHoles \ + $(GRCHOMBO_SOURCE)/BlackHoles \ $(GRCHOMBO_SOURCE)/AMRInterpolator \ - $(GRCHOMBO_SOURCE)/InitialConditions/BlackHoles + $(GRCHOMBO_SOURCE)/ApparentHorizonFinder include $(CHOMBO_HOME)/mk/Make.test diff --git a/Examples/KerrBH/KerrBHLevel.cpp b/Examples/KerrBH/KerrBHLevel.cpp index 8d2898f79..35cbb5b85 100644 --- a/Examples/KerrBH/KerrBHLevel.cpp +++ b/Examples/KerrBH/KerrBHLevel.cpp @@ -27,8 +27,9 @@ void KerrBHLevel::specificAdvance() // Check for nan's if (m_p.nan_check) - BoxLoops::loop(NanCheck(), m_state_new, m_state_new, - EXCLUDE_GHOST_CELLS, disable_simd()); + BoxLoops::loop( + NanCheck(m_dx, m_p.center, "NaNCheck in specific Advance"), + m_state_new, m_state_new, EXCLUDE_GHOST_CELLS, disable_simd()); } void KerrBHLevel::initialData() @@ -47,10 +48,20 @@ void KerrBHLevel::initialData() fillAllGhosts(); BoxLoops::loop(GammaCalculator(m_dx), m_state_new, m_state_new, EXCLUDE_GHOST_CELLS); + + // Diagnostics needed for AHFinder + BoxLoops::loop(Constraints(m_dx), m_state_new, m_state_diagnostics, + EXCLUDE_GHOST_CELLS); } void KerrBHLevel::prePlotLevel() { +#ifdef USE_AHFINDER + // already calculated in 'specificPostTimeStep' + if (m_bh_amr.m_ah_finder.need_diagnostics(m_dt, m_time)) + return; +#endif + fillAllGhosts(); BoxLoops::loop(Constraints(m_dx), m_state_new, m_state_diagnostics, EXCLUDE_GHOST_CELLS); @@ -63,8 +74,7 @@ void KerrBHLevel::specificEvalRHS(GRLevelData &a_soln, GRLevelData &a_rhs, BoxLoops::loop(make_compute_pack(TraceARemoval(), PositiveChiAndAlpha()), a_soln, a_soln, INCLUDE_GHOST_CELLS); - // Calculate CCZ4 right hand side and set constraints to zero to avoid - // undefined values + // Calculate CCZ4 right hand side BoxLoops::loop(CCZ4(m_p.ccz4_params, m_dx, m_p.sigma, m_p.formulation), a_soln, a_rhs, EXCLUDE_GHOST_CELLS); } @@ -81,3 +91,19 @@ void KerrBHLevel::computeTaggingCriterion(FArrayBox &tagging_criterion, { BoxLoops::loop(ChiTaggingCriterion(m_dx), current_state, tagging_criterion); } + +void KerrBHLevel::specificPostTimeStep() +{ + CH_TIME("KerrBHLevel::specificPostTimeStep"); +#ifdef USE_AHFINDER + // if print is on and there are Diagnostics to write, calculate them! + if (m_bh_amr.m_ah_finder.need_diagnostics(m_dt, m_time)) + { + fillAllGhosts(); + BoxLoops::loop(Constraints(m_dx), m_state_new, m_state_diagnostics, + EXCLUDE_GHOST_CELLS); + } + if (m_p.AH_activate && m_level == m_p.AH_params.level_to_run) + m_bh_amr.m_ah_finder.solve(m_dt, m_time, m_restart_time); +#endif +} diff --git a/Examples/KerrBH/KerrBHLevel.hpp b/Examples/KerrBH/KerrBHLevel.hpp index 8698631e6..bfc526380 100644 --- a/Examples/KerrBH/KerrBHLevel.hpp +++ b/Examples/KerrBH/KerrBHLevel.hpp @@ -6,6 +6,7 @@ #ifndef KERRBHLEVEL_HPP_ #define KERRBHLEVEL_HPP_ +#include "BHAMR.hpp" #include "DefaultLevelFactory.hpp" #include "GRAMRLevel.hpp" @@ -15,6 +16,8 @@ class KerrBHLevel : public GRAMRLevel // Inherit the contructors from GRAMRLevel using GRAMRLevel::GRAMRLevel; + BHAMR &m_bh_amr = dynamic_cast(m_gr_amr); + /// Things to do at every full timestep ///(might include several substeps, e.g. in RK4) virtual void specificAdvance() override; @@ -37,6 +40,8 @@ class KerrBHLevel : public GRAMRLevel virtual void computeTaggingCriterion(FArrayBox &tagging_criterion, const FArrayBox ¤t_state) override; + + virtual void specificPostTimeStep() override; }; #endif /* KERRBHLEVEL_HPP_ */ diff --git a/Examples/KerrBH/Main_KerrBH.cpp b/Examples/KerrBH/Main_KerrBH.cpp index 46f8c28f0..f2e82f89c 100644 --- a/Examples/KerrBH/Main_KerrBH.cpp +++ b/Examples/KerrBH/Main_KerrBH.cpp @@ -10,8 +10,8 @@ #include // Our general includes +#include "BHAMR.hpp" #include "DefaultLevelFactory.hpp" -#include "GRAMR.hpp" #include "GRParmParse.hpp" #include "SetupFunctions.hpp" #include "SimulationParameters.hpp" @@ -36,22 +36,51 @@ int runGRChombo(int argc, char *argv[]) // The line below selects the problem that is simulated // (To simulate a different problem, define a new child of AMRLevel // and an associated LevelFactory) - GRAMR gr_amr; - DefaultLevelFactory kerr_bh_level_fact(gr_amr, sim_params); - setupAMRObject(gr_amr, kerr_bh_level_fact); + BHAMR bh_amr; + DefaultLevelFactory kerr_bh_level_fact(bh_amr, sim_params); + setupAMRObject(bh_amr, kerr_bh_level_fact); + + // Set up interpolator: + // call this after amr object setup so grids known + // and need it to stay in scope throughout run + // Note: 'interpolator' needs to be in scope when bh_amr.run() is called, + // otherwise pointer is lost + AMRInterpolator> interpolator( + bh_amr, sim_params.origin, sim_params.dx, sim_params.boundary_params, + sim_params.verbosity); + bh_amr.set_interpolator(&interpolator); + +#ifdef USE_AHFINDER + if (sim_params.AH_activate) + { + AHSphericalGeometry sph(sim_params.kerr_params.center); + +#ifdef USE_CHI_CONTOURS // uncomment in UserVariables + std::string str_chi = std::to_string(sim_params.look_for_chi_contour); + bh_amr.m_ah_finder.add_ah( + sph, sim_params.AH_initial_guess, sim_params.AH_params, + sim_params.look_for_chi_contour, "stats_chi_" + str_chi + "_", + "coords_chi_" + str_chi + "_"); +#else + bh_amr.m_ah_finder.add_ah(sph, sim_params.AH_initial_guess, + sim_params.AH_params); +#endif + } +#endif using Clock = std::chrono::steady_clock; using Minutes = std::chrono::duration>; std::chrono::time_point start_time = Clock::now(); - gr_amr.run(sim_params.stop_time, sim_params.max_steps); + // Engage! Run the evolution + bh_amr.run(sim_params.stop_time, sim_params.max_steps); auto now = Clock::now(); auto duration = std::chrono::duration_cast(now - start_time); pout() << "Total simulation time (mins): " << duration.count() << ".\n"; - gr_amr.conclude(); + bh_amr.conclude(); CH_TIMER_REPORT(); // Report results when running with Chombo timers. diff --git a/Examples/KerrBH/SimulationParameters.hpp b/Examples/KerrBH/SimulationParameters.hpp index b8143d7ec..f9d7a7708 100644 --- a/Examples/KerrBH/SimulationParameters.hpp +++ b/Examples/KerrBH/SimulationParameters.hpp @@ -30,6 +30,14 @@ class SimulationParameters : public SimulationParametersBase pp.load("kerr_mass", kerr_params.mass); pp.load("kerr_spin", kerr_params.spin); pp.load("kerr_center", kerr_params.center, center); + +#ifdef USE_AHFINDER + pp.load("AH_initial_guess", AH_initial_guess, 0.5 * kerr_params.mass); +#ifdef USE_CHI_CONTOURS + pp.load("look_for_chi_contour", look_for_chi_contour); + CH_assert(sim_params.look_for_chi_contour > 0.); +#endif +#endif } void check_params() @@ -52,6 +60,15 @@ class SimulationParameters : public SimulationParametersBase } KerrBH::params_t kerr_params; + +#ifdef USE_AHFINDER + double AH_initial_guess; + // example of how to change the expansion function + double look_for_chi_contour; // look for a chi contour instead of the AH + // (negative number to look for AH) + // changes only the expansion function + // (ApparentHorizon::get_expansion) +#endif }; #endif /* SIMULATIONPARAMETERS_HPP_ */ diff --git a/Examples/KerrBH/UserVariables.hpp b/Examples/KerrBH/UserVariables.hpp index b3ca4c817..b0bc8f7a6 100644 --- a/Examples/KerrBH/UserVariables.hpp +++ b/Examples/KerrBH/UserVariables.hpp @@ -26,4 +26,12 @@ static const std::array variable_names = #include "UserVariables.inc.hpp" +// uncomment to look for chi instead of expansion +// #define USE_CHI_CONTOURS + +#ifdef USE_CHI_CONTOURS +#include "AHFunctions.hpp" +#define AHFunction ChiContourFunction // change default to chi contours +#endif + #endif /* USERVARIABLES_HPP */ diff --git a/Examples/KerrBH/params.txt b/Examples/KerrBH/params.txt index f0e2604f0..6b3108fa9 100644 --- a/Examples/KerrBH/params.txt +++ b/Examples/KerrBH/params.txt @@ -107,3 +107,21 @@ extraction_radii = 30. 40. 50. extraction_levels = 2 1 0 num_points_phi = 50 num_points_theta = 52 + +#Apparent Horizon finder +AH_activate = 1 +AH_num_ranks = 20 +AH_num_points_u = 30 +AH_num_points_v = 50 +AH_solve_interval = 1 +AH_print_interval = 2 +AH_track_center = false +AH_predict_origin = false +#AH_level_to_run = 0 +#AH_allow_re_attempt = 0 +#AH_start_time = 0.0 +#AH_give_up_time = -1. # -1 to never +#AH_max_fails_after_lost = 0 # -1 to never +#AH_verbose = 1 + +#AH_initial_guess = 0.5 diff --git a/Examples/KerrBH/params_cheap.txt b/Examples/KerrBH/params_cheap.txt index da48ba071..9c079f2d3 100644 --- a/Examples/KerrBH/params_cheap.txt +++ b/Examples/KerrBH/params_cheap.txt @@ -3,7 +3,7 @@ verbosity = 0 chk_prefix = KerrBH_ plot_prefix = KerrBHp_ -#restart_file = KerrBH_000060.3d.hdf5 +#restart_file = KerrBH_000004.3d.hdf5 # 'N' is the number of subdivisions in each direction of a cubic box # 'L' is the length of the longest side of the box, dx_coarsest = L/N @@ -26,14 +26,14 @@ kerr_spin = 0.5 # Level data # Maximum number of times you can regrid above coarsest level -max_level = 4 #4 There are (max_level+1) grids, so min is zero +max_level = 3 # There are (max_level+1) grids, so min is zero # Frequency of regridding at each level and thresholds on the tagging # Need one for each level, ie max_level+1 items # Generally you do not need to regrid frequently on every level # Level Regridding: 0 1 2 3 4 5 6 regrid_interval = 50 25 5 5 5 5 5 -regrid_thresholds = 0.3 0.3 0.3 0.3 0.3 +regrid_thresholds = 0.3 0.3 0.3 0.3 0.3 0.3 # Max box sizes max_grid_size = 16 @@ -74,11 +74,12 @@ vars_asymptotic_values = 1.0 1.0 0.0 0.0 1.0 0.0 1.0 #chi and hij # dt will be dx*dt_multiplier on each grid level # HDF5files are written every dt = L/N*dt_multiplier*checkpoint_interval checkpoint_interval = 1 -plot_interval = 0 -num_plot_vars = 0 +plot_interval = 2 +num_plot_vars = 2 +plot_vars = chi Ham dt_multiplier = 0.25 stop_time = 10.0 -max_steps = 2 +max_steps = 4 #Lapse evolution lapse_power = 1.0 @@ -87,7 +88,7 @@ lapse_advec_coeff = 1 # 1 makes the lapse gauge 1+log slicing # Shift evolution coefficients shift_advec_coeff = 0 # Usually no advection for beta -shift_Gamma_coeff = 0.75 # +shift_Gamma_coeff = 0.75 # eta = 1.0 # This is gamma driver, usually of order 1/M_ADM of spacetime # CCZ4 parameters @@ -109,3 +110,27 @@ extraction_radii = 30. 40. 50. extraction_levels = 1 1 0 num_points_phi = 50 num_points_theta = 52 + +#Apparent Horizon finder +AH_activate = 1 +AH_num_ranks = 2 #20 +AH_num_points_u = 11 #30 +AH_num_points_v = 10 #50 +AH_solve_interval = 2 +AH_print_interval = 2 +AH_track_center = false +AH_predict_origin = false +AH_level_to_run = 0 +#AH_allow_re_attempt = 0 +#AH_stop_if_max_fails = 0 +AH_start_time = 0. +AH_verbose = 3 +AH_print_geometry_data = 0 +AH_re_solve_at_restart = 0 + +#AH_initial_guess = 0.5 + +look_for_chi_contour = 0.2 + +AH_num_write_vars = 2 +AH_write_vars = chi d1_Ham \ No newline at end of file diff --git a/Examples/ScalarField/GNUmakefile b/Examples/ScalarField/GNUmakefile index aff1cf3a1..bc745a483 100644 --- a/Examples/ScalarField/GNUmakefile +++ b/Examples/ScalarField/GNUmakefile @@ -20,6 +20,8 @@ src_dirs := $(GRCHOMBO_SOURCE)/utils \ $(GRCHOMBO_SOURCE)/GRChomboCore \ $(GRCHOMBO_SOURCE)/TaggingCriteria \ $(GRCHOMBO_SOURCE)/AMRInterpolator \ - $(GRCHOMBO_SOURCE)/InitialConditions/BlackHoles + $(GRCHOMBO_SOURCE)/ApparentHorizonFinder \ + $(GRCHOMBO_SOURCE)/InitialConditions/BlackHoles \ + $(GRCHOMBO_SOURCE)/BlackHoles include $(CHOMBO_HOME)/mk/Make.test diff --git a/Examples/ScalarField/Main_ScalarField.cpp b/Examples/ScalarField/Main_ScalarField.cpp index f6eea5b50..ca2b7c491 100644 --- a/Examples/ScalarField/Main_ScalarField.cpp +++ b/Examples/ScalarField/Main_ScalarField.cpp @@ -10,8 +10,8 @@ #include // Our general includes +#include "BHAMR.hpp" #include "DefaultLevelFactory.hpp" -#include "GRAMR.hpp" #include "GRParmParse.hpp" #include "SetupFunctions.hpp" #include "SimulationParameters.hpp" @@ -36,14 +36,42 @@ int runGRChombo(int argc, char *argv[]) // The line below selects the problem that is simulated // (To simulate a different problem, define a new child of AMRLevel // and an associated LevelFactory) - GRAMR gr_amr; - DefaultLevelFactory scalar_field_level_fact(gr_amr, + BHAMR bh_amr; + DefaultLevelFactory scalar_field_level_fact(bh_amr, sim_params); - setupAMRObject(gr_amr, scalar_field_level_fact); + setupAMRObject(bh_amr, scalar_field_level_fact); + + // call this after amr object setup so grids known + // and need it to stay in scope throughout run + AMRInterpolator> interpolator( + bh_amr, sim_params.origin, sim_params.dx, sim_params.boundary_params, + sim_params.verbosity); + bh_amr.set_interpolator(&interpolator); + +#ifdef USE_AHFINDER + if (sim_params.AH_activate) + { + AHSphericalGeometry sph(sim_params.center); + bh_amr.m_ah_finder.add_ah(sph, sim_params.AH_initial_guess, + sim_params.AH_params); + } +#endif + + using Clock = std::chrono::steady_clock; + using Minutes = std::chrono::duration>; + + std::chrono::time_point start_time = Clock::now(); // Engage! Run the evolution - gr_amr.run(sim_params.stop_time, sim_params.max_steps); - gr_amr.conclude(); + bh_amr.run(sim_params.stop_time, sim_params.max_steps); + + auto now = Clock::now(); + auto duration = std::chrono::duration_cast(now - start_time); + pout() << "Total simulation time (mins): " << duration.count() << ".\n"; + + bh_amr.conclude(); + + CH_TIMER_REPORT(); // Report results when running with Chombo timers. return 0; } diff --git a/Examples/ScalarField/ScalarFieldLevel.cpp b/Examples/ScalarField/ScalarFieldLevel.cpp index 568ccde79..97c1bb522 100644 --- a/Examples/ScalarField/ScalarFieldLevel.cpp +++ b/Examples/ScalarField/ScalarFieldLevel.cpp @@ -39,8 +39,9 @@ void ScalarFieldLevel::specificAdvance() // Check for nan's if (m_p.nan_check) - BoxLoops::loop(NanCheck(), m_state_new, m_state_new, - EXCLUDE_GHOST_CELLS, disable_simd()); + BoxLoops::loop( + NanCheck(m_dx, m_p.center, "NaNCheck in specific Advance"), + m_state_new, m_state_new, EXCLUDE_GHOST_CELLS, disable_simd()); } // Initial data for field and metric variables @@ -108,3 +109,10 @@ void ScalarFieldLevel::computeTaggingCriterion(FArrayBox &tagging_criterion, FixedGridsTaggingCriterion(m_dx, m_level, 2.0 * m_p.L, m_p.center), current_state, tagging_criterion); } +void ScalarFieldLevel::specificPostTimeStep() +{ +#ifdef USE_AHFINDER + if (m_p.AH_activate && m_level == m_p.AH_params.level_to_run) + m_bh_amr.m_ah_finder.solve(m_dt, m_time, m_restart_time); +#endif +} \ No newline at end of file diff --git a/Examples/ScalarField/ScalarFieldLevel.hpp b/Examples/ScalarField/ScalarFieldLevel.hpp index eed9906de..6c54c7414 100644 --- a/Examples/ScalarField/ScalarFieldLevel.hpp +++ b/Examples/ScalarField/ScalarFieldLevel.hpp @@ -6,6 +6,7 @@ #ifndef SCALARFIELDLEVEL_HPP_ #define SCALARFIELDLEVEL_HPP_ +#include "BHAMR.hpp" #include "DefaultLevelFactory.hpp" #include "GRAMRLevel.hpp" // Problem specific includes @@ -27,6 +28,8 @@ class ScalarFieldLevel : public GRAMRLevel // Inherit the contructors from GRAMRLevel using GRAMRLevel::GRAMRLevel; + BHAMR &m_bh_amr = dynamic_cast(m_gr_amr); + // Typedef for scalar field typedef ScalarField ScalarFieldWithPotential; @@ -50,6 +53,9 @@ class ScalarFieldLevel : public GRAMRLevel //! Tell Chombo how to tag cells for regridding virtual void computeTaggingCriterion(FArrayBox &tagging_criterion, const FArrayBox ¤t_state); + + //! to do post each time step on every level + virtual void specificPostTimeStep() override; }; #endif /* SCALARFIELDLEVEL_HPP_ */ diff --git a/Examples/ScalarField/SimulationParameters.hpp b/Examples/ScalarField/SimulationParameters.hpp index 111b26932..f6bc21f6f 100644 --- a/Examples/ScalarField/SimulationParameters.hpp +++ b/Examples/ScalarField/SimulationParameters.hpp @@ -41,6 +41,12 @@ class SimulationParameters : public SimulationParametersBase pp.load("kerr_mass", kerr_params.mass, 1.0); pp.load("kerr_spin", kerr_params.spin, 0.0); pp.load("kerr_center", kerr_params.center, center); + +#ifdef USE_AHFINDER + double AH_guess = + 8. * initial_params.amplitude * initial_params.amplitude; + pp.load("AH_initial_guess", AH_initial_guess, AH_guess); +#endif } void check_params() @@ -75,6 +81,10 @@ class SimulationParameters : public SimulationParametersBase InitialScalarData::params_t initial_params; Potential::params_t potential_params; KerrBH::params_t kerr_params; + +#ifdef USE_AHFINDER + double AH_initial_guess; +#endif }; #endif /* SIMULATIONPARAMETERS_HPP_ */ diff --git a/Examples/ScalarField/params.txt b/Examples/ScalarField/params.txt index 783580913..79446b4dc 100644 --- a/Examples/ScalarField/params.txt +++ b/Examples/ScalarField/params.txt @@ -109,3 +109,22 @@ scalar_mass = 0.2 # Kerr data kerr_mass = 1.0 kerr_spin = 0.0 + + +#Apparent Horizon finder +AH_activate = 0 +AH_num_ranks = 20 +AH_num_points_u = 30 +AH_num_points_v = 50 +AH_solve_interval = 1 +AH_print_interval = 1 +AH_track_center = false +#AH_predict_origin = true +#AH_level_to_run = 0 +#AH_allow_re_attempt = 0 +#AH_start_time = 0.0 +#AH_give_up_time = -1. # -1 to never +#AH_max_fails_after_lost = 0 # -1 to never +#AH_verbose = 1 + +#AH_initial_guess = 1.0 \ No newline at end of file diff --git a/Examples/SingleBH/DiagnosticVariables.hpp b/Examples/SingleBH/DiagnosticVariables.hpp new file mode 100644 index 000000000..88ff5b7dc --- /dev/null +++ b/Examples/SingleBH/DiagnosticVariables.hpp @@ -0,0 +1,29 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef DIAGNOSTICVARIABLES_HPP +#define DIAGNOSTICVARIABLES_HPP + +// assign an enum to each variable +enum +{ + c_Ham, + + c_Mom1, + c_Mom2, + c_Mom3, + + NUM_DIAGNOSTIC_VARS +}; + +namespace DiagnosticVariables +{ +static const std::array variable_names = { + "Ham", + + "Mom1", "Mom2", "Mom3"}; +} + +#endif /* DIAGNOSTICVARIABLES_HPP */ diff --git a/Examples/SingleBH/GNUmakefile b/Examples/SingleBH/GNUmakefile new file mode 100644 index 000000000..b7c938eac --- /dev/null +++ b/Examples/SingleBH/GNUmakefile @@ -0,0 +1,26 @@ +# -*- Mode: Makefile -*- + +### This makefile produces an executable for each name in the `ebase' +### variable using the libraries named in the `LibNames' variable. + +# Included makefiles need an absolute path to the Chombo installation +# CHOMBO_HOME := Please set the CHOMBO_HOME locally (e.g. export CHOMBO_HOME=... in bash) + +GRCHOMBO_SOURCE = ../../Source + +ebase := Main_SingleBH + +LibNames := AMRTimeDependent AMRTools BoxTools + +src_dirs := $(GRCHOMBO_SOURCE)/utils \ + $(GRCHOMBO_SOURCE)/simd \ + $(GRCHOMBO_SOURCE)/CCZ4 \ + $(GRCHOMBO_SOURCE)/BoxUtils \ + $(GRCHOMBO_SOURCE)/GRChomboCore \ + $(GRCHOMBO_SOURCE)/AMRInterpolator \ + $(GRCHOMBO_SOURCE)/TaggingCriteria \ + $(GRCHOMBO_SOURCE)/ApparentHorizonFinder \ + $(GRCHOMBO_SOURCE)/InitialConditions/BlackHoles \ + $(GRCHOMBO_SOURCE)/BlackHoles + +include $(CHOMBO_HOME)/mk/Make.test diff --git a/Examples/SingleBH/Main_SingleBH.cpp b/Examples/SingleBH/Main_SingleBH.cpp new file mode 100644 index 000000000..45fcb60a8 --- /dev/null +++ b/Examples/SingleBH/Main_SingleBH.cpp @@ -0,0 +1,82 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#include "CH_Timer.H" +#include "parstream.H" //Gives us pout() +#include +#include + +#include "BHAMR.hpp" +#include "DefaultLevelFactory.hpp" +#include "GRParmParse.hpp" +#include "SetupFunctions.hpp" +#include "SimulationParameters.hpp" + +// Problem specific includes: +#include "SingleBHLevel.hpp" + +int runGRChombo(int argc, char *argv[]) +{ + // Load the parameter file and construct the SimulationParameter class + // To add more parameters edit the SimulationParameters file. + char *in_file = argv[1]; + GRParmParse pp(argc - 2, argv + 2, NULL, in_file); + SimulationParameters sim_params(pp); + + // The line below selects the problem that is simulated + // (To simulate a different problem, define a new child of AMRLevel + // and an associated LevelFactory) + BHAMR bh_amr; + DefaultLevelFactory single_bh_level_fact(bh_amr, sim_params); + setupAMRObject(bh_amr, single_bh_level_fact); + + // call this after amr object setup so grids known + // and need it to stay in scope throughout run + AMRInterpolator> interpolator( + bh_amr, sim_params.origin, sim_params.dx, sim_params.boundary_params, + sim_params.verbosity); + bh_amr.set_interpolator(&interpolator); + +#ifdef USE_AHFINDER + if (sim_params.AH_activate) + { + AHSphericalGeometry sph(sim_params.bh_params.center); + bh_amr.m_ah_finder.add_ah(sph, sim_params.AH_initial_guess, + sim_params.AH_params); + } +#endif + + using Clock = std::chrono::steady_clock; + using Minutes = std::chrono::duration>; + + std::chrono::time_point start_time = Clock::now(); + + bh_amr.run(sim_params.stop_time, sim_params.max_steps); + + auto now = Clock::now(); + auto duration = std::chrono::duration_cast(now - start_time); + pout() << "Total simulation time (mins): " << duration.count() << ".\n"; + + bh_amr.conclude(); + + CH_TIMER_REPORT(); // Report results when running with Chombo timers. + + return 0; +} + +int main(int argc, char *argv[]) +{ + mainSetup(argc, argv); + + int status = runGRChombo(argc, argv); + + if (status == 0) + pout() << "GRChombo finished." << std::endl; + else + pout() << "GRChombo failed with return code " << status << std::endl; + + mainFinalize(); + return status; +} diff --git a/Examples/SingleBH/SimulationParameters.hpp b/Examples/SingleBH/SimulationParameters.hpp new file mode 100644 index 000000000..07ea18a66 --- /dev/null +++ b/Examples/SingleBH/SimulationParameters.hpp @@ -0,0 +1,55 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef SIMULATIONPARAMETERS_HPP +#define SIMULATIONPARAMETERS_HPP + +// General includes +#include "GRParmParse.hpp" +#include "SimulationParametersBase.hpp" + +// Problem specific includes: +#include "BoostedBH.hpp" + +class SimulationParameters : public SimulationParametersBase +{ + public: + SimulationParameters(GRParmParse &pp) : SimulationParametersBase(pp) + { + readParams(pp); + } + + /// Read parameters from the parameter file + void readParams(GRParmParse &pp) + { + // Initial data + pp.load("mass", bh_params.mass); + pp.load("momentum", bh_params.momentum); + pp.load("activate_extraction", activate_extraction, 0); + + // Get the centers of the BHs either explicitly or as + // an offset (not both, or they will be offset from center + // provided) + std::array offset; + pp.load("offset", offset, {0.0, 0.0, 0.0}); + FOR1(idir) { bh_params.center[idir] = center[idir] + offset[idir]; } + +#ifdef USE_AHFINDER + pp.load("AH_initial_guess", AH_initial_guess, 0.5 * bh_params.mass); +#endif + } + + // Initial data + int activate_extraction; + + // Collection of parameters necessary for initial conditions + BoostedBH::params_t bh_params; + +#ifdef USE_AHFINDER + double AH_initial_guess; +#endif +}; + +#endif /* SIMULATIONPARAMETERS_HPP_ */ diff --git a/Examples/SingleBH/SingleBHLevel.cpp b/Examples/SingleBH/SingleBHLevel.cpp new file mode 100644 index 000000000..4f8f3fca4 --- /dev/null +++ b/Examples/SingleBH/SingleBHLevel.cpp @@ -0,0 +1,88 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#include "SingleBHLevel.hpp" +#include "BoxLoops.hpp" +#include "CCZ4.hpp" +#include "ChiExtractionTaggingCriterion.hpp" +#include "ComputePack.hpp" +#include "Constraints.hpp" +#include "NanCheck.hpp" +#include "PositiveChiAndAlpha.hpp" +#include "SetValue.hpp" +#include "SingleBH.hpp" +#include "TraceARemoval.hpp" + +void SingleBHLevel::specificAdvance() +{ + // Enforce the trace free A_ij condition and positive chi and alpha + BoxLoops::loop(make_compute_pack(TraceARemoval(), PositiveChiAndAlpha()), + m_state_new, m_state_new, INCLUDE_GHOST_CELLS); + + // Check for nan's + if (m_p.nan_check) + BoxLoops::loop( + NanCheck(m_dx, m_p.center, "NaNCheck in specific Advance"), + m_state_new, m_state_new, EXCLUDE_GHOST_CELLS, disable_simd()); +} + +void SingleBHLevel::initialData() +{ + CH_TIME("SingleBHLevel::initialData"); + if (m_verbosity) + pout() << "SingleBHLevel::initialData " << m_level << endl; + + // Set up the compute class for the SingleBH initial data + SingleBH single(m_p.bh_params, m_dx); + + // First set everything to zero (to avoid undefinded values in constraints) + // then calculate initial data + BoxLoops::loop(make_compute_pack(SetValue(0.), single), m_state_new, + m_state_new, INCLUDE_GHOST_CELLS); +} + +void SingleBHLevel::prePlotLevel() +{ + fillAllGhosts(); + BoxLoops::loop(Constraints(m_dx), m_state_new, m_state_diagnostics, + EXCLUDE_GHOST_CELLS); +} + +void SingleBHLevel::specificEvalRHS(GRLevelData &a_soln, GRLevelData &a_rhs, + const double a_time) +{ + // Enforce positive chi and alpha and trace free A + BoxLoops::loop(make_compute_pack(TraceARemoval(), PositiveChiAndAlpha()), + a_soln, a_soln, INCLUDE_GHOST_CELLS); + + // Calculate CCZ4 right hand side + BoxLoops::loop(CCZ4(m_p.ccz4_params, m_dx, m_p.sigma, m_p.formulation), + a_soln, a_rhs, EXCLUDE_GHOST_CELLS); +} + +void SingleBHLevel::specificUpdateODE(GRLevelData &a_soln, + const GRLevelData &a_rhs, Real a_dt) +{ + // Enforce the trace free A_ij condition + BoxLoops::loop(TraceARemoval(), a_soln, a_soln, INCLUDE_GHOST_CELLS); +} + +void SingleBHLevel::computeTaggingCriterion(FArrayBox &tagging_criterion, + const FArrayBox ¤t_state) +{ + BoxLoops::loop(ChiExtractionTaggingCriterion(m_dx, m_level, + m_p.extraction_params, + m_p.activate_extraction), + current_state, tagging_criterion); +} + +void SingleBHLevel::specificPostTimeStep() +{ + CH_TIME("SingleBHLevel::specificPostTimeStep"); +#ifdef USE_AHFINDER + if (m_p.AH_activate && m_level == m_p.AH_params.level_to_run) + m_bh_amr.m_ah_finder.solve(m_dt, m_time, m_restart_time); +#endif +} diff --git a/Examples/SingleBH/SingleBHLevel.hpp b/Examples/SingleBH/SingleBHLevel.hpp new file mode 100644 index 000000000..8efd617cf --- /dev/null +++ b/Examples/SingleBH/SingleBHLevel.hpp @@ -0,0 +1,49 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef SINGLEBHLEVEL_HPP_ +#define SINGLEBHLEVEL_HPP_ + +#include "BHAMR.hpp" +#include "DefaultLevelFactory.hpp" +#include "GRAMRLevel.hpp" + +class SingleBHLevel : public GRAMRLevel +{ + friend class DefaultLevelFactory; + // Inherit the contructors from GRAMRLevel + using GRAMRLevel::GRAMRLevel; + + BHAMR &m_bh_amr = dynamic_cast(m_gr_amr); + + /// Things to do at every full timestep + ///(might include several substeps, e.g. in RK4) + virtual void specificAdvance() override; + + /// Initial data calculation + virtual void initialData() override; + + /// Any actions that should happen just before checkpointing + virtual void prePlotLevel() override; + + /// Calculation of the right hand side for the time stepping + virtual void specificEvalRHS(GRLevelData &a_soln, GRLevelData &a_rhs, + const double a_time) override; + + /// Things to do after dt*rhs has been added to the solution + virtual void specificUpdateODE(GRLevelData &a_soln, + const GRLevelData &a_rhs, + Real a_dt) override; + + /// Identify and tag the cells that need higher resolution + virtual void + computeTaggingCriterion(FArrayBox &tagging_criterion, + const FArrayBox ¤t_state) override; + + // to do post each time step on every level + virtual void specificPostTimeStep() override; +}; + +#endif /* SINGLEBHLEVEL_HPP_ */ diff --git a/Examples/SingleBH/UserVariables.hpp b/Examples/SingleBH/UserVariables.hpp new file mode 100644 index 000000000..b3ca4c817 --- /dev/null +++ b/Examples/SingleBH/UserVariables.hpp @@ -0,0 +1,29 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef USERVARIABLES_HPP +#define USERVARIABLES_HPP + +#include "ArrayTools.hpp" +#include "CCZ4UserVariables.hpp" +#include "DiagnosticVariables.hpp" + +/// This enum gives the index of every variable stored in the grid +enum +{ + // Note that it is important that the first enum value is set to 1 more than + // the last CCZ4 var enum + NUM_VARS = NUM_CCZ4_VARS, +}; + +namespace UserVariables +{ +static const std::array variable_names = + ccz4_variable_names; +} // namespace UserVariables + +#include "UserVariables.inc.hpp" + +#endif /* USERVARIABLES_HPP */ diff --git a/Examples/SingleBH/params.txt b/Examples/SingleBH/params.txt new file mode 100644 index 000000000..d05962cbf --- /dev/null +++ b/Examples/SingleBH/params.txt @@ -0,0 +1,106 @@ +verbosity = 0 +chk_prefix = SingleBH_ +plot_prefix = SingleBHPlot_ +#restart_file = SingleBH_000004.3d.hdf5 + +# Set up grid spacings and regrid params +# NB - the N values need to be multiples of block_factor + +N_full = 32 +L_full = 16 + +mass = 1.0 +offset = 0 0 0 +momentum = 0. -0.5 0.0 + +regrid_threshold = 0.05 +max_level = 2 +regrid_interval = 1 2 4 8 16 + + +#boundaries and periodicity of grid +#Periodic directions - 0 = false, 1 = true +isPeriodic = 0 0 0 +# if not periodic, then specify the boundary type +# 0 = static, 1 = sommerfeld, 2 = reflective +# (see BoundaryConditions.hpp for details) +hi_boundary = 1 1 1 +lo_boundary = 1 1 2 + +# if reflective boundaries selected, must set +# parity of all vars (in order given by UserVariables.hpp) +# 0 = even +# 1,2,3 = odd x, y, z +# 4,5,6 = odd xy, yz, xz +# 7 = odd xyz +vars_parity = 0 0 4 6 0 5 0 #chi and hij + 0 0 4 6 0 5 0 #K and Aij + 0 1 2 3 #Theta and Gamma + 0 1 2 3 1 2 3 #lapse shift and B +vars_parity_diagnostic = 0 1 2 3 #Ham and Mom + +# if sommerfeld boundaries selected, must select +# asymptotic values (in order given by UserVariables.hpp) +vars_asymptotic_values = 1.0 1.0 0.0 0.0 1.0 0.0 1.0 #chi and hij + 0.0 0.0 0.0 0.0 0.0 0.0 0.0 #K and Aij + 0.0 0.0 0.0 0.0 #Theta and Gamma + 1.0 0.0 0.0 0.0 0.0 0.0 0.0 #lapse shift and B + +#Max and min box sizes +max_grid_size = 16 +block_factor = 16 + +# Set up time steps +# dt will be dx*dt_multiplier on each grid level +# HDF5files are written every dt = L/N*dt_multiplier*checkpoint_interval + +checkpoint_interval = 2 +# set to zero to turn off plot files +plot_interval = 1 +num_plot_vars = 1 +plot_vars = chi +dt_multiplier = 0.25 +stop_time = 10.0 +max_steps = 10 + +#Lapse evolution +lapse_advec_coeff = 0.0 +lapse_coeff = 2.0 +lapse_power = 1.0 + +# Shift evolution +shift_advec_coeff = 0.0 +shift_Gamma_coeff = 0.75 +eta = 1.82 + +# CCZ4 parameters +kappa1 = 0.1 +kappa2 = 0 +kappa3 = 1. +covariantZ4 = 1 # 0: default. 1: dampk1 -> dampk1/lapse + +#coefficient for KO numerical dissipation +sigma = 0.3 + +#Apparent Horizon finder +AH_activate = 1 +AH_num_ranks = 2 +AH_num_points_u = 11 +AH_num_points_v = 20 +AH_solve_interval = 1 +AH_print_interval = 1 +#AH_track_center = true +#AH_predict_origin = true +#AH_merger_search_factor = 1. #-1 to deactivate +#AH_merger_pre_factor = 1. +#AH_level_to_run = 0 +#AH_allow_re_attempt = 0 +#AH_start_time = 0.0 +#AH_give_up_time = -1. # -1 to never +#AH_max_fails_after_lost = 0 # -1 to never +AH_verbose = 3 + +#AH_initial_guess = 0.25 + +AH_num_write_vars = 2 +AH_write_vars = chi d1_chi \ No newline at end of file diff --git a/InstallNotes/MakeDefsLocalExamples/Cartesius.Make.defs.local b/InstallNotes/MakeDefsLocalExamples/Cartesius.Make.defs.local index b4e994568..f1740e8c6 100644 --- a/InstallNotes/MakeDefsLocalExamples/Cartesius.Make.defs.local +++ b/InstallNotes/MakeDefsLocalExamples/Cartesius.Make.defs.local @@ -19,4 +19,3 @@ cxxdbgflags = -g -Wl,--eh-frame-hdr cxxoptflags = -O3 -qoverride-limits -xSSE4.2 -axAVX fdbgflags = -g -Wl,--eh-frame-hdr foptflags = -O3 -qoverride-limits -xSSE4.2 -axAVX -cxxcppflags = -DDISABLE_AHFINDER diff --git a/InstallNotes/MakeDefsLocalExamples/MN4.Make.defs.local b/InstallNotes/MakeDefsLocalExamples/MN4.Make.defs.local index 59c04d333..5cd8fe717 100644 --- a/InstallNotes/MakeDefsLocalExamples/MN4.Make.defs.local +++ b/InstallNotes/MakeDefsLocalExamples/MN4.Make.defs.local @@ -18,4 +18,3 @@ cxxdbgflags = -g -Wl,--eh-frame-hdr cxxoptflags = -O3 -qoverride-limits -xSSE4.2 -axAVX fdbgflags = -g -Wl,--eh-frame-hdr foptflags = -O3 -qoverride-limits -xSSE4.2 -axAVX -cxxcppflags = -DDISABLE_AHFINDER diff --git a/InstallNotes/MakeDefsLocalExamples/athena.Make.defs.local b/InstallNotes/MakeDefsLocalExamples/athena.Make.defs.local index aa2e69557..87b271b3c 100644 --- a/InstallNotes/MakeDefsLocalExamples/athena.Make.defs.local +++ b/InstallNotes/MakeDefsLocalExamples/athena.Make.defs.local @@ -18,4 +18,4 @@ cxxdbgflags = -g -Wl,--eh-frame-hdr cxxoptflags = -O3 -qoverride-limits -xSSE4.2 -axAVX fdbgflags = -g -Wl,--eh-frame-hdr foptflags = -O3 -qoverride-limits -xSSE4.2 -axAVX -cxxcppflags := ${cxxcppflags} -DDISABLE_AHFINDER -DUSE_PAPI -UCH_USE_MEMORY_TRACKING +cxxcppflags := ${cxxcppflags} -DUSE_PAPI -UCH_USE_MEMORY_TRACKING diff --git a/InstallNotes/MakeDefsLocalExamples/cosmos.Make.defs.local b/InstallNotes/MakeDefsLocalExamples/cosmos.Make.defs.local index de08d3006..b5851c2f7 100644 --- a/InstallNotes/MakeDefsLocalExamples/cosmos.Make.defs.local +++ b/InstallNotes/MakeDefsLocalExamples/cosmos.Make.defs.local @@ -20,5 +20,5 @@ cxxdbgflags = -g -Wl,--eh-frame-hdr cxxoptflags = -O3 -override-limits -xSSE4.2 -axAVX fdbgflags = -g -Wl,--eh-frame-hdr foptflags = -O3 -override-limits -xSSE4.2 -axAVX -syslibflags = -L/nfs/software/apps/papi/5.4.1/lib -lpapi -cxxcppflags = -DDISABLE_AHFINDER -DUSE_PAPI -UCH_USE_MEMORY_TRACKING +syslibflags = -L/nfs/software/apps/papi/5.4.1/lib -lpapi +cxxcppflags = -DUSE_PAPI -UCH_USE_MEMORY_TRACKING diff --git a/InstallNotes/MakeDefsLocalExamples/docker_hpc-base-stretch.Make.defs.local b/InstallNotes/MakeDefsLocalExamples/docker_hpc-base-stretch.Make.defs.local index 07cb71518..805d0308d 100644 --- a/InstallNotes/MakeDefsLocalExamples/docker_hpc-base-stretch.Make.defs.local +++ b/InstallNotes/MakeDefsLocalExamples/docker_hpc-base-stretch.Make.defs.local @@ -22,5 +22,5 @@ cxxoptflags = -Ofast -march=native -std=c++14 fdbgflags = -g -Wl,--eh-frame-hdr foptflags = -Ofast -march=native syslibflags = -lgfortran -cxxcppflags = -ftemplate-depth=150 -DDISABLE_AHFINDER -DUSE_PAPI -XTRALIBFLAGS = -L/usr/lib/openblas-base/ -lopenblas +cxxcppflags = -ftemplate-depth=150 -DUSE_PAPI +XTRALIBFLAGS = -L/usr/lib/openblas-base/ -lopenblas diff --git a/InstallNotes/MakeDefsLocalExamples/gwdg.Make.defs.local b/InstallNotes/MakeDefsLocalExamples/gwdg.Make.defs.local index c32b78dd2..7eca2127c 100644 --- a/InstallNotes/MakeDefsLocalExamples/gwdg.Make.defs.local +++ b/InstallNotes/MakeDefsLocalExamples/gwdg.Make.defs.local @@ -20,4 +20,3 @@ cxxdbgflags = -g -Wl,--eh-frame-hdr cxxoptflags = -O3 -override-limits -xSSE4.2 -axAVX fdbgflags = -g -Wl,--eh-frame-hdr foptflags = -O3 -override-limits -xSSE4.2 -axAVX -cxxcppflags = -DDISABLE_AHFINDER diff --git a/InstallNotes/MakeDefsLocalExamples/nesi-pan.Make.defs.local b/InstallNotes/MakeDefsLocalExamples/nesi-pan.Make.defs.local index c27e1efd7..d5749a6e6 100644 --- a/InstallNotes/MakeDefsLocalExamples/nesi-pan.Make.defs.local +++ b/InstallNotes/MakeDefsLocalExamples/nesi-pan.Make.defs.local @@ -19,4 +19,3 @@ cxxdbgflags = -g -Wl,--eh-frame-hdr cxxoptflags = -O3 -override-limits -xSSE4.2 -axAVX fdbgflags = -g -Wl,--eh-frame-hdr foptflags = -O3 -override-limits -xSSE4.2 -axAVX -cxxcppflags = -DDISABLE_AHFINDER diff --git a/InstallNotes/MakeDefsLocalExamples/ubuntu-18.04.Make.defs.local b/InstallNotes/MakeDefsLocalExamples/ubuntu-18.04.Make.defs.local index e39380c96..96d368db3 100644 --- a/InstallNotes/MakeDefsLocalExamples/ubuntu-18.04.Make.defs.local +++ b/InstallNotes/MakeDefsLocalExamples/ubuntu-18.04.Make.defs.local @@ -5,7 +5,7 @@ # libhdf5-openmpi-dev openmpi-bin libblas-dev liblapack-dev \ # libgetopt-complete-perl ssh -DIM = 3 +DIM ?= 3 DEBUG = TRUE PRECISION = DOUBLE OPT = TRUE diff --git a/Source/AMRInterpolator/AMRInterpolator.hpp b/Source/AMRInterpolator/AMRInterpolator.hpp index b05c5401b..bbfc032ee 100644 --- a/Source/AMRInterpolator/AMRInterpolator.hpp +++ b/Source/AMRInterpolator/AMRInterpolator.hpp @@ -9,6 +9,7 @@ // Chombo includes #include "AMR.H" #include "AMRLevel.H" +#include "LoHiSide.H" // Our includes #include "BoundaryConditions.hpp" @@ -43,8 +44,10 @@ template class AMRInterpolator void limit_num_levels(unsigned int num_levels); void interp(InterpolationQuery &query); const AMR &getAMR() const; - const std::array &get_coarsest_dx(); - const std::array &get_coarsest_origin(); + const std::array &get_coarsest_dx() const; + const std::array &get_coarsest_origin() const; + bool get_boundary_reflective(Side::LoHiSide a_side, int a_dir) const; + bool get_boundary_periodic(int a_dir) const; private: void computeLevelLayouts(); diff --git a/Source/AMRInterpolator/AMRInterpolator.impl.hpp b/Source/AMRInterpolator/AMRInterpolator.impl.hpp index f4ed5e3c1..5a443f4eb 100644 --- a/Source/AMRInterpolator/AMRInterpolator.impl.hpp +++ b/Source/AMRInterpolator/AMRInterpolator.impl.hpp @@ -3,6 +3,10 @@ * Please refer to LICENSE in GRChombo's root directory. */ +#if !defined(AMRINTERPOLATOR_HPP_) +#error "This file should only be included through AMRInterpolator.hpp" +#endif + #ifndef AMRINTERPOLATOR_IMPL_HPP_ #define AMRINTERPOLATOR_IMPL_HPP_ @@ -49,7 +53,7 @@ template void AMRInterpolator::refresh() for (int level_idx = 0; level_idx < m_num_levels; ++level_idx) { AMRLevel &level = *levels[level_idx]; - InterpSource &interp_source = dynamic_cast(level); + InterpSource<> &interp_source = dynamic_cast &>(level); interp_source.fillAllGhosts(VariableType::evolution); if (NUM_DIAGNOSTIC_VARS > 0) interp_source.fillAllGhosts(VariableType::diagnostic); @@ -64,17 +68,32 @@ const AMR &AMRInterpolator::getAMR() const template const std::array & -AMRInterpolator::get_coarsest_dx() +AMRInterpolator::get_coarsest_dx() const { return m_coarsest_dx; } template const std::array & -AMRInterpolator::get_coarsest_origin() +AMRInterpolator::get_coarsest_origin() const { return m_coarsest_origin; } +template +bool AMRInterpolator::get_boundary_reflective(Side::LoHiSide a_side, + int a_dir) const +{ + if (a_side == Side::Lo) + return m_lo_boundary_reflective[a_dir]; + else + return m_hi_boundary_reflective[a_dir]; +} +template +bool AMRInterpolator::get_boundary_periodic(int a_dir) const +{ + return m_bc_params.is_periodic[a_dir]; +} + template void AMRInterpolator::limit_num_levels(unsigned int num_levels) { @@ -229,10 +248,14 @@ void AMRInterpolator::computeLevelLayouts() if (m_verbosity >= 2) { _pout << " Level " << level_idx << "\t" - << "dx=(" << m_dx[level_idx][0] << "," << m_dx[level_idx][1] - << "," << m_dx[level_idx][2] << ")\t" - << "grid_origin=(" << m_origin[level_idx][0] << "," - << m_origin[level_idx][1] << "," << m_origin[level_idx][2] + << "dx=(" + << D_TERM(m_dx[level_idx][0], << "," << m_dx[level_idx][1], + << "," << m_dx[level_idx][2]) + << ")\t" + << "grid_origin=(" + << D_TERM(m_origin[level_idx][0], + << "," << m_origin[level_idx][1], + << "," << m_origin[level_idx][2]) << ")" << endl; } @@ -282,7 +305,7 @@ AMRInterpolator::findBoxes(InterpolationQuery &query) // const AMRLevel& level = *levels[level_idx]; // const LevelData& level_data = dynamic_cast(level).getLevelData(); const DisjointBoxLayout& + // InterpSource<>&>(level).getLevelData(); const DisjointBoxLayout& // box_layout = level_data.disjointBoxLayout(); const Box& domain_box = // level.problemDomain().domainBox(); @@ -370,7 +393,7 @@ AMRInterpolator::findBoxes(InterpolationQuery &query) const AMRLevel &level = *levels[level_idx]; const LevelData &level_data = - dynamic_cast(level).getLevelData( + dynamic_cast &>(level).getLevelData( VariableType::evolution); const DisjointBoxLayout &box_layout = level_data.disjointBoxLayout(); const Box &domain_box = level.problemDomain().domainBox(); @@ -619,7 +642,6 @@ void AMRInterpolator::calculateAnswers(InterpolationQuery &query) const int num_answers = m_mpi.totalAnswerCount(); std::array grid_coord; - IntVect nearest; for (int answer_idx = 0; answer_idx < num_answers; ++answer_idx) { @@ -627,7 +649,8 @@ void AMRInterpolator::calculateAnswers(InterpolationQuery &query) const int level_idx = m_answer_level[answer_idx]; const AMRLevel &level = *levels[level_idx]; - const InterpSource &source = dynamic_cast(level); + const InterpSource<> &source = + dynamic_cast &>(level); const LevelData *const evolution_level_data_ptr = &source.getLevelData(VariableType::evolution); const LevelData *diagnostics_level_data_ptr; @@ -645,10 +668,6 @@ void AMRInterpolator::calculateAnswers(InterpolationQuery &query) &diagnostics_level_data_ptr->disjointBoxLayout(); } - const Box &domain_box = level.problemDomain().domainBox(); - const IntVect &small_end = domain_box.smallEnd(); - const IntVect &big_end = domain_box.bigEnd(); - // Convert the LayoutIndex to DataIndex const DataIndex evolution_data_idx( evolution_box_layout_ptr->layoutIterator()[box_idx]); @@ -684,22 +703,6 @@ void AMRInterpolator::calculateAnswers(InterpolationQuery &query) << (box.bigEnd()[i] + 0.5) << "]"; MayDay::Abort(s.str().c_str()); } - - // point lies beyond the "small end" of the whole domain, but still - // within the boundary cell - if (grid_coord[i] < small_end[i] && - grid_coord[i] >= small_end[i] - 0.5) - nearest[i] = small_end[i]; - - // point lies beyond the "big end" of the whole domain, but still - // within the boundary cell - else if (grid_coord[i] > big_end[i] && - grid_coord[i] <= big_end[i] + 0.5) - nearest[i] = big_end[i]; - - // otherwise we round to nearest grid point - else - nearest[i] = (int)ceil(grid_coord[i] - 0.5); } if (m_verbosity >= 2) @@ -725,7 +728,7 @@ void AMRInterpolator::calculateAnswers(InterpolationQuery &query) typedef std::vector comps_t; comps_t &comps = deriv_it->second; - algo.setup(deriv, m_dx[level_idx], grid_coord, nearest); + algo.setup(deriv, m_dx[level_idx], grid_coord); for (typename comps_t::iterator it = comps.begin(); it != comps.end(); ++it) diff --git a/Source/AMRInterpolator/CylindricalGeometry.hpp b/Source/AMRInterpolator/CylindricalGeometry.hpp index 14b34ae1f..8e1195580 100644 --- a/Source/AMRInterpolator/CylindricalGeometry.hpp +++ b/Source/AMRInterpolator/CylindricalGeometry.hpp @@ -32,36 +32,43 @@ class CylindricalGeometry { } + ALWAYS_INLINE double get_domain_u_min() const { return 0.; } + ALWAYS_INLINE double get_domain_u_max() const { return m_z_length; } + ALWAYS_INLINE double get_domain_v_min() const { return 0.; } + ALWAYS_INLINE double get_domain_v_max() const { return 2. * M_PI; } + //! returns the grid spacing in z - inline double du(int a_num_points_z) const + ALWAYS_INLINE double du(int a_num_points_z) const { - return m_z_length / (a_num_points_z - 1); + return (get_domain_u_max() - get_domain_u_min()) / (a_num_points_z - 1); } //! returns the grid spacing in phi - inline double dv(int a_num_points_phi) const + ALWAYS_INLINE double dv(int a_num_points_phi) const { - return 2.0 * M_PI / ((double)a_num_points_phi); + return (get_domain_v_max() - get_domain_v_min()) / + ((double)a_num_points_phi); } //! returns the z coordinate associated to the theta/u index - inline double u(int a_iz, int a_num_points_z) const + ALWAYS_INLINE double u(int a_iz, int a_num_points_z) const { - return a_iz * du(a_num_points_z) - (m_z_length / 2.0); + return a_iz * du(a_num_points_z) - + ((get_domain_v_max() - get_domain_v_min()) / 2.0); } //! returns the phi coordinate associated to the phi/v index - inline double v(int a_iphi, int a_num_points_phi) const + ALWAYS_INLINE double v(int a_iphi, int a_num_points_phi) const { return a_iphi * dv(a_num_points_phi); } - inline bool is_u_periodic() const { return false; } - inline bool is_v_periodic() const { return true; } + ALWAYS_INLINE bool is_u_periodic() const { return false; } + ALWAYS_INLINE bool is_v_periodic() const { return true; } //! returns the Cartesian coordinate in direction a_dir with specified //! radius, z and phi. - inline double get_grid_coord(int a_dir, double a_radius, double a_z, - double a_phi) const + ALWAYS_INLINE double get_grid_coord(int a_dir, double a_radius, double a_z, + double a_phi) const { switch (a_dir) { @@ -78,16 +85,17 @@ class CylindricalGeometry //! returns the area element on a cylinder with radius a_radius at the point //! (a_z, a_phi) - inline double area_element(double a_radius, double a_z, double a_phi) const + ALWAYS_INLINE double area_element(double a_radius, double a_z, + double a_phi) const { return a_radius; } - inline std::string param_name() const { return "r"; } + ALWAYS_INLINE std::string param_name() const { return "r"; } - inline std::string u_name() const { return "z"; } + ALWAYS_INLINE std::string u_name() const { return "z"; } - inline std::string v_name() const { return "phi"; } + ALWAYS_INLINE std::string v_name() const { return "phi"; } }; #endif /* CYLINDRICALGEOMETRY_HPP_ */ diff --git a/Source/AMRInterpolator/Derivative.hpp b/Source/AMRInterpolator/Derivative.hpp index f9b8943e0..2e9808224 100644 --- a/Source/AMRInterpolator/Derivative.hpp +++ b/Source/AMRInterpolator/Derivative.hpp @@ -108,15 +108,17 @@ class Derivative : public std::array static const Derivative dx; static const Derivative dy; - static const Derivative dz; static const Derivative dxdx; + static const Derivative dxdy; static const Derivative dydy; - static const Derivative dzdz; - static const Derivative dxdy; +#if CH_SPACEDIM == 3 + static const Derivative dz; static const Derivative dxdz; static const Derivative dydz; + static const Derivative dzdz; +#endif static std::string name(const Derivative &deriv) { @@ -124,20 +126,22 @@ class Derivative : public std::array return "dx"; else if (deriv == dy) return "dy"; - else if (deriv == dz) - return "dz"; else if (deriv == dxdx) return "dxdx"; + else if (deriv == dxdy) + return "dxdy"; else if (deriv == dydy) return "dydy"; +#if CH_SPACEDIM == 3 + else if (deriv == dz) + return "dz"; else if (deriv == dzdz) return "dzdz"; - else if (deriv == dxdy) - return "dxdy"; else if (deriv == dxdz) return "dxdz"; else if (deriv == dydz) return "dydz"; +#endif else return ""; } diff --git a/Source/AMRInterpolator/DerivativeSetup.hpp b/Source/AMRInterpolator/DerivativeSetup.hpp old mode 100644 new mode 100755 index 9d22c1c65..6b5cf0b4c --- a/Source/AMRInterpolator/DerivativeSetup.hpp +++ b/Source/AMRInterpolator/DerivativeSetup.hpp @@ -12,14 +12,16 @@ const Derivative Derivative::LOCAL; const Derivative Derivative::dx(0); const Derivative Derivative::dy(1); -const Derivative Derivative::dz(2); const Derivative Derivative::dxdx(0, 0); +const Derivative Derivative::dxdy(0, 1); const Derivative Derivative::dydy(1, 1); -const Derivative Derivative::dzdz(2, 2); -const Derivative Derivative::dxdy(0, 1); +#if CH_SPACEDIM == 3 +const Derivative Derivative::dz(2); const Derivative Derivative::dxdz(0, 2); const Derivative Derivative::dydz(1, 2); +const Derivative Derivative::dzdz(2, 2); +#endif #endif /* DERIVATIVESETUP_HPP_ */ diff --git a/Source/AMRInterpolator/InterpSource.hpp b/Source/AMRInterpolator/InterpSource.hpp index 4ef749cb8..2c9b8c951 100644 --- a/Source/AMRInterpolator/InterpSource.hpp +++ b/Source/AMRInterpolator/InterpSource.hpp @@ -17,13 +17,12 @@ #include "UsingNamespace.H" // Abstrace base class to get the FABs out of an AMRLevel -class InterpSource +template class InterpSource { public: virtual const LevelData &getLevelData( const VariableType var_type = VariableType::evolution) const = 0; - virtual bool - contains(const std::array &point) const = 0; + virtual bool contains(const std::array &point) const = 0; virtual void fillAllGhosts(const VariableType var_type = VariableType::evolution) = 0; }; diff --git a/Source/AMRInterpolator/Lagrange.hpp b/Source/AMRInterpolator/Lagrange.hpp index c54aa13a6..199ada2ea 100644 --- a/Source/AMRInterpolator/Lagrange.hpp +++ b/Source/AMRInterpolator/Lagrange.hpp @@ -9,9 +9,9 @@ #include "InterpSource.hpp" #include -template class Lagrange +template class Lagrange { - const InterpSource &m_source; + const InterpSource &m_source; bool m_verbosity; struct Stencil; @@ -41,10 +41,10 @@ template class Lagrange // Helper function to generate tensor product weights // Argument 'dim' is used for recursion over dimensions. pair, std::vector> - generateStencil(const std::array &deriv, - const std::array &dx, - const std::array &evalCoord, - const IntVect &nearest, int dim = CH_SPACEDIM - 1); + generateStencil(const std::array &deriv, + const std::array &dx, + const std::array &evalCoord, + int dim = N_DIMS - 1); std::vector m_interp_points; std::vector m_interp_weights; @@ -56,13 +56,17 @@ template class Lagrange multiset m_interp_pos; public: - Lagrange(const InterpSource &source, bool verbosity = false); + Lagrange(const InterpSource &source, bool verbosity = false); - void setup(const std::array &deriv, - const std::array &dx, - const std::array &evalCoord, - const IntVect &nearest); - double interpData(const FArrayBox &fab, int comp); + // evalCoord is in 'index' coordinates, not physical coordinates + void setup(const std::array &deriv, + const std::array &dx, + const std::array &evalCoord); + + // any class with a method: + // Real get(const IntVect &a_iv, int a_comp) const + template + double interpData(const GeneralArrayBox &fab, int comp = 0); const static string TAG; }; diff --git a/Source/AMRInterpolator/Lagrange.impl.hpp b/Source/AMRInterpolator/Lagrange.impl.hpp index 48bc7ec68..8b0cb99ba 100644 --- a/Source/AMRInterpolator/Lagrange.impl.hpp +++ b/Source/AMRInterpolator/Lagrange.impl.hpp @@ -6,8 +6,8 @@ #ifndef LAGRANGE_IMPL_HPP_ #define LAGRANGE_IMPL_HPP_ -template -const string Lagrange::TAG = "\x1b[36;1m[Lagrange]\x1b[0m "; +template +const string Lagrange::TAG = "\x1b[36;1m[Lagrange]\x1b[0m "; /* Finite difference weight generation algorithm * @@ -20,9 +20,9 @@ const string Lagrange::TAG = "\x1b[36;1m[Lagrange]\x1b[0m "; * routine: - change type of c1,c2,c3 to 'double' - replace '0', 'i', 'j' in the * annotated lines with 'grid[0]', 'grid[i]', 'grid[j]'. */ -template -Lagrange::Stencil::Stencil(int width, int deriv, double dx, - double point_offset) +template +Lagrange::Stencil::Stencil(int width, int deriv, double dx, + double point_offset) : m_width(width), m_deriv(deriv), m_dx(dx), m_point_offset(point_offset) { int c1 = 1; @@ -102,17 +102,17 @@ Lagrange::Stencil::Stencil(int width, int deriv, double dx, } } -template -bool Lagrange::Stencil::operator==( - const Lagrange::Stencil &rhs) const +template +bool Lagrange::Stencil::operator==( + const Lagrange::Stencil &rhs) const { return (rhs.m_width == m_width) && (rhs.m_deriv == m_deriv) && (rhs.m_point_offset == m_point_offset) && (rhs.dx == m_dx); } -template -bool Lagrange::Stencil::isSameAs(int width, int deriv, double dx, - double point_offset) const +template +bool Lagrange::Stencil::isSameAs(int width, int deriv, double dx, + double point_offset) const { return (width == m_width) && (deriv == m_deriv) && (dx == m_dx) && (point_offset == m_point_offset); @@ -120,10 +120,10 @@ bool Lagrange::Stencil::isSameAs(int width, int deriv, double dx, /* STENCIL ACCESSOR */ -template -typename Lagrange::Stencil -Lagrange::getStencil(int width, int deriv, double dx, - double point_offset) +template +typename Lagrange::Stencil +Lagrange::getStencil(int width, int deriv, double dx, + double point_offset) { for (typename stencil_collection_t::iterator it = m_memoized_stencils.begin(); @@ -142,8 +142,8 @@ Lagrange::getStencil(int width, int deriv, double dx, Stencil(width, deriv, dx, point_offset)); } -template -const double &Lagrange::Stencil::operator[](unsigned int i) const +template +const double &Lagrange::Stencil::operator[](unsigned int i) const { CH_assert(i < m_width); return m_weights[i]; @@ -151,26 +151,26 @@ const double &Lagrange::Stencil::operator[](unsigned int i) const /* LAGRANGE TENSOR PRODUCT LOGIC */ -template -Lagrange::Lagrange(const InterpSource &source, bool verbosity) +template +Lagrange::Lagrange(const InterpSource &source, + bool verbosity) : m_source(source), m_verbosity(verbosity) { } -template -void Lagrange::setup(const std::array &deriv, - const std::array &dx, - const std::array &evalCoord, - const IntVect &nearest) +template +void Lagrange::setup(const std::array &deriv, + const std::array &dx, + const std::array &evalCoord) { pair, std::vector> result = - generateStencil(deriv, dx, evalCoord, nearest); + generateStencil(deriv, dx, evalCoord); m_interp_points = result.first; m_interp_weights = result.second; /* pout() << TAG << "Stencil: coord = { "; - for (int i = 0; i < CH_SPACEDIM; ++i) + for (int i = 0; i < N_DIMS; ++i) { pout() << evalCoord[i] << " "; } @@ -183,8 +183,9 @@ void Lagrange::setup(const std::array &deriv, */ } -template -double Lagrange::interpData(const FArrayBox &fab, int comp) +template +template +double Lagrange::interpData(const GeneralArrayBox &fab, int comp) { /* m_interp_neg.clear(); @@ -236,13 +237,11 @@ double Lagrange::interpData(const FArrayBox &fab, int comp) return accum; } -template +template pair, std::vector> -Lagrange::generateStencil( - const std::array &deriv, - const std::array &dx, - const std::array &evalCoord, const IntVect &nearest, - int dim) +Lagrange::generateStencil( + const std::array &deriv, const std::array &dx, + const std::array &evalCoord, int dim) { std::vector out_points; std::vector out_weights; @@ -266,9 +265,13 @@ Lagrange::generateStencil( int points_min = Order + deriv[dim]; int points_max = Order + deriv[dim]; - std::array interp_coord = evalCoord; - int candidate = nearest[dim]; - int grown_direction = (nearest[dim] - evalCoord[dim] < 0) ? DOWN : UP; + std::array interp_coord = evalCoord; + + // TF: assume nearest point is at 'std::round(evalCoord[dim])' + // this used to be an argument, but I think this assumption should always + // hold + int candidate = std::round(evalCoord[dim]); + int grown_direction = (candidate - evalCoord[dim] < 0) ? DOWN : UP; while ((can_grow[DOWN] || can_grow[UP]) && (points_max - points_min < Order + deriv[dim])) @@ -330,7 +333,7 @@ Lagrange::generateStencil( { // Descend to the next dimension pair, std::vector> sub_result = - generateStencil(deriv, dx, interp_coord, nearest, dim - 1); + generateStencil(deriv, dx, interp_coord, dim - 1); std::vector &sub_points = sub_result.first; std::vector &sub_weights = sub_result.second; diff --git a/Source/AMRInterpolator/QuinticConvolution.hpp b/Source/AMRInterpolator/QuinticConvolution.hpp index c01a6337a..ea92d574d 100644 --- a/Source/AMRInterpolator/QuinticConvolution.hpp +++ b/Source/AMRInterpolator/QuinticConvolution.hpp @@ -9,22 +9,27 @@ #include "InterpSource.hpp" #include -class QuinticConvolution +template class QuinticConvolution { - const InterpSource &m_source; + const InterpSource &m_source; bool m_verbosity; std::vector m_interp_points; std::vector m_interp_weights; public: - QuinticConvolution(const InterpSource &source, bool verbosity = false); - - void setup(const std::array &deriv, - const std::array &dx, - const std::array &evalCoord, - const IntVect &nearest); - double interpData(const FArrayBox &fab, int comp); + QuinticConvolution(const InterpSource &source, + bool verbosity = false); + + // evalCoord is in 'index' coordinates, not physical coordinates + void setup(const std::array &deriv, + const std::array &dx, + const std::array &evalCoord); + + // any class with a method: + // Real get(const IntVect &a_iv, int a_comp) const + template + double interpData(const GeneralArrayBox &fab, int comp = 0); const static string TAG; }; diff --git a/Source/AMRInterpolator/QuinticConvolution.impl.hpp b/Source/AMRInterpolator/QuinticConvolution.impl.hpp index 38c9783fa..b9c57a8a6 100644 --- a/Source/AMRInterpolator/QuinticConvolution.impl.hpp +++ b/Source/AMRInterpolator/QuinticConvolution.impl.hpp @@ -6,26 +6,29 @@ #ifndef QUINTICCONVOLUTION_IMPL_HPP_ #define QUINTICCONVOLUTION_IMPL_HPP_ -const string QuinticConvolution::TAG = "\x1b[36;1m[QuinticConvolution]\x1b[0m "; +template +const string QuinticConvolution::TAG = + "\x1b[36;1m[QuinticConvolution]\x1b[0m "; -QuinticConvolution::QuinticConvolution(const InterpSource &source, - bool verbosity) +template +QuinticConvolution::QuinticConvolution( + const InterpSource &source, bool verbosity) : m_source(source), m_verbosity(verbosity) { - CH_assert(CH_SPACEDIM <= 3); + CH_assert(N_DIMS <= 3); } -void QuinticConvolution::setup(const std::array &deriv, - const std::array &dx, - const std::array &evalCoord, - const IntVect &nearest) +template +void QuinticConvolution::setup( + const std::array &deriv, const std::array &dx, + const std::array &evalCoord) { m_interp_points.clear(); m_interp_weights.clear(); - double weights_1d[CH_SPACEDIM][6]; + double weights_1d[N_DIMS][6]; - for (int dim = 0; dim < CH_SPACEDIM; ++dim) + for (int dim = 0; dim < N_DIMS; ++dim) { double s = evalCoord[dim] - floor(evalCoord[dim]); @@ -98,16 +101,18 @@ void QuinticConvolution::setup(const std::array &deriv, } } - std::array interp_coord; + std::array interp_coord; -#if CH_SPACEDIM >= 3 +#if N_DIMS >= 3 for (int z = 0; z < 6; ++z) { interp_coord[2] = floor(evalCoord[2]) + z - 2; #endif +#if N_DIMS >= 2 for (int y = 0; y < 6; ++y) { interp_coord[1] = floor(evalCoord[1]) + y - 2; +#endif for (int x = 0; x < 6; ++x) { @@ -117,17 +122,27 @@ void QuinticConvolution::setup(const std::array &deriv, m_interp_points.push_back(IntVect(D_DECL6( interp_coord[0], interp_coord[1], interp_coord[2], interp_coord[3], interp_coord[4], interp_coord[5]))); - m_interp_weights.push_back(D_TERM6(weights_1d[0][x], - *weights_1d[1][y], - *weights_1d[2][z], , , )); +#if N_DIMS >= 3 + m_interp_weights.push_back(weights_1d[0][x] * weights_1d[1][y] * + weights_1d[2][z]); +#elif N_DIMS >= 2 + m_interp_weights.push_back(weights_1d[0][x] * weights_1d[1][y]); +#else + m_interp_weights.push_back(weights_1d[0][x]); +#endif } +#if N_DIMS >= 2 } -#if CH_SPACEDIM >= 3 +#endif +#if N_DIMS >= 3 } #endif } -double QuinticConvolution::interpData(const FArrayBox &fab, int comp) +template +template +double QuinticConvolution::interpData(const GeneralArrayBox &fab, + int comp) { long double accum = 0.0; diff --git a/Source/AMRInterpolator/SimpleArrayBox.hpp b/Source/AMRInterpolator/SimpleArrayBox.hpp new file mode 100644 index 000000000..86ea592f8 --- /dev/null +++ b/Source/AMRInterpolator/SimpleArrayBox.hpp @@ -0,0 +1,75 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef SIMPLEARRAYBOX +#define SIMPLEARRAYBOX + +#include "CH_assert.H" +#include "MayDay.H" + +// Global Array Box class to interpolate N_DIM dimensional data +// where 'm_f' is a flat array over the dimensions +// Created to replace an FArrayBox in interpolation ::get methods (e.g. +// Lagrange) +template class SimpleArrayBox +{ + // coordinates (u,v,f) + std::array m_points_per_dir; + std::vector m_f; + std::array m_is_periodic; + + int apply_periodic(const IntVect &a_iv, int dir) const + { + int idx = a_iv[dir]; + if (idx < 0 || idx >= m_points_per_dir[dir]) + { + if (m_is_periodic[dir]) + { + while (idx < 0) + idx += m_points_per_dir[dir]; + while (idx >= m_points_per_dir[dir]) + idx -= m_points_per_dir[dir]; + } + else + { + pout() << "Direction " << dir << " invalid in IntVect " << a_iv + << std::endl; + MayDay::Error("[SimpleArrayBox] Trying to access index out of " + "valid range."); + } + } + return idx; + } + + public: + // IntVect will be CH_SPACEDIM dimension, but ignore the lasts components if + // N_DIMS < CH_SPACEDIM + // the 'comp' argument doesn't matter, always assume 'm_f' is what matters + Real get(const IntVect &a_iv, int a_comp) const + { + int global_idx = apply_periodic(a_iv, N_DIMS - 1); + + for (int i = N_DIMS - 2; i >= 0; --i) + global_idx = + global_idx * m_points_per_dir[i] + apply_periodic(a_iv, i); + + return m_f[global_idx]; + } + + SimpleArrayBox(std::array a_points_per_dir, + std::vector a_f, + std::array a_is_periodic = {false}) + : m_points_per_dir(a_points_per_dir), m_f(a_f), + m_is_periodic(a_is_periodic) + { + CH_assert(N_DIMS <= CH_SPACEDIM); + int total = 1.; + for (int i = 0; i < N_DIMS; ++i) + total *= m_points_per_dir[i]; + CH_assert(m_f.size() == total); + } +}; + +#endif /* SIMPLEARRAYBOX */ diff --git a/Source/AMRInterpolator/SimpleInterpSource.hpp b/Source/AMRInterpolator/SimpleInterpSource.hpp new file mode 100644 index 000000000..07f8f8d43 --- /dev/null +++ b/Source/AMRInterpolator/SimpleInterpSource.hpp @@ -0,0 +1,46 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef SIMPLEINTERPSOURCE_H_ +#define SIMPLEINTERPSOURCE_H_ + +#include "InterpSource.hpp" + +// Simple interpolation source where all grid points are in the same "Box" +// Relevant to be able to use Interpolation classes by themselves +template class SimpleInterpSource : public InterpSource +{ + LevelData m_fake; + + std::array m_points_per_dir; + std::array m_is_periodic; + + public: + SimpleInterpSource(std::array a_points_per_dir, + std::array a_is_periodic = {false}) + : m_points_per_dir(a_points_per_dir), m_is_periodic(a_is_periodic) + { + } + + const LevelData & + getLevelData(const VariableType var_type = VariableType::evolution) const + { + return m_fake; + }; + bool contains(const std::array &point) const + { + bool in = true; + for (int idir = 0; idir < N_DIMS; ++idir) + { + if (!m_is_periodic[idir] && + (point[idir] < 0. || point[idir] > m_points_per_dir[idir] - 1)) + in = false; + } + return in; + }; + void fillAllGhosts(const VariableType var_type = VariableType::evolution){}; +}; + +#endif /* SIMPLEINTERPSOURCE_H_ */ diff --git a/Source/AMRInterpolator/SphericalGeometry.hpp b/Source/AMRInterpolator/SphericalGeometry.hpp index 2645fe8d9..8293dcd85 100644 --- a/Source/AMRInterpolator/SphericalGeometry.hpp +++ b/Source/AMRInterpolator/SphericalGeometry.hpp @@ -6,10 +6,17 @@ #ifndef SPHERICALGEOMETRY_HPP_ #define SPHERICALGEOMETRY_HPP_ +#if CH_SPACEDIM != 3 +#error "This file should only be included for 3+1D simulations" +#endif + // Chombo includes +#include "AlwaysInline.hpp" #include "MayDay.H" // Other includes +#include "AlwaysInline.hpp" +#include "MayDay.H" #include #include #include @@ -22,7 +29,7 @@ //! u = theta, v = phi class SphericalGeometry { - private: + protected: std::array m_center; public: @@ -31,37 +38,58 @@ class SphericalGeometry { } + ALWAYS_INLINE double get_domain_u_min() const { return 0.; } + ALWAYS_INLINE double get_domain_u_max() const { return M_PI; } + ALWAYS_INLINE double get_domain_v_min() const { return 0.; } + ALWAYS_INLINE double get_domain_v_max() const { return 2. * M_PI; } + //! returns the grid spacing in theta - inline double du(int a_num_points_theta) const + ALWAYS_INLINE double du(int a_num_points_theta) const { - return M_PI / (double)(a_num_points_theta - 1); + return (get_domain_u_max() - get_domain_u_min()) / + (double)(a_num_points_theta - 1); } //! returns the grid spacing in phi - inline double dv(int a_num_points_phi) const + ALWAYS_INLINE double dv(int a_num_points_phi) const { - return 2.0 * M_PI / ((double)a_num_points_phi); + return (get_domain_v_max() - get_domain_v_min()) / + ((double)a_num_points_phi); } //! returns the theta coordinate associated to the theta/u index - inline double u(int a_itheta, int a_num_points_theta) const + ALWAYS_INLINE double u(int a_itheta, int a_num_points_theta) const { return a_itheta * du(a_num_points_theta); } //! returns the phi coordinate associated to the phi/v index - inline double v(int a_iphi, int a_num_points_phi) const + ALWAYS_INLINE double v(int a_iphi, int a_num_points_phi) const { return a_iphi * dv(a_num_points_phi); } - inline bool is_u_periodic() const { return false; } - inline bool is_v_periodic() const { return true; } + ALWAYS_INLINE bool is_u_periodic() const { return false; } + ALWAYS_INLINE bool is_v_periodic() const { return true; } + + ALWAYS_INLINE std::string param_name() const { return "r"; } + + ALWAYS_INLINE std::string u_name() const { return "theta"; } + + ALWAYS_INLINE std::string v_name() const { return "phi"; } + + //! returns the area element on a sphere with radius a_radius at the point + //! (a_theta, a_phi) + ALWAYS_INLINE double area_element(double a_radius, double a_theta, + double a_phi) const + { + return a_radius * a_radius * sin(a_theta); + } //! returns the Cartesian coordinate in direction a_dir with specified //! radius, theta and phi. - inline double get_grid_coord(int a_dir, double a_radius, double a_theta, - double a_phi) const + ALWAYS_INLINE double get_grid_coord(int a_dir, double a_radius, + double a_theta, double a_phi) const { switch (a_dir) { @@ -75,20 +103,6 @@ class SphericalGeometry MayDay::Error("SphericalGeometry: Direction not supported"); } } - - //! returns the area element on a sphere with radius a_radius at the point - //! (a_theta, a_phi) - inline double area_element(double a_radius, double a_theta, - double a_phi) const - { - return a_radius * a_radius * sin(a_theta); - } - - inline std::string param_name() const { return "r"; } - - inline std::string u_name() const { return "theta"; } - - inline std::string v_name() const { return "phi"; } }; #endif /* SPHERICALGEOMETRY_HPP_ */ diff --git a/Source/ApparentHorizonFinder/AHData.hpp b/Source/ApparentHorizonFinder/AHData.hpp new file mode 100644 index 000000000..c273ad82b --- /dev/null +++ b/Source/ApparentHorizonFinder/AHData.hpp @@ -0,0 +1,88 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef _AHDATA_HPP_ +#define _AHDATA_HPP_ + +#include "Derivative.hpp" +#include "DimensionDefinitions.hpp" // make sure GR_SPACEDIM exists +#include "InterpolationQuery.hpp" +#include "Tensor.hpp" + +#include // memset + +// The d prefix refers to partial derivatives wrt Cartesian coordinates + +template struct AHData +{ + // vars user might want to export + std::map vars; + std::map> d1; + std::map> d2; + + void set_vars(InterpolationQuery &query, int var, key k, VariableType type, + int total) + { + vars[k].resize(total); + query.addComp(var, &vars[k][0], Derivative::LOCAL, type); + } + void set_d1(InterpolationQuery &query, int var, key k, VariableType type, + int total) + { + for (int j = 0; j < CH_SPACEDIM; ++j) + d1[k][j].resize(total); + + query.addComp(var, &d1[k][0][0], Derivative::dx, type) + .addComp(var, &d1[k][1][0], Derivative::dy, type); +#if CH_SPACEDIM == 3 + query.addComp(var, &d1[k][2][0], Derivative::dz, type); +#endif + } + void set_d2(InterpolationQuery &query, int var, key k, VariableType type, + int total) + { + for (int j = 0; j < CH_SPACEDIM * (CH_SPACEDIM + 1) / 2; ++j) + d2[k][j].resize(total); + + query.addComp(var, &d2[k][0][0], Derivative::dxdx, type) + .addComp(var, &d2[k][1][0], Derivative::dxdy, type); +#if CH_SPACEDIM == 3 + query.addComp(var, &d2[k][2][0], Derivative::dxdz, type) + .addComp(var, &d2[k][3][0], Derivative::dydy, type) + .addComp(var, &d2[k][4][0], Derivative::dydz, type) + .addComp(var, &d2[k][5][0], Derivative::dzdz, type); +#elif CH_SPACEDIM == 2 + query.addComp(var, &d2[k][2][0], Derivative::dydy, type); +#endif + } +}; + +template +AHData +get_AHData_idx(int idx, const AHData> a_data) +{ + AHData data; + + for (auto &vec : a_data.vars) + data.vars[vec.first] = (vec.second[idx]); + + for (auto &vec : a_data.d1) + { + data.d1[vec.first] = {0.}; + for (int i = 0; i < CH_SPACEDIM; ++i) + data.d1[vec.first][i] = vec.second[i][idx]; + } + + for (auto &vec : a_data.d2) + { + data.d2[vec.first] = {0.}; + for (int i = 0; i < CH_SPACEDIM * (CH_SPACEDIM + 1) / 2; ++i) + data.d2[vec.first][i] = vec.second[i][idx]; + } + + return data; +} + +#endif /* _AHDATA_HPP_ */ diff --git a/Source/ApparentHorizonFinder/AHDeriv.hpp b/Source/ApparentHorizonFinder/AHDeriv.hpp new file mode 100644 index 000000000..4c0fe5883 --- /dev/null +++ b/Source/ApparentHorizonFinder/AHDeriv.hpp @@ -0,0 +1,48 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef _AHDERIV_HPP_ +#define _AHDERIV_HPP_ + +#include "DimensionDefinitions.hpp" // make sure GR_SPACEDIM exists + +#include // memset + +#define DWIDTH 5 +#define DDWIDTH 6 + +//! class to store 1st and 2nd derivatives of 'F' +struct AHDeriv +{ + double duF; + double duduF; +#if CH_SPACEDIM == 3 + double dvF; + double dvdvF; + double dudvF; +#endif + + int du_stencil_start; + int dudu_stencil_start; +#if CH_SPACEDIM == 3 + int dv_stencil_start; + int dvdv_stencil_start; +#endif + + double du_weights[DWIDTH]; + double dudu_weights[DDWIDTH]; +#if CH_SPACEDIM == 3 + double dv_weights[DWIDTH]; + double dvdv_weights[DDWIDTH]; +#endif + + AHDeriv() + { + // force all (double) elements of AHDeriv to be 0 + memset(this, 0, sizeof(AHDeriv)); + } +}; + +#endif /* _AHDERIV_HPP_ */ diff --git a/Source/ApparentHorizonFinder/AHFinder.cpp b/Source/ApparentHorizonFinder/AHFinder.cpp new file mode 100644 index 000000000..a2a9214e9 --- /dev/null +++ b/Source/ApparentHorizonFinder/AHFinder.cpp @@ -0,0 +1,540 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifdef USE_AHFINDER + +#include "AHFinder.hpp" + +#include "AHInterpolation.hpp" +#include "ApparentHorizon.hpp" + +AHFinder::~AHFinder() +{ + // destroy horizon pointers and finalize PETSc + for (auto &ah : m_apparent_horizons) + delete ah; + PETSc_finalize(); +} + +int AHFinder::add_ah(const AHSurfaceGeometry &a_coord_system, + double a_initial_guess, const AHFinder::params &a_params, + const std::string &a_stats, const std::string &a_coords, + bool solve_first_step) +{ + return add_ah(a_coord_system, a_initial_guess, a_params, + AHFunction::params(), a_stats, a_coords, solve_first_step); +} + +int AHFinder::add_ah(const AHSurfaceGeometry &a_coord_system, + double a_initial_guess, const AHFinder::params &a_params, + const typename AHFunction::params &a_func_params, + const std::string &a_stats, const std::string &a_coords, + bool solve_first_step) +{ + if (!AHFinder::m_initialized) + AHFinder::PETSc_initialize(a_params.num_ranks); + + // determine how many AH there are already + const int num_ah = m_apparent_horizons.size(); + + AHInterpolation interp(a_coord_system, + m_interpolator); + + m_apparent_horizons.push_back( + new ApparentHorizon( + interp, a_initial_guess, a_params, a_func_params, + a_stats + std::to_string(num_ah + 1), + a_coords + std::to_string(num_ah + 1) + "_", solve_first_step)); + m_merger_pairs.push_back({-1, -1}); + + return num_ah; +} + +int AHFinder::add_ah_merger(int ah1, int ah2, const params &a_params) +{ + CH_assert(a_params.track_center); // center tracking must be on for mergers + + int num_ah = m_apparent_horizons.size(); + CH_assert(ah1 >= 0 && ah1 < num_ah); + CH_assert(ah2 >= 0 && ah2 < num_ah); + CH_assert(ah1 != ah2); + + double initial_guess_merger; + std::array origin_merger; + bool do_solve = solve_merger(ah1, ah2, initial_guess_merger, origin_merger); + + // assume same coordiante system between merging horizons + AHSurfaceGeometry coord_system = + m_apparent_horizons[ah1]->get_ah_interp().get_coord_system(); + coord_system.set_origin(origin_merger); + + std::string stats = m_apparent_horizons[ah1]->m_stats; + std::string coords = m_apparent_horizons[ah1]->m_coords; + // remove number of AH1 from file names + stats = stats.substr(0, stats.size() - std::to_string(ah1).size()); + coords = coords.substr(0, coords.size() - std::to_string(ah1).size() - 1); + + auto &function_to_optimize_params = m_apparent_horizons[ah1]->m_func_params; + + int num = add_ah(coord_system, initial_guess_merger, a_params, + function_to_optimize_params, stats, coords, do_solve); + m_merger_pairs[num] = {ah1, ah2}; + + return num; +} + +void AHFinder::solve(double a_dt, double a_time, double a_restart_time) +{ + CH_TIME("AHFinder::solve"); + + enum + { + NOT_SOLVED, + TOO_FAR, + SKIPPED, + SOLVED + }; + + // solve if it has never converged before OR if has already converged in the + // past this means it will stop solving as soon as it stops converging but + // keeps trying (with same initial guess) if never found one + // (e.g. in the BinaryBH case, AH1 and AH2 stop solving once they + // disappear, leaving the merged AH alone from then on) + // (e.g. in the ScalarField case, only after a few time steps does an AH + // form, and from then on it keeps solving forever) + + const int num_ah = m_apparent_horizons.size(); + + // skip if AH 0 is not supposed to be solved + // assumes all AHs have same 'solve_interval' + if (num_ah == 0 || !m_apparent_horizons[0]->do_solve(a_dt, a_time)) + return; + + // the mess below is done so that AH are searched for iteratively + // taking into account merger priorities + // (a merger will only be solved if it's 'parents' have already been solved) + int ahs_solved = 0; + // sets whether or not AH[i] has already been solved + // 0 - not solved; 1 - too far away; 2 - skipped; 3 - solved + int *ah_solved = + new int[num_ah]; // with '-g' compilation flag was complaining that int + // ah_solved[num_ah] was invalid + for (int i = 0; i < num_ah; ++i) + ah_solved[i] = NOT_SOLVED; + + int max_attempts = 100, attempt = 0; + while (ahs_solved < num_ah) + { + if (max_attempts < attempt++) + { + pout() << "Reached maximum number of attempts in AH solving loop." + << std::endl; + break; + } + + for (int i = 0; i < num_ah; ++i) + { + if (ah_solved[i] != NOT_SOLVED) + continue; // continue if already solved + + // if is a merger and has not yet converged + if (m_merger_pairs[i].first >= 0 && + !m_apparent_horizons[i]->has_been_found()) + { + auto pair = m_merger_pairs[i]; + + if (!((ah_solved[pair.first] == SKIPPED || + ah_solved[pair.first] == SOLVED) && + (ah_solved[pair.second] == SKIPPED || + ah_solved[pair.second] == SOLVED))) + continue; // continue if one of the 'parents' was not + // solved/skipped yet + + if (!(m_apparent_horizons[pair.first]->has_been_found() && + m_apparent_horizons[pair.second]->has_been_found())) + { + ++ahs_solved; + ah_solved[i] = SKIPPED; + continue; // skip if one of the parents was never found yet + } + + if (ah_solved[pair.first] == TOO_FAR || + ah_solved[pair.second] == TOO_FAR) + { // if one of the 'parents' is a merger too far away to be + // solved, skip children too + if (m_apparent_horizons[i]->m_params.verbose > NONE) + { + pout() << "Skipping BH #" << i << std::endl; + } + ++ahs_solved; + ah_solved[i] = TOO_FAR; + } + + double initial_guess_merger; + std::array center_merger; + bool do_solve = + solve_merger(pair.first, pair.second, initial_guess_merger, + center_merger); + + // if distance is too large, ignore this one and move on + if (!do_solve) + { + ++ahs_solved; + ah_solved[i] = TOO_FAR; + continue; + } + + // if it already converged, don't do anything; + // no need to change the initial guess, as it is the same since + // the beginning; + // if one was skipped don't update the center (because it will + // not be central anymore) + if (!m_apparent_horizons[i]->get_converged() && + ah_solved[pair.first] == SOLVED && + ah_solved[pair.second] == SOLVED) + { + m_apparent_horizons[i]->set_origin(center_merger); + // m_apparent_horizons[i]->set_initial_guess( + // initial_guess_merger); // should still be the same + } + } + + // solve if it converged last time or if is has never been found in + // the past + if (m_apparent_horizons[i]->good_to_go(a_dt, a_time)) + { + if (m_apparent_horizons[i]->m_params.verbose > NONE) + pout() << "Solving AH #" << i << std::endl; + + m_apparent_horizons[i]->solve(a_dt, a_time, a_restart_time); + + ++ahs_solved; + ah_solved[i] = SOLVED; + } + else + { + ++ahs_solved; + ah_solved[i] = SKIPPED; + } + } + } + + delete ah_solved; +} + +bool AHFinder::need_diagnostics(double a_dt, double a_time) const +{ + bool out = false; + for (auto &ah : m_apparent_horizons) + out |= + ah->m_params.extra_contain_diagnostic && ah->do_print(a_dt, a_time); + return out; +} + +bool AHFinder::solve_merger(int ah1, int ah2, double &initial_guess_merger, + std::array ¢er_merger) +{ + // SKIP if 'parents' not yet close enough + auto AH1 = m_apparent_horizons[ah1]; + auto AH2 = m_apparent_horizons[ah2]; + auto center1 = AH1->get_center(); + auto center2 = AH2->get_center(); + + double initial_guess_sum = + (AH1->get_initial_guess() + AH2->get_initial_guess()); + + // some tests revealed it was about 1.2 * initial_guess_sum, but 1.5 to + // ensure we don't catch the inner AH + double merger_pre_factor = std::max(AH1->m_params.merger_pre_factor, + AH2->m_params.merger_pre_factor); + initial_guess_merger = merger_pre_factor * initial_guess_sum; + + // update center of merged, otherwise it does it by + // itself in solve + FOR1(a) + { + if (!AH1->get_ah_interp().get_interpolator()->get_boundary_reflective( + Side::Lo, a)) + center_merger[a] = (center1[a] + center2[a]) / 2.; + else + center_merger[a] = 0.; + } + + // check if distance is too big + double distance = + sqrt((center1[0] - center2[0]) * (center1[0] - center2[0]) + + (center1[1] - center2[1]) * (center1[1] - center2[1]) +#if CH_SPACEDIM == 3 + + (center1[2] - center2[2]) * (center1[2] - center2[2]) +#endif + ); + + double merger_search_factor = std::max(AH1->m_params.merger_search_factor, + AH2->m_params.merger_search_factor); + + double min_distance = merger_search_factor * 4.0 * initial_guess_sum; + + bool do_solve = false; + + if (AH1->has_been_found() && AH2->has_been_found()) + { + do_solve = merger_search_factor <= 0. || distance <= min_distance; + + if (do_solve) + // radius of guess is bigger than AH distance + CH_assert(min_distance < initial_guess_merger); + + if (AH1->m_params.verbose > NONE) + { + pout() << "BHs #" << ah1 << " and #" << ah2 + << " at distance = " << distance; + // if distance is too large, ignore this one and move on + if (!do_solve) + pout() << " > minimum distance = " << min_distance + << ". Skipping solve for merger..."; + + pout() << std::endl; + } + } + + return do_solve; +} + +void AHFinder::params::read_params(GRParmParse &pp, const ChomboParameters &a_p) +{ + pp.load("AH_num_ranks", num_ranks, 0); // 0 means "all" + pp.load("AH_num_points_u", num_points_u); +#if CH_SPACEDIM == 3 + pp.load("AH_num_points_v", num_points_v); +#endif + pp.load("AH_solve_interval", solve_interval, 1); + pp.load("AH_print_interval", print_interval, 1); + pp.load("AH_track_center", track_center, true); + pp.load("AH_predict_origin", predict_origin, track_center); + // can't predict if center is not being tracked + CH_assert(!(predict_origin && !track_center)); + + pp.load("AH_level_to_run", level_to_run, 0); + CH_assert(level_to_run <= a_p.max_level && + level_to_run > -(a_p.max_level + 1)); + if (level_to_run < 0) // if negative, count backwards + level_to_run += a_p.max_level + 1; + + pp.load("AH_start_time", start_time, 0.0); + pp.load("AH_give_up_time", give_up_time, -1.0); + + pp.load("AH_merger_search_factor", merger_search_factor, 1.); + pp.load("AH_merger_pre_factor", merger_pre_factor, 1.); + + pp.load("AH_allow_re_attempt", allow_re_attempt, false); + pp.load("AH_max_fails_after_lost", max_fails_after_lost, 0); + pp.load("AH_stop_if_max_fails", stop_if_max_fails, false); + + pp.load("AH_verbose", verbose, (int)AHFinder::MIN); + pp.load("AH_print_geometry_data", print_geometry_data, false); + pp.load("AH_re_solve_at_restart", re_solve_at_restart, false); + + // sanity checks + CH_assert(solve_interval > 0 && print_interval > 0); + CH_assert(level_to_run >= 0 && level_to_run <= a_p.max_level); + + // if box division ends up with size less than 3, PETSc will + // complain (this only gives an estimate of the box side) + int size = 1; +#if CH_MPI + MPI_Comm_size(Chombo_MPI::comm, &size); +#endif + size = std::min(num_ranks, size); +#if CH_SPACEDIM == 3 + CH_assert(num_points_u > 0 && num_points_v > 0); + CH_assert(num_points_u / sqrt(size) >= 3); // make sure for size 'u' + CH_assert(num_points_v / sqrt(size) >= 3); // make sure for size 'v' +#elif CH_SPACEDIM == 2 + CH_assert(num_points_u > 0); + CH_assert(num_points_u / size >= 3); // make sure for size 'u' +#endif + + // load vars to write to coord files + num_extra_vars = 0; + extra_contain_diagnostic = false; + + int AH_num_write_vars; + pp.load("AH_num_write_vars", AH_num_write_vars, 0); + if (AH_num_write_vars > 0) + { + std::vector AH_write_var_names(AH_num_write_vars, ""); + pp.load("AH_write_vars", AH_write_var_names, AH_num_write_vars); + for (const std::string &full_name : AH_write_var_names) + { + std::string var_name = full_name; + + // variable names might start with "d1_" or "d2_" to indicate + // the user wants derivatives + int der_type = 0; + std::string der = var_name.substr(0, 3); + if (der == "d1_") + { + der_type = 1; + var_name = var_name.substr(3); + } + else if (der == "d2_") + { + der_type = 2; + var_name = var_name.substr(3); + } + + // first assume write_var is a normal evolution var + int var = UserVariables::variable_name_to_enum(var_name); + VariableType var_type = VariableType::evolution; + if (var < 0) + { + // if not an evolution var check if it's a diagnostic var + var = DiagnosticVariables::variable_name_to_enum(var_name); + if (var < 0) + { + // it's neither :( + pout() << "Variable with name " << var_name + << " not found.\n"; + } + else + { + var_type = VariableType::diagnostic; + extra_contain_diagnostic = true; + } + } + if (var >= 0) + { + extra_vars[full_name] = + std::tuple(var, var_type, der_type); + if (der_type == 0) + num_extra_vars += 1; + else if (der_type == 1) + num_extra_vars += CH_SPACEDIM; + else // if (der_type == 2) + num_extra_vars += CH_SPACEDIM * (CH_SPACEDIM + 1) / 2; + } + } + } +} + +///////////////////////////////////////////////////////// +// PETSc control methods +///////////////////////////////////////////////////////// + +void AHFinder::set_num_ranks(int a_num_ranks) +{ + if (m_initialized) + return; + +#ifdef CH_MPI + if (m_mpi_group != MPI_GROUP_NULL) + MPI_Group_free(&m_mpi_group); + + if (m_mpi_comm != MPI_COMM_NULL) + MPI_Comm_free(&m_mpi_comm); + + if (a_num_ranks > 0) // otherwise use the whole Chombo communicator + { + // don't make more ranks than there are processes + int size; + MPI_Comm_size(Chombo_MPI::comm, &size); + a_num_ranks = std::min(a_num_ranks, size); + + int rank; + MPI_Comm_rank(Chombo_MPI::comm, &rank); + if (rank == 0) + std::cout << "Using PETSc with " << a_num_ranks << " ranks" + << std::endl; + + MPI_Group MPI_GROUP_WORLD; + MPI_Comm_group(Chombo_MPI::comm, &MPI_GROUP_WORLD); + + // (TF): for things like 'SmallDataIO', rank 0 of Chombo_MPI::comm + // is the one that prints (not rank 0 from PETSC_COMM_WORLD) so pay + // attention before taking rank 0 of Chombo out of PETSc. As of + // 26/05/2019, I tested with 'int range[3] = { 1, a_num_ranks , 1 }' + // and all the AH stuff was compatible with doing so + // Not true anymore (19/08/2020) due to how + // ApparentHorizon::write_coords_file uses rank 0 of PETSc to transfer + // data and to write, but SmallDataIO uses rank 0 of Chombo to write. + int range[3] = {0, a_num_ranks - 1, 1}; + MPI_Group_range_incl(MPI_GROUP_WORLD, 1, + reinterpret_cast(&range), &m_mpi_group); + MPI_Group_free(&MPI_GROUP_WORLD); + + MPI_Comm_create(Chombo_MPI::comm, m_mpi_group, &m_mpi_comm); + } +#endif +} + +//! initialize PETSc and its MPI sub-communicator +PetscErrorCode AHFinder::PETSc_initialize(int a_num_ranks) +{ + set_num_ranks(a_num_ranks); + + PetscErrorCode err = 0; + if (!m_initialized) + { +#ifdef CH_MPI + if (m_mpi_comm != MPI_COMM_NULL) + PETSC_COMM_WORLD = m_mpi_comm; + else // use Chombo communicator if no 'set_num_ranks' was called (or + // if called with 'a_num_ranks'<=0) + PETSC_COMM_WORLD = Chombo_MPI::comm; +#endif + + if (AHFinder::is_rank_active()) + err = PetscInitializeNoArguments(); + + if (!err) + m_initialized = true; + } + return err; +} + +//! finalize PETSc +PetscErrorCode AHFinder::PETSc_finalize() +{ + PetscErrorCode err = 0; + if (m_initialized) + { + if (AHFinder::is_rank_active()) + err = PetscFinalize(); + + if (!err) + m_initialized = false; + } + return err; +} + +//! true if part of PETSc MPI sub-communicator +bool AHFinder::is_rank_active() +{ +#ifdef CH_MPI + if (m_mpi_group == MPI_GROUP_NULL) + return true; + else + { + int rank; + MPI_Group_rank(m_mpi_group, &rank); + return rank != MPI_UNDEFINED; + } +#else + return true; +#endif +} + +///////////////////////////////////////////////////////// +// initialize some "static" variables of AHFinder class +///////////////////////////////////////////////////////// + +bool AHFinder::m_initialized = false; + +#ifdef CH_MPI +MPI_Group AHFinder::m_mpi_group = MPI_GROUP_NULL; +MPI_Comm AHFinder::m_mpi_comm = MPI_COMM_NULL; +#endif + +#endif diff --git a/Source/ApparentHorizonFinder/AHFinder.hpp b/Source/ApparentHorizonFinder/AHFinder.hpp new file mode 100644 index 000000000..54011f721 --- /dev/null +++ b/Source/ApparentHorizonFinder/AHFinder.hpp @@ -0,0 +1,233 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef _AHFINDER_HPP_ +#define _AHFINDER_HPP_ + +// Chombo includes +#include "SPMD.H" //Chombo_MPI::comm + +// Other includes: + +// included so that user can re-define default AHSurfaceGeometry or AHFunction +#include "UserVariables.hpp" + +// define SurfaceGeometry of AHFinder +#ifndef AHSurfaceGeometry +#if CH_SPACEDIM == 3 +#include "AHSphericalGeometry.hpp" +#define AHSurfaceGeometry AHSphericalGeometry +#elif CH_SPACEDIM == 2 +#include "AHStringGeometry.hpp" +#define AHSurfaceGeometry AHStringGeometry +#endif +#endif + +// default to expansion +#ifndef AHFunction +#include "AHFunctions.hpp" +#define AHFunction ExpansionFunction +#endif + +#ifdef CH_MPI +#include +#endif + +#include + +#include "AHData.hpp" +#include "AHDeriv.hpp" +#include "AHGeometryData.hpp" +#include "AMRInterpolator.hpp" +#include "BoundaryConditions.hpp" +#include "ChomboParameters.hpp" +#include "Lagrange.hpp" +#include "VariableType.hpp" + +// Chombo namespace +#include "UsingNamespace.H" + +// declare classes, define them later +template class AHInterpolation; +template class ApparentHorizon; + +//! Class to manage AHs and its mergers + control PETSc MPI sub-communicator +class AHFinder +{ + public: + struct params + { + int num_ranks; //!< number of ranks for PETSc sub-communicator (default + //!< 0, which is 'all') + int num_points_u; //!< number of points for 2D coordinate grid +#if CH_SPACEDIM == 3 + int num_points_v; //!< number of points for 2D coordinate grid +#endif + int solve_interval; //!< same as checkpoint_interval, for + //!< ApparentHorizon::solve (default 1) + int print_interval; //!< same as solve_interval, but for prints (default + //!< 1) + bool track_center; //!< whether or not to update the center + //!< (set to false if you know it won't move) + //!< (default true) + bool predict_origin; //!< whether or not to estimate where the next + //!< origin will be at (default = track_center) + + int level_to_run; // if negative, it will count backwards (e.g. -1 is + // 'last level') (default 0) + + double start_time; //!< time after which AH can start (default 0.) + //!< Useful for ScalarField collapse + double give_up_time; //!< stop if at this time nothing was found + //!< (<0 to never, which is default) + //!< Useful for ScalarField collapse + + //! mergers will be searched when distance between 'parent' BHs is + //! distance < merger_search_factor * 4. * (AH_initial_guess_1 + + //! AH_initial_guess_2) should be roughly '2M' for initial guess at M/2 + //! (set to non-positive to 'always search') + double merger_search_factor; // typically around ~0.8*[2(M1+M2)] + // (default 1) + //! initial guess for merger is 'merger_pre_factor * (AH_initial_guess_1 + //! + AH_initial_guess_2)' + double merger_pre_factor; // typically ~0.6*[M1+M2] (default 1. to + // avoid finding the inner AH) + bool allow_re_attempt; //!< re-attempt with initial guess if + //!< previous convergence failed (default false) + int max_fails_after_lost; //!< number of time steps to try again after + //!< (-1 to never) the AH was lost + //!< (default is < 0) + + int verbose; //!< print information during execution (default is 1) + + bool print_geometry_data; //!< print metric and extrinsic + //!< curvature of the AHs (default false) + + bool re_solve_at_restart; //!< whether or not to re-run even if AH + //!< already exists (useful in order to be + //!< able to provide an initial guess and + //!< re-run the AHFinder on that time step) + //!< (default false) + + bool stop_if_max_fails; //! breaks the run if AH doesn't converge + //! 'max_fails_after_lost' times or if + //! 'give_up_time' is reached without + //! convergence (default is 'false') + + std::map> + extra_vars; //! extra vars to write to coordinates file () + int num_extra_vars; // total number of extra vars (!=extra_vars.size() + // as derivative count for multiple vars) + + bool extra_contain_diagnostic; // not a parameter (set internally) + + void read_params(GRParmParse &pp, const ChomboParameters &a_p); + }; + + enum verbose_level + { + NONE, + MIN, // minimal + SOME, // a bit technical + MORE // debug + }; + + private: + //! if this AH is supposed to track the formation of a merger + //! this pair indicates the indices of the 2 AHs in the vector + //! 'm_apparent_horizons' + std::vector> m_merger_pairs; + AMRInterpolator> *m_interpolator; //!< The interpolator pointer + + std::vector *> + m_apparent_horizons; //!< public in case user wants to solve by himself + + public: + AHFinder(){}; + ~AHFinder(); + + ALWAYS_INLINE ApparentHorizon * + get(unsigned AH_i) + { + CH_assert(AH_i < m_apparent_horizons.size()); + return m_apparent_horizons[AH_i]; + } + + //! returns the index of the AH in m_apparent_horizons + int + add_ah(const AHSurfaceGeometry &a_coord_system, + double a_initial_guess, //!< Initial guess for radius (or whatever + //!< coordinate you're solving for) + const params &a_params, //!< set of AH parameters + const std::string &a_stats = + "stats_AH", //!< name for stats file with + //!< area, spin and AH origin/center + const std::string &a_coords = + "coords_AH", //!< name for coords file with AH + //!< coordinates at each time step) + bool solve_first_step = true //!< whether or not to solve if t=0 + ); + + //! returns the index of the AH in m_apparent_horizons + //! allows for personalized optimizer that finds zero of function + //! 'AHFunction' (that can have some ::params) + int add_ah(const AHSurfaceGeometry &a_coord_system, double a_initial_guess, + const params &a_params, + const typename AHFunction::params &a_func_params, + const std::string &a_stats = "stats", + const std::string &a_coords = "coords", + bool solve_first_step = true); + + // returns the index of the AH in m_apparent_horizons + int add_ah_merger(int ah1, int ah2, const params &a_params); + + //! Find AH; Calculate area and spin; Update center; Print outputs + void solve(double a_dt, double a_time, double a_restart_time); + + bool need_diagnostics(double a_dt, double a_time) + const; //!< is any AH printing diagnostics? Useful if Diagnostics need + //!< to be calculated + + ALWAYS_INLINE void + set_interpolator(AMRInterpolator> *a_interpolator) + { + m_interpolator = a_interpolator; + } + + private: + //! returns false if 'parent' AHs are too far + //! sets the initial guess and the origin for the merger + bool solve_merger(int ah1, int ah2, double &initial_guess_merger, + std::array &origin_merged); + + ///////////////////////////////////////////////////////// + ///////////////////////////////////////////////////////// + ///////////////// PETSc control methods ///////////////// + ///////////////////////////////////////////////////////// + ///////////////////////////////////////////////////////// + + static bool m_initialized; //!< is initialized? + +#ifdef CH_MPI + static MPI_Group m_mpi_group; //!< set of MPI ranks for PETSc + static MPI_Comm m_mpi_comm; //!< MPI sub-communicator +#endif + + //! define number of ranks of PETSc sub-communicator + static void set_num_ranks(int a_num_ranks); + + //! initialize PETSc and its MPI sub-communicator + static PetscErrorCode PETSc_initialize(int a_num_ranks); + + //! finalize PETSc + static PetscErrorCode PETSc_finalize(); + + public: + //! true if part of PETSc MPI sub-communicator + static bool is_rank_active(); +}; // namespace AHFinder + +#endif /* _AHFINDER_HPP_ */ diff --git a/Source/ApparentHorizonFinder/AHFunctionDefault.hpp b/Source/ApparentHorizonFinder/AHFunctionDefault.hpp new file mode 100644 index 000000000..e5f626eaf --- /dev/null +++ b/Source/ApparentHorizonFinder/AHFunctionDefault.hpp @@ -0,0 +1,56 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef _AHFUNCTION_HPP_ +#define _AHFUNCTION_HPP_ + +#include "AlwaysInline.hpp" +#include "Tensor.hpp" + +///////////////////////////////////////////////////////// +// Predefined optimization functions +///////////////////////////////////////////////////////// + +struct AHFunctionDefault +{ + // set the minimum/maximum variable needed (and same for 1st and 2nd + // derivatives) + static ALWAYS_INLINE int vars_min() { return 0; } + static ALWAYS_INLINE int vars_max() { return -1; } + static ALWAYS_INLINE int d1_vars_min() { return 0; } + static ALWAYS_INLINE int d1_vars_max() { return -1; } + static ALWAYS_INLINE int d2_vars_min() { return 0; } + static ALWAYS_INLINE int d2_vars_max() { return -1; } + + // set how many and what variables to print for the geometry when + // 'AH_print_geometry_data' is on + // (useful to print more complex variables as the non-conformal metric or + // the extrinsic curvature, which are not elementary variables in + // BSSN/CCZ4) + static ALWAYS_INLINE int num_write_vars() { return 0; } + static ALWAYS_INLINE void write_headers(std::string *vec) {} + ALWAYS_INLINE void write_vars(double *vec) {} + + // metric to use when calculating the spin and area of the AH + // (override with spacetime metric) + ALWAYS_INLINE const Tensor<2, double> get_metric() const + { + // cartesian flat metric + Tensor<2, double> g; + FOR1(i) { g[i][i] = 1.; } + return g; + } + + // case of reduced Cartoon methods that have an extra metric component +#if GR_SPACEDIM != CH_SPACEDIM // hd - higher dimensions + ALWAYS_INLINE double get_metric_hd() const { return 1.; } +#endif + + struct params // no params needed + { + }; +}; + +#endif /* _AHFUNCTION_HPP_ */ diff --git a/Source/ApparentHorizonFinder/AHFunctions.hpp b/Source/ApparentHorizonFinder/AHFunctions.hpp new file mode 100644 index 000000000..420bcd769 --- /dev/null +++ b/Source/ApparentHorizonFinder/AHFunctions.hpp @@ -0,0 +1,327 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef _AHFUNCTIONS_HPP_ +#define _AHFUNCTIONS_HPP_ + +// Chombo includes +#include "CH_Timer.H" + +// Other includes +#include "AHData.hpp" +#include "AHDeriv.hpp" +#include "AHGeometryData.hpp" +#include "DimensionDefinitions.hpp" // make sure GR_SPACEDIM exists +#include "TensorAlgebra.hpp" +#include "UserVariables.hpp" +#include // sqrt + +#include "AHFunctionDefault.hpp" + +// Chombo namespace +#include "UsingNamespace.H" + +///////////////////////////////////////////////////////// +// Predefined optimization functions +///////////////////////////////////////////////////////// + +struct ExpansionFunction : AHFunctionDefault +{ + ////////////////////////////////// + // needed to calculate expansion: + + // induced metric + Tensor<2, double> g; + Tensor<2, double> g_UU; + Tensor<3, double> dg; + + // extrinsic curvature + Tensor<2, double> K; + double trK; + +// case of reduced Cartoon methods that have an extra metric component and +// require the coordinates to calculate the expansion +#if GR_SPACEDIM != CH_SPACEDIM + // hd - higher dimensions + Tensor<1, double> x; // coordinates + double g_hd; + Tensor<1, double> dg_hd; + + double Kww; // not needed to calculate expansion, but may be needed for ut + ALWAYS_INLINE double get_metric_hd() const { return g_hd; } +#endif + ////////////////////////////////// + + // only require variables up to Aij (chi, hij, K, Aij) + // this assumes c_Theta comes after the last of Aij + // (and like this the code is generic for 2D and 3D) + static ALWAYS_INLINE int vars_min() { return c_chi; } + static ALWAYS_INLINE int vars_max() { return c_Theta - 1; } + // Derivatives required only for the metric component (chi and hij) + // this assumes c_K comes after the last hij + // (and like this the code is generic for 2D and 3D) + static ALWAYS_INLINE int d1_vars_min() { return c_chi; } + static ALWAYS_INLINE int d1_vars_max() { return c_K - 1; } + + ALWAYS_INLINE const Tensor<2, double> get_metric() const { return g; } + + static int num_write_vars() + { + int num_components = CH_SPACEDIM * (CH_SPACEDIM + 1) / 2 * 2; +#if GR_SPACEDIM != CH_SPACEDIM + num_components += 2; // hww and Kww +#endif + return num_components; + } + static void write_headers(std::string *vec) + { + int el = 0; + for (int i = 0; i < CH_SPACEDIM; ++i) + for (int j = i; j < CH_SPACEDIM; ++j) + vec[el++] = "g" + std::to_string(i + 1) + std::to_string(j + 1); +#if GR_SPACEDIM != CH_SPACEDIM + vec[el++] = "gww"; +#endif + + for (int i = 0; i < CH_SPACEDIM; ++i) + for (int j = i; j < CH_SPACEDIM; ++j) + vec[el++] = "K" + std::to_string(i + 1) + std::to_string(j + 1); +#if GR_SPACEDIM != CH_SPACEDIM + vec[el++] = "Kww"; +#endif + } + void write_vars(double *vec) + { + int el = 0; + for (int i = 0; i < CH_SPACEDIM; ++i) + for (int j = i; j < CH_SPACEDIM; ++j) + vec[el++] = g[i][j]; +#if GR_SPACEDIM != CH_SPACEDIM + vec[el++] = g_hd; +#endif + for (int i = 0; i < CH_SPACEDIM; ++i) + for (int j = i; j < CH_SPACEDIM; ++j) + vec[el++] = K[i][j]; +#if GR_SPACEDIM != CH_SPACEDIM + vec[el++] = Kww; +#endif + } + + ExpansionFunction(const AHData &a_data, + const Tensor<1, double> &a_coords) + { + CH_TIME("ExpansionFunction::calculate_data"); + + // * --------------------------- + // * GR-RELATED DATA + // * --------------------------- + + int h[CH_SPACEDIM][CH_SPACEDIM]; + int A[CH_SPACEDIM][CH_SPACEDIM]; + + int comp_h = c_h11; + int comp_A = c_A11; + for (int i = 0; i < CH_SPACEDIM; ++i) + { + for (int j = i; j < CH_SPACEDIM; ++j) + { + h[i][j] = comp_h; + A[i][j] = comp_A; + if (i != j) + { + h[j][i] = comp_h; + A[j][i] = comp_A; + } + ++comp_h; + ++comp_A; + } + } + + const double chi = a_data.vars.at(c_chi); + trK = a_data.vars.at(c_K); + + // INVERSE METRIC + Tensor<2, double, CH_SPACEDIM> h_DD; + FOR2(i, j) { h_DD[i][j] = a_data.vars.at(h[i][j]); } + + Tensor<2, double, CH_SPACEDIM> h_UU = + TensorAlgebra::compute_inverse_sym(h_DD); + FOR2(i, j) { g_UU[i][j] = chi * h_UU[i][j]; } + + // Reconstructing ADM variables + Tensor<1, double, CH_SPACEDIM> dchi; + + FOR1(i) { dchi[i] = a_data.d1.at(c_chi)[i]; } + + for (int i = 0; i < CH_SPACEDIM; ++i) + { + for (int j = i; j < CH_SPACEDIM; ++j) + { + { + const double gij = h_DD[i][j] / (chi); + g[i][j] = gij; + g[j][i] = gij; + } + + { + const double Aij = a_data.vars.at(A[i][j]); + const double Kij = Aij / chi + trK * g[i][j] / GR_SPACEDIM; + K[i][j] = Kij; + K[j][i] = Kij; + } + + for (int k = 0; k < CH_SPACEDIM; ++k) + { + { + const double dhij = a_data.d1.at(h[i][j])[k]; + const double dgijk = + (dhij - (h_DD[i][j] * dchi[k]) / chi) / chi; + dg[i][j][k] = dgijk; + dg[j][i][k] = dgijk; + } + } + } + } + +// part for higher dimensions that use Cartoon Method +// Uli's paper does it for 3+1D (from 5D) - arxiv 1808.05834 +#if GR_SPACEDIM != CH_SPACEDIM + int comp_hww = + c_K - 1; // c_K-1 expected to be c_hww (is c_hww direct better?) + int comp_Aww = c_Aww; // is (c_Theta-1) more general? + Tensor<1, double, CH_SPACEDIM> dhww; + + x = a_coords; + + double hww = a_data.vars.at(comp_hww); + FOR1(a) { dhww[a] = a_data.d1.at(comp_hww)[a]; } + double Aww = a_data.vars.at(comp_Aww); + + g_hd = hww / chi; + FOR1(a) { dg_hd[a] = (dhww[a] - (hww * dchi[a]) / chi) / chi; } + + Kww = Aww / chi + trK * g_hd / GR_SPACEDIM; +#endif + } + + double get(const AHGeometryData &geo_data, const AHDeriv &deriv, + const params &a_params) const + { + // first calculate D_a L of 6.7.12 of Alcubierre for some level function + // L picking L = f - F(u,v) + // D_a L = d_a f - dF/du * du/dx^a - dF/dv * dv/dx^a + +#if CH_SPACEDIM == 3 + Tensor<1, double> s = {0.}; + FOR1(a) + { + s[a] = geo_data.df[a] - (deriv.duF * geo_data.du[a]) - + (deriv.dvF * geo_data.dv[a]); + } + + // just the partial derivative of the above + Tensor<2, double> ds = {0.}; + FOR2(a, b) + { + ds[a][b] = geo_data.ddf[a][b] - (deriv.duF * geo_data.ddu[a][b]) - + (deriv.dvF * geo_data.ddv[a][b]) - + (deriv.duduF * geo_data.du[a] * geo_data.du[b]) - + (deriv.dvdvF * geo_data.dv[a] * geo_data.dv[b]) - + (deriv.dudvF * (geo_data.du[a] * geo_data.dv[b] + + geo_data.du[b] * geo_data.dv[a])); + } +#elif CH_SPACEDIM == 2 + Tensor<1, double> s = {0.}; + FOR1(a) { s[a] = geo_data.df[a] - (deriv.duF * geo_data.du[a]); } + + // just the partial derivative of the above + Tensor<2, double> ds = {0.}; + FOR2(a, b) + { + ds[a][b] = geo_data.ddf[a][b] - (deriv.duF * geo_data.ddu[a][b]) - + (deriv.duduF * geo_data.du[a] * geo_data.du[b]); + } +#endif + + // calculate the norm. This is also the norm of s_U (s^a)) + double norm_s = 0.0; + FOR2(a, b) { norm_s += g_UU[a][b] * s[a] * s[b]; } + norm_s = sqrt(norm_s); + + // calculate Christoffels on this point (u,v,f) + Tensor<3, double> chris = {0.}; + FOR4(a, b, c, d) + { + chris[a][b][c] += + 0.5 * g_UU[a][d] * (dg[b][d][c] + dg[c][d][b] - dg[b][c][d]); + } + + // raise s_a + Tensor<1, double> s_U = {0.}; + FOR2(a, b) { s_U[a] += g_UU[a][b] * s[b]; } + + // S = the real 's' of Alcubierre + Tensor<1, double> S_U = {0.}; + FOR1(a) { S_U[a] = s_U[a] / norm_s; } + + // covariant derivatrive of s_a to use for DS + Tensor<2, double> Ds = {0.}; + FOR2(a, b) + { + Ds[a][b] = ds[a][b]; + FOR1(c) { Ds[a][b] -= chris[c][a][b] * s[c]; } + } + + // calculate D_i S^i and S^i S^j K_ij + double DiSi = 0.; + double Kij_dot_Si_Sj = 0.; + FOR2(a, b) + { + DiSi += (g_UU[a][b] - S_U[a] * S_U[b]) * Ds[a][b] / norm_s; + Kij_dot_Si_Sj += S_U[a] * S_U[b] * K[a][b]; + } + + // Calculation of expansion - as in (6.7.9) of Alcubierre + double expansion = DiSi - trK + Kij_dot_Si_Sj; + + // part from extra dimensions in the case of Cartoon methods +#if GR_SPACEDIM != CH_SPACEDIM + expansion += (GR_SPACEDIM - CH_SPACEDIM) * S_U[CH_SPACEDIM - 1] / + x[CH_SPACEDIM - 1]; + FOR1(a) + { + expansion += + (GR_SPACEDIM - CH_SPACEDIM) * 0.5 * dg_hd[a] / g_hd * S_U[a]; + } +#endif + + return expansion; + } +}; + +// to look for chi contours +struct ChiContourFunction : AHFunctionDefault +{ + double chi; + + static int vars_min() { return c_chi; } + static int vars_max() { return c_chi; } + + ChiContourFunction(const AHData &a_data, + const Tensor<1, double> &a_coords) + { + chi = a_data.vars.at(c_chi); + } + + using params = double; // chi contour to look for + double get(const AHGeometryData &geo_data, const AHDeriv &deriv, + const params &a_params) const + { + double look_for_chi_contour = a_params; + return chi - look_for_chi_contour; + } +}; + +#endif /* _AHFUNCTIONS_HPP_ */ diff --git a/Source/ApparentHorizonFinder/AHGeometryData.hpp b/Source/ApparentHorizonFinder/AHGeometryData.hpp new file mode 100644 index 000000000..67ae91ba7 --- /dev/null +++ b/Source/ApparentHorizonFinder/AHGeometryData.hpp @@ -0,0 +1,51 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef _AHGEOMETRYDATA_HPP_ +#define _AHGEOMETRYDATA_HPP_ + +#include "DimensionDefinitions.hpp" // make sure GR_SPACEDIM exists +#include "Tensor.hpp" + +#include // memset + +// The d prefix refers to partial derivatives wrt Cartesian coordinates + +struct AHGeometryData +{ + + // jacobian + Tensor<1, double> du; +#if CH_SPACEDIM == 3 + Tensor<1, double> dv; +#endif + Tensor<1, double> df; + + // hessian + Tensor<2, double> ddu; +#if CH_SPACEDIM == 3 + Tensor<2, double> ddv; +#endif + Tensor<2, double> ddf; + + // inverse jacobian + Tensor<1, double> dxdu; +#if CH_SPACEDIM == 3 + Tensor<1, double> dxdv; +#endif + Tensor<1, double> dxdf; + + // inverse hessian + // never needed + + AHGeometryData() + { + // TF: no need to set to 0 + // force all (double) elements of AHGeometryData to be 0 + // memset(this, 0, sizeof(AHGeometryData)); + } +}; + +#endif /* _AHGEOMETRYDATA_HPP_ */ diff --git a/Source/ApparentHorizonFinder/AHInterpolation.hpp b/Source/ApparentHorizonFinder/AHInterpolation.hpp new file mode 100644 index 000000000..273902a21 --- /dev/null +++ b/Source/ApparentHorizonFinder/AHInterpolation.hpp @@ -0,0 +1,138 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef _AHINTERPOLATION_HPP_ +#define _AHINTERPOLATION_HPP_ + +#include "AHData.hpp" +#include "AHGeometryData.hpp" +#include "AMRInterpolator.hpp" +#include "Lagrange.hpp" + +#include +#include + +//! Class used for interpolation of the variables needed to calculate expansion +//! with the data from a given 'SurfaceGeometry' +template class AHInterpolation +{ + private: + SurfaceGeometry m_coord_system; + AMRInterpolator> *m_interpolator; + + // variables of AH in 'SurfaceGeometry' + // (in Spherical Coordinates Class 'AHSphericalCoords', + // they correspond to 'theta', 'phi' and 'sqrt(radius)') + std::vector m_u; +#if CH_SPACEDIM == 3 + std::vector m_v; +#endif + std::vector m_f; + + // variables of AH in cartesian (for AMR interpolation) + std::vector m_x; + std::vector m_y; +#if CH_SPACEDIM == 3 + std::vector m_z; +#endif + + AHData> m_data; + AHData> m_extra; + + std::array m_coord_min, + m_coord_max; //!< maximum and minimum of level 0 box, used in + //!< 'fit_in_grid' + + //! when PETSc tried to diverge out of the grid, this doesn't let him do so + //! it forces him to stay on the grid, causing non-convergence + bool fit_in_grid(double &x, double &y +#if CH_SPACEDIM == 3 + , + double &z +#endif + ); + + public: + AHInterpolation(const SurfaceGeometry &a_coordSystem, + AMRInterpolator> *a_interpolator); + + const AMRInterpolator> *get_interpolator() const; + const SurfaceGeometry &get_coord_system() const; + + // several of the methods below just call the correspondent method of the + // CoordSystem, as these are needed in the 'ApparentHorizon' class + bool is_u_periodic() const; + double get_domain_u_min() const; //!< lower bound of (u,v) coordinates + double get_domain_u_max() const; //!< upper bound of (u,v) coordinates +#if CH_SPACEDIM == 3 + bool is_v_periodic() const; + double get_domain_v_min() const; //!< lower bound of (u,v) coordinates + double get_domain_v_max() const; //!< upper bound of (u,v) coordinates +#endif + + std::vector get_labels() const; //!< get all names (u, v, f) + + void set_origin( + const std::array &); //!< set origin of CoordSystem + const std::array & + get_origin() const; //!< get origin of CoordSystem + + void refresh_interpolator(); //!< refresh AMRInterpolator 'm_interpolator' + + double get_grid_coord(int a_dir, double f, double u +#if CH_SPACEDIM == 3 + , + double v +#endif + ) const; //!< transform from spherical to cartesian + + //! returns whether any pointis outside of grid (==> diverging) + bool set_coordinates(const std::vector &f, + const std::vector &u, +#if CH_SPACEDIM == 3 + const std::vector &v, +#endif + double add_epsilon = 0.); + const AHGeometryData get_geometry_data(int idx) const; + const Tensor<1, double> get_cartesian_coords(int idx) const; + const AHData get_data(int idx) const; + + //! verify origin +- initial guess is inside the grid + bool is_in_grid(const std::array &a_origin, + double a_initial_guess); + + //! triplet of functions to be used together in blocks of code that require + //! PETSc AND AMRInterpolator to interpolate + //! 'keep_interpolating_if_inactive' returns 'true' immediately for PETSc + //! ranks for non-PETSc ranks, it loops in a sequence of 'interpolate()' + //! waiting for PETSc ranks to call 'interpolate()'' as well (as this needs + //! to be called by all Chombo processes) can be aborted by calling + //! 'break_interpolation_loop()' for PETSc ranks + /** Example of usage: + * if(m_geom.keep_interpolating_if_inactive()) + * { + * (... PETSc code that calls 'set_coordinates()'...) + * m_geom.break_interpolation_loop(); + * } + */ + bool keep_interpolating_if_inactive(); + void + break_interpolation_loop() const; //!< see 'keep_interpolating_if_inactive' + //!< (this one could be static) + //! returns whether or not to keep interpolating. See + //! 'keep_interpolating_if_inactive' + //! Again, all Chombo_MPI:comm need to run 'interpolate' for it to work (if + //! some are not part of PETSc, they still need to run 'interpolate' on the + //! side) + int interpolate(); + + void interpolate_extra_vars( + std::map> extra_vars); + const AHData get_extra_data(int idx) const; +}; + +#include "AHInterpolation.impl.hpp" + +#endif /* _AHINTERPOLATION_HPP_ */ diff --git a/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp b/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp new file mode 100644 index 000000000..4406769f4 --- /dev/null +++ b/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp @@ -0,0 +1,517 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef _AHINTERPOLATION_HPP_ +#error "This file should only be included through AHInterpolation.hpp" +#endif + +#ifndef _AHINTERPOLATION_IMPL_HPP_ +#define _AHINTERPOLATION_IMPL_HPP_ + +#include "AHFinder.hpp" +#include "DimensionDefinitions.hpp" // make sure GR_SPACEDIM exists +#include "TensorAlgebra.hpp" + +template +AHInterpolation::AHInterpolation( + const SurfaceGeometry &a_coord_system, + AMRInterpolator> *a_interpolator) + : m_coord_system(a_coord_system), m_interpolator(a_interpolator) +{ + // below: + // determine maximum and minimum physical coordinate of the grid + // (so that 'fit_in_grid' knows when PETSc has diverged out of the grid and + // doesn't let him do so, + // as it would cause an error in the AMRInterpolator) + + const Box &domainBox = const_cast(m_interpolator->getAMR()) + .getAMRLevels()[0] + ->problemDomain() + .domainBox(); + const IntVect &small_end = domainBox.smallEnd(); + const IntVect &big_end = domainBox.bigEnd(); + + const std::array &coarsest_dx = + m_interpolator->get_coarsest_dx(); + const std::array &coarsest_origin = + m_interpolator->get_coarsest_origin(); + + // set coordinate minimum and maximum as ONE cells before the boundary + for (unsigned i = 0; i < CH_SPACEDIM; ++i) + { + if (m_interpolator->get_boundary_reflective(Side::Lo, i)) + { + m_coord_min[i] = + -(big_end[i] - 1) * coarsest_dx[i] - coarsest_origin[i]; + } + else if (m_interpolator->get_boundary_periodic(i)) + { + m_coord_min[i] = small_end[i] * coarsest_dx[i]; + } + else + { + m_coord_min[i] = + (small_end[i] + 1) * coarsest_dx[i] + coarsest_origin[i]; + } + + if (m_interpolator->get_boundary_reflective(Side::Hi, i)) + { + m_coord_max[i] = + (2 * big_end[i] - 1) * coarsest_dx[i] + coarsest_origin[i]; + } + else if (m_interpolator->get_boundary_periodic(i)) + { + m_coord_max[i] = (big_end[i] + 1) * coarsest_dx[i]; + } + else + { + m_coord_max[i] = + (big_end[i] - 1) * coarsest_dx[i] + coarsest_origin[i]; + } + // pout() << "m_coord = " << m_coord_min[i] << "\t" << m_coord_max[i] + // << " with coarse origin " << coarsest_origin[i] << " and dx " + // << coarsest_dx[i] << std::endl; + } +} + +template +const AMRInterpolator> * +AHInterpolation::get_interpolator() const +{ + return m_interpolator; +} + +template +const SurfaceGeometry & +AHInterpolation::get_coord_system() const +{ + return m_coord_system; +} + +template +bool AHInterpolation::is_u_periodic() const +{ + return m_coord_system.is_u_periodic(); +} +template +double AHInterpolation::get_domain_u_min() const +{ + return m_coord_system.get_domain_u_min(); +} +template +double AHInterpolation::get_domain_u_max() const +{ + return m_coord_system.get_domain_u_max(); +} + +#if CH_SPACEDIM == 3 +template +bool AHInterpolation::is_v_periodic() const +{ + return m_coord_system.is_v_periodic(); +} +template +double AHInterpolation::get_domain_v_min() const +{ + return m_coord_system.get_domain_v_min(); +} +template +double AHInterpolation::get_domain_v_max() const +{ + return m_coord_system.get_domain_v_max(); +} +#endif + +template +std::vector +AHInterpolation::get_labels() const +{ + return + { + m_coord_system.u_name(), +#if CH_SPACEDIM == 3 + m_coord_system.v_name(), +#endif + m_coord_system.param_name() + }; +} + +template +void AHInterpolation::set_origin( + const std::array &origin) +{ + m_coord_system.set_origin(origin); +} + +template +const std::array & +AHInterpolation::get_origin() const +{ + return m_coord_system.get_origin(); +} + +template +bool AHInterpolation::is_in_grid( + const std::array &a_origin, double a_initial_guess) +{ + bool out_of_grid = false; + + double x = a_origin[0]; + double y = a_origin[1]; +#if CH_SPACEDIM == 3 + double z = a_origin[2]; +#endif + +#if CH_SPACEDIM == 3 + out_of_grid |= fit_in_grid(x, y, z); + + x = a_origin[0] - a_initial_guess; + out_of_grid |= fit_in_grid(x, y, z); + + x = a_origin[0] + a_initial_guess; + out_of_grid |= fit_in_grid(x, y, z); + + x = a_origin[0]; + y = a_origin[1] - a_initial_guess; + out_of_grid |= fit_in_grid(x, y, z); + + y = a_origin[1] + a_initial_guess; + out_of_grid |= fit_in_grid(x, y, z); + + y = a_origin[1]; + z = a_origin[2] - a_initial_guess; + out_of_grid |= fit_in_grid(x, y, z); + + z = a_origin[2] + a_initial_guess; + out_of_grid |= fit_in_grid(x, y, z); +#elif CH_SPACEDIM == 2 + out_of_grid |= fit_in_grid(x, y); + /* + x = a_origin[0] - a_initial_guess; + out_of_grid |= fit_in_grid(x, y); + + x = a_origin[0] + a_initial_guess; + out_of_grid |= fit_in_grid(x, y); + */ + y = a_origin[1] - a_initial_guess; + out_of_grid |= fit_in_grid(x, y); + + y = a_origin[1] + a_initial_guess; + out_of_grid |= fit_in_grid(x, y); +#endif + + return out_of_grid; +} +template +bool AHInterpolation::fit_in_grid(double &x, + double &y +#if CH_SPACEDIM == 3 + , + double &z +#endif +) +{ + CH_TIME("AHInterpolation::fit_in_grid"); + + bool out_of_grid = false; + + // if out of bounds, put back in grid + if (x < m_coord_min[0]) + { + out_of_grid = true; + x = m_coord_min[0]; + } + if (x > m_coord_max[0]) + { + out_of_grid = true; + x = m_coord_max[0]; + } + + if (y < m_coord_min[1]) + { + out_of_grid = true; + y = m_coord_min[1]; + } + if (y > m_coord_max[1]) + { + out_of_grid = true; + y = m_coord_max[1]; + } + +#if CH_SPACEDIM == 3 + if (z < m_coord_min[2]) + { + out_of_grid = true; + z = m_coord_min[2]; + } + if (z > m_coord_max[2]) + { + out_of_grid = true; + z = m_coord_max[2]; + } +#endif + + return out_of_grid; +} + +template +double AHInterpolation::get_grid_coord(int a_dir, + double f, + double u +#if CH_SPACEDIM == 3 + , + double v +#endif +) const +{ + CH_TIME("AHInterpolation::get_grid_coord"); + + return m_coord_system.get_grid_coord(a_dir, f, u +#if CH_SPACEDIM == 3 + , + v +#endif + ); +} + +//! triplet of functions to be used together in blocks of code that require +//! PETSc AND AMRInterpolator to interpolate 'keep_interpolating_if_inactive' +//! returns 'true' immediately for PETSc ranks for non-PETSc ranks, it loops in +//! a sequence of 'interpolate()' waiting for PETSc ranks to call +//! 'interpolate()'' as well (as this needs to be called by all Chombo +//! processes) can be aborted by calling 'break_interpolation_loop()' for PETSc +//! ranks +/** Example of usage: + * if(m_geom.keep_interpolating_if_inactive()) + * { + * (... PETSc code that calls 'set_coordinates()'...) + * m_geom.break_interpolation_loop(); + * } + */ +template +bool AHInterpolation::keep_interpolating_if_inactive() +{ + CH_TIME("AMRInterpolator::keep_interpolating_if_inactive"); + + if (!AHFinder::is_rank_active()) + { + int keep_interpolating = 1; + while (keep_interpolating) + keep_interpolating = interpolate(); + return false; + } + else + return true; +} + +template +void AHInterpolation::break_interpolation_loop() + const +{ + CH_TIME("AMRInterpolator::break_interpolation_loop"); + +#ifdef CH_MPI + // break "keep_interpolating" loop + int ZERO = 0; + int keep_interpolating = 0; + // this is also called in 'interpolate()', breaking the loop + MPI_Allreduce(&ZERO, &keep_interpolating, 1, MPI_INT, MPI_LAND, + Chombo_MPI::comm); +#endif +} +template +int AHInterpolation::interpolate() +{ + CH_TIME("AHInterpolation::interpolate"); + + // Code below used to allow PETSc to run in a sub-communicator. + // For that, PETSc processes run 'set_coordinates' and 'interpolate', while + // non-PETSc processes loop through 'interpolate'. If all cores run + // interpolate, 'MPI_Allreduce' will return 'keep_interpolating' as true. To + // exit the loop for non-PETSc cores, simply do on them a call to: + // MPI_Allreduce(&ZERO, &keep_interpolating, 1, MPI_INT, MPI_LAND, + // Chombo_MPI::comm); (this is done in 'break_interpolation_loop()') + + int keep_interpolating = 1; +#ifdef CH_MPI + int ONE = 1; + MPI_Allreduce(&ONE, &keep_interpolating, 1, MPI_INT, MPI_LAND, + Chombo_MPI::comm); +#endif + + if (!keep_interpolating) + return 0; + + const int n = m_x.size(); + + InterpolationQuery query(n); + query.setCoords(0, m_x.data()).setCoords(1, m_y.data()); +#if CH_SPACEDIM == 3 + query.setCoords(2, m_z.data()); +#endif + + for (int i = AHFunction::vars_min(); i <= AHFunction::vars_max(); ++i) + m_data.set_vars(query, i, i, VariableType::evolution, n); + for (int i = AHFunction::d1_vars_min(); i <= AHFunction::d1_vars_max(); ++i) + m_data.set_d1(query, i, i, VariableType::evolution, n); + for (int i = AHFunction::d2_vars_min(); i <= AHFunction::d2_vars_max(); ++i) + m_data.set_d2(query, i, i, VariableType::evolution, n); + + m_interpolator->interp(query); + + return 1; +} + +template +void AHInterpolation::refresh_interpolator() +{ + CH_TIME("AHInterpolation::refresh_interpolator"); + m_interpolator->refresh(); +} + +// 'set_coordinates' calls 'interpolate'. All Chombo_MPI:comm need to run +// 'interpolate' for it to work +template +bool AHInterpolation::set_coordinates( + const vector &f, const vector &u, +#if CH_SPACEDIM == 3 + const vector &v, +#endif + double add_epsilon) +{ + CH_TIME("AHInterpolation::set_coordinates"); + + CH_assert(f.size() == u.size()); +#if CH_SPACEDIM == 3 + CH_assert(u.size() == v.size()); +#endif + + const int n = f.size(); + + // resize vectors + m_f.resize(n); + m_u.resize(n); +#if CH_SPACEDIM == 3 + m_v.resize(n); +#endif + + m_x.resize(n); + m_y.resize(n); +#if CH_SPACEDIM == 3 + m_z.resize(n); +#endif + + bool out_of_grid = false; + + // Transform to Cartesian + for (int i = 0; i < n; ++i) + { + m_f[i] = f[i] + add_epsilon; + m_u[i] = u[i]; +#if CH_SPACEDIM == 3 + m_v[i] = v[i]; +#endif + +#if CH_SPACEDIM == 3 + m_x[i] = m_coord_system.get_grid_coord(0, m_f[i], m_u[i], m_v[i]); + m_y[i] = m_coord_system.get_grid_coord(1, m_f[i], m_u[i], m_v[i]); + m_z[i] = m_coord_system.get_grid_coord(2, m_f[i], m_u[i], m_v[i]); +#elif CH_SPACEDIM == 2 + m_x[i] = m_coord_system.get_grid_coord(0, m_f[i], m_u[i]); + m_y[i] = m_coord_system.get_grid_coord(1, m_f[i], m_u[i]); +#endif + + // don't let PETSc diverge to outside of the grid (this can happen if + // there is no BH) + out_of_grid |= fit_in_grid(m_x[i], m_y[i] +#if CH_SPACEDIM == 3 + , + m_z[i] +#endif + ); + } + + return out_of_grid; +} + +template +const AHGeometryData +AHInterpolation::get_geometry_data(int idx) const +{ + CH_TIME("AHInterpolation::get_geometry_data"); + + return m_coord_system.get_geometry_data(m_f[idx], m_u[idx] +#if CH_SPACEDIM == 3 + , + m_v[idx] +#endif + ); +} + +template +const Tensor<1, double> +AHInterpolation::get_cartesian_coords( + int idx) const +{ + return + { + m_x[idx], m_y[idx] +#if CH_SPACEDIM == 3 + , + m_z[idx] +#endif + }; +} + +template +const AHData +AHInterpolation::get_data(int idx) const +{ + return get_AHData_idx(idx, m_data); +} + +template +void AHInterpolation::interpolate_extra_vars( + std::map> extra_vars) +{ + CH_TIME("AHInterpolation::interpolate_extra_vars"); + + if (extra_vars.size() == 0) + return; + + int n = m_x.size(); + + InterpolationQuery query(n); + query.setCoords(0, m_x.data()).setCoords(1, m_y.data()); +#if CH_SPACEDIM == 3 + query.setCoords(2, m_z.data()); +#endif + + for (auto &var : extra_vars) + { + int var_enum = std::get<0>(var.second); + VariableType var_type = std::get<1>(var.second); + int der_type = std::get<2>(var.second); + + if (der_type == 0) + m_extra.set_vars(query, var_enum, var.first, var_type, n); + else if (der_type == 1) + m_extra.set_d1(query, var_enum, var.first, var_type, n); + else // if(der_type == 2) + m_extra.set_d2(query, var_enum, var.first, var_type, n); + } + + m_interpolator->interp(query); +} + +template +const AHData +AHInterpolation::get_extra_data(int idx) const +{ + return get_AHData_idx(idx, m_extra); +} + +#endif // _AHINTERPOLATION_IMPL_HPP_ \ No newline at end of file diff --git a/Source/ApparentHorizonFinder/AHSphericalGeometry.hpp b/Source/ApparentHorizonFinder/AHSphericalGeometry.hpp new file mode 100644 index 000000000..42de6ca54 --- /dev/null +++ b/Source/ApparentHorizonFinder/AHSphericalGeometry.hpp @@ -0,0 +1,138 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef AHSPHERICALGEOMETRY_HPP_ +#define AHSPHERICALGEOMETRY_HPP_ + +#if CH_SPACEDIM != 3 +#error "This file should only be included for 3+1D simulations" +#endif + +// Chombo includes +#include "CH_Timer.H" + +// Other includes +#include "AHGeometryData.hpp" +#include "AlwaysInline.hpp" +#include "SphericalGeometry.hpp" +#include +#include + +// Chombo namespace +#include "UsingNamespace.H" + +//! This SurfaceGeometry template class provides spherical shell geometry +//! implementation for the SurfaceExtraction class +//! u = theta, v = phi +class AHSphericalGeometry : public SphericalGeometry +{ + public: + AHSphericalGeometry(const std::array &a_origin) + : SphericalGeometry(a_origin) + { + } + + ALWAYS_INLINE void + set_origin(const std::array &a_origin) + { + m_center = a_origin; + } + ALWAYS_INLINE const std::array &get_origin() const + { + return m_center; + } + + AHGeometryData get_geometry_data(double r, double theta, double phi) const + { + CH_TIME("AHSphericalGeometry::get_geometry_data"); + + double costheta = cos(theta); + double sintheta = sin(theta); + double tantheta = tan(theta); + + double cosphi = cos(phi); + double sinphi = sin(phi); + + double cos2theta = cos(2. * theta); + double sin2theta = sin(2. * theta); + double cos2phi = cos(2. * phi); + double sin2phi = sin(2. * phi); + + AHGeometryData out; + + out.du[0] = (costheta * cosphi) / r; + out.du[1] = (costheta * sinphi) / r; + out.du[2] = -sintheta / r; + + out.dv[0] = (-sinphi) / (r * sintheta); + out.dv[1] = (cosphi) / (r * sintheta); + out.dv[2] = 0; + + double dfdx = sintheta * cosphi; + double dfdy = sintheta * sinphi; + double dfdz = costheta; + + out.ddu[0][0] = + (cos2theta * cosphi * cosphi - cos2phi) / (r * r * tantheta); + out.ddu[0][1] = ((cos2theta - 2.) * sin2phi) / (2. * r * r * tantheta); + out.ddu[0][2] = -(cos2theta * cosphi) / (r * r); + out.ddu[1][1] = + (cos2theta * sinphi * sinphi + cos2phi) / (r * r * tantheta); + out.ddu[1][2] = -(cos2theta * sinphi) / (r * r); + out.ddu[2][2] = (sin2theta) / (r * r); + out.ddu[1][0] = out.ddu[0][1]; + out.ddu[2][0] = out.ddu[0][2]; + out.ddu[2][1] = out.ddu[1][2]; + + out.ddv[0][0] = sin2phi / (r * r * sintheta * sintheta); + out.ddv[0][1] = -cos2phi / (r * r * sintheta * sintheta); + out.ddv[0][2] = 0.; + out.ddv[1][1] = -sin2phi / (r * r * sintheta * sintheta); + out.ddv[1][2] = 0.; + out.ddv[2][2] = 0.; + out.ddv[1][0] = out.ddv[0][1]; + out.ddv[2][0] = out.ddv[0][2]; + out.ddv[2][1] = out.ddv[1][2]; + + double ddfdxdx = + (costheta * costheta + sintheta * sintheta * sinphi * sinphi) / r; + double ddfdxdy = -(sintheta * sintheta * sinphi * cosphi) / r; + double ddfdxdz = -(sintheta * costheta * cosphi) / r; + double ddfdydy = + (costheta * costheta + sintheta * sintheta * cosphi * cosphi) / r; + double ddfdydz = -(sintheta * costheta * sinphi) / r; + double ddfdzdz = (sintheta * sintheta) / r; + + out.df[0] = dfdx; + out.df[1] = dfdy; + out.df[2] = dfdz; + + out.ddf[0][0] = ddfdxdx; + out.ddf[0][1] = ddfdxdy; + out.ddf[0][2] = ddfdxdz; + out.ddf[1][1] = ddfdydy; + out.ddf[1][2] = ddfdydz; + out.ddf[2][2] = ddfdzdz; + out.ddf[1][0] = out.ddf[0][1]; + out.ddf[2][0] = out.ddf[0][2]; + out.ddf[2][1] = out.ddf[1][2]; + + out.dxdu[0] = r * costheta * cosphi; + out.dxdu[1] = r * costheta * sinphi; + out.dxdu[2] = -r * sintheta; + + out.dxdv[0] = -r * sintheta * sinphi; + out.dxdv[1] = r * sintheta * cosphi; + out.dxdv[2] = 0.; + + out.dxdf[0] = sintheta * cosphi; + out.dxdf[1] = sintheta * sinphi; + out.dxdf[2] = costheta; + + return out; + } +}; + +#endif /* AHSPHERICALGEOMETRY_HPP_ */ diff --git a/Source/ApparentHorizonFinder/AHStringGeometry.hpp b/Source/ApparentHorizonFinder/AHStringGeometry.hpp new file mode 100644 index 000000000..4cb410733 --- /dev/null +++ b/Source/ApparentHorizonFinder/AHStringGeometry.hpp @@ -0,0 +1,109 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef _AHSPHERICALGEOMETRY_HPP_ +#define _AHSPHERICALGEOMETRY_HPP_ + +#if CH_SPACEDIM != 2 +#error "This file should only be included for 2+1D simulations" +#endif + +// Chombo includes +#include "CH_Timer.H" +#include "MayDay.H" + +// Other includes +#include "AHGeometryData.hpp" +#include "AlwaysInline.hpp" +#include + +// Chombo namespace +#include "UsingNamespace.H" + +//! Class for coordinate system to use +//! String assumed to be along in 'x' direction +//! Periodic in x coordinate, y direction may be reflective +class AHStringGeometry +{ + private: + double m_string_length; + std::array fake_origin = {0., 0.}; + + public: + AHStringGeometry(double a_string_length) : m_string_length(a_string_length) + { + } + + ALWAYS_INLINE void + set_origin(const std::array &a_origin) + { + } + ALWAYS_INLINE const std::array &get_origin() const + { + return fake_origin; + } + + ALWAYS_INLINE double get_domain_u_min() const { return 0.; } + ALWAYS_INLINE double get_domain_u_max() const { return m_string_length; } + + ALWAYS_INLINE bool is_u_periodic() const { return true; } + + ALWAYS_INLINE std::string param_name() const { return "y"; } + + ALWAYS_INLINE std::string u_name() const { return "x"; } + + ALWAYS_INLINE double get_grid_coord(int a_dir, double a_y, double a_x) const + { + switch (a_dir) + { + case (0): + return a_x; + case (1): + return a_y; + default: + MayDay::Error("AHStringGeometry: Direction not supported"); + } + } + + AHGeometryData get_geometry_data(double y, double x) const + { + CH_TIME("AHStringGeometry::get_geometry_data"); + + AHGeometryData out; + + out.du[0] = 1.; + out.du[1] = 0.; + + double dfdx = 0.; + double dfdy = 1.; + + out.ddu[0][0] = 0.; + out.ddu[0][1] = 0.; + out.ddu[1][1] = 0.; + out.ddu[1][0] = out.ddu[0][1]; + + double ddfdxdx = 0.; + double ddfdxdy = 0.; + double ddfdydy = 0.; + + out.df[0] = dfdx; + out.df[1] = dfdy; + + out.ddf[0][0] = ddfdxdx; + out.ddf[0][1] = ddfdxdy; + out.ddf[1][1] = ddfdydy; + out.ddf[1][0] = out.ddf[0][1]; + + out.dxdu[0] = 1.; + out.dxdu[1] = 0.; + + out.dxdf[0] = 0.; + out.dxdf[1] = 1.; + + return out; + } +}; + +#endif /* _AHSPHERICALGEOMETRY_HPP_ */ diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.hpp new file mode 100644 index 000000000..5310672cc --- /dev/null +++ b/Source/ApparentHorizonFinder/ApparentHorizon.hpp @@ -0,0 +1,296 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef _APPARENTHORIZON_HPP_ +#define _APPARENTHORIZON_HPP_ + +#include +// #include + +#include "AHData.hpp" +#include "AHDeriv.hpp" +#include "AHFinder.hpp" +#include "AHGeometryData.hpp" +#include "AHInterpolation.hpp" +#include "IntegrationMethod.hpp" + +// Class to manage ApparentHorizon for 2+1D and 3+1D simulations +// Class AHFinder manages it +//! AHFunction defines the optimizing function (see AHFunction.hpp for +//! expansion example calculation) +template class ApparentHorizon +{ + using Interpolation = AHInterpolation; + + public: + //! AH that finds the zero of expansion + ApparentHorizon( + const Interpolation &a_interp, //!< Geometry class to exchange data + double a_initial_guess, //!< Initial guess for radius (or whatever + //!< coordinate you're solving for) + const AHFinder::params &a_params, //!< set of AH parameters + const std::string &a_stats = + "stats.dat", //!< name for output file with area, spin and AH origin + const std::string &a_coords = + "coords_", //!< name for output file with AH coordinates at each + //!< time step + bool solve_first_step = true //!< whether or not to solve if t=0 + ); + //! personalized optimizer that finds zero of function + //! 'a_function_to_optimize' (a void* 'a_function_to_optimize_params' can be + //! passed for auxiliary parameters passed to 'a_function_to_optimize') + ApparentHorizon(const Interpolation &a_interp, double a_initial_guess, + const AHFinder::params &a_params, + const typename AHFunction::params &a_func_params, + const std::string &a_stats = "stats", + const std::string &a_coords = "coords", + bool solve_first_step = true); + ~ApparentHorizon(); + + void solve(double a_dt, double a_time, double a_restart_time); + + bool good_to_go(double a_dt, double a_time) const; + bool get_converged() + const; //!< PETSc didn't converge last time solve() was called + int get_failed_convergences() + const; //!< PETSc didn't converge last time solve() was called + bool has_been_found() const; //!< PETSc converged once and then + //!< stopped converging (AH collapsed) + + const std::array &get_origin() const; + const std::array &get_center() const; + void set_origin(const std::array &a_origin); + double get_initial_guess() const; + void set_initial_guess(double a_initial_guess); + + const Interpolation &get_ah_interp() const; + + double get_max_F() const; + double get_min_F() const; + double get_ave_F() const; + double get_std_F() const; + + bool do_solve(double a_dt, double a_time) + const; //!< decide (based times passed to 'solve') whether or not + //!< to print (uses AHFinder::params::solve_interval) + bool do_print(double a_dt, double a_time) + const; //!< decide when to print (only AHFinder::params::print_interval + //!< out of all 'solve's) + + const AHFinder::params &m_params; //!< set of AH parameters + const std::string m_stats, + m_coords; //!< public base names for output files (no need for a set as + //!< they are const) + + // any parameters that want to be saved to be passed to optimization + // function + typename AHFunction::params m_func_params; + + private: + void write_outputs(double a_dt, double a_time, double a_restart_time); + + void + restart(bool solve_first_step); //!< restart AH, updating the coordinates + //!< based on the last output file and the + //!< origin based on the stats file + + double calculate_area(); //!< calculate AH area + +#if CH_SPACEDIM == 3 + double calculate_spin_dimensionless( + double a_area); //!< calculate spin with 'z' direction, ONLY FOR 3D +#endif + // estimate based on area and spin + ALWAYS_INLINE double calculate_mass(double area, double spin_dimensionless) + { + return sqrt( + area / (8. * M_PI * + (1. + sqrt(1. - spin_dimensionless * spin_dimensionless)))); + } + + void calculate_minmax_F() const; + void calculate_average_F() const; + + void reset_initial_guess(); + + void update_old_centers(std::array); + void predict_next_origin(); + std::array + calculate_center(); //!< update location of center by calculating the + //!< centroid of the AH + + void write_coords_file(double a_dt, double a_time, double a_restart_time, + const std::string &filename, + bool write_geometry_data = false) + const; //!< write coords in (m_u, m_v, m_F) to 'filename' + + //! write metric and extrinsic curvature for each point of AH + void write_geometry_data(const std::string &a_file_prefix, double a_dt, + double a_time, double a_restart_time) const; + + void check_convergence(); //!< check if PETSc has converged and share that + //!< info across all Chombo processes + + // default to simpson rule and change if invalid + void check_integration_methods(); + + void initialise_PETSc(); //!< initialise automatically done in constructor + void finalise_PETSc(); //!< finalise automatically done in destructor + + // variables + private: + bool m_printed_once; + + int m_converged; //!< flag saying if PETSc has converged the last 'N' times + //!<(read using 'get_converged()') + bool m_has_been_found; //!< flag saying if an horizon has ever been found + //!< (read using 'has_been_found()') + int m_num_failed_convergences; //!< the number of failed consecutive + //!< convergences + + double m_initial_guess; //!< initial guess for AH (saved so that it can be + //!< re-used when atempting to solve again) + + //! used to estimate where the new center will be before solving + //! (currently last 2 centers saved to make a parabolic regression) + //! (elements ordered in reverse time - older with higher index) + std::vector> m_old_centers; + + // mutable so that 'const' objects can still change them + //! stores the global maximum and minimum of F - calculate with + //! calculate_minmax_F + mutable double m_max_F, m_min_F, m_ave_F, m_std_F; + + // Integration method to calculate area and spin of AH. Defaults to Simpson + // method + std::array m_integration_methods; + + double m_area, m_spin, + m_mass; // just to save the result temporarily at each iteration + + ///////////////////////////////////////////////////////////////////////// + /////////////////////////// PETSc stuff below /////////////////////////// + ///////////////////////////////////////////////////////////////////////// + +#if CH_SPACEDIM == 3 + typedef PetscScalar **dmda_arr_t; +#elif CH_SPACEDIM == 2 + typedef PetscScalar *dmda_arr_t; +#endif + + //! Geometries of the AH + //! 'm_geom_plus' and 'm_geom_minus' are used to calculate the + //! jacobian of the expansion using a 'delta' numerical differentiation + Interpolation m_interp; + Interpolation m_interp_plus; + Interpolation m_interp_minus; + + //!< used to compute jacobian of expansion (numerical differentiation) + static constexpr double eps = 1e-7; + + const bool m_periodic_u; //!< is 'u' periodic? + PetscInt m_num_global_u; //!< total number of grid points in 'u' coordinate + double m_du; //!< physical 'delta' in 'u' coordinate + +#if CH_SPACEDIM == 3 + const bool m_periodic_v; //!< is 'v' periodic? + PetscInt m_num_global_v; //!< total number of grid points in 'u' coordinate + double m_dv; //!< physical 'delta' in 'v' coordinate +#endif + + //! vectors to store and manipulate 'F', u', and 'v' + //! internally, in interaction with the 'Interpolation's + std::vector m_F; + std::vector m_u; +#if CH_SPACEDIM == 3 + std::vector m_v; +#endif + + //! minimums and maximums of coordinates 'u' and 'v' + //! of the PETSc grid specific to the current rank + PetscInt m_umin; + PetscInt m_umax; + +#if CH_SPACEDIM == 3 + PetscInt m_vmin; + PetscInt m_vmax; +#endif + + //! number of points in 'u' and 'v' direction + //! (m_nu = m_umax - m_umin) + PetscInt m_nu; +#if CH_SPACEDIM == 3 + PetscInt m_nv; +#endif + + // PETSc main object + DM m_dmda; + //! Scalable Nonlinear Equations Solvers + SNES m_snes; + + Vec m_snes_soln; + Vec m_snes_rhs; + Mat m_snes_jac; + + //! interpolate (u,v) 2D grid points at restart if number of points in + //! either direction changed + //! This only interpolates the points that the PETSc rank that called it has + //! returns whether or not points were interpolated + bool interpolate_ah(dmda_arr_t f, + const std::vector> &old_coords); + + //! set the default stencils of AHDeriv at position {u,v} + void set_stencils(AHDeriv &out, int u +#if CH_SPACEDIM == 3 + , + int v +#endif + ); + + //! function to calculate 1st and 2nd derivatives of 'in' + //! (tipically corresponds to our 'f' function) + //! in the 'u' and 'v' directions + AHDeriv diff(const dmda_arr_t in, int u +#if CH_SPACEDIM == 3 + , + int v +#endif + ); + + //! private functions used to compute the RHS (the expansion) and it's + //! jacobian + void form_function(Vec F, Vec Rhs); + void form_jacobian(Vec F, Mat J); + + //! helper for 'form_jacobian' + double point_jacobian(int u, int u_stencil, +#if CH_SPACEDIM == 3 + int v, int v_stencil, +#endif + dmda_arr_t in, int idx, + const Interpolation &interp_plus, + const Interpolation &interp_minus); + + //! functions used by PETSc based on 'form_function' and 'form_jacobian' + static PetscErrorCode Petsc_form_function(SNES snes, Vec F, Vec Rhs, + void *ptr); + + static PetscErrorCode +#if PETSC_VERSION_GE(3, 5, 0) + Petsc_form_jacobian(SNES snes, Vec F, Mat Amat, Mat Pmat, void *ptr); +#else + Petsc_form_jacobian(SNES snes, Vec F, Mat *Amat, Mat *Pmat, + MatStructure *flag, void *ptr); +#endif + + // monitor function required for SNES + static PetscErrorCode Petsc_SNES_monitor(SNES snes, PetscInt its, + PetscReal norm, void *ptr); +}; + +#include "ApparentHorizon.impl.hpp" +#include "ApparentHorizon_petsc.impl.hpp" + +#endif /* _APPARENTHORIZON_HPP_ */ diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp new file mode 100644 index 000000000..90b037589 --- /dev/null +++ b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp @@ -0,0 +1,1712 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef _APPARENTHORIZON_HPP_ +#error "This file should only be included through ApparentHorizon.hpp" +#endif + +#ifndef _APPARENTHORIZON_IMPL_HPP_ +#define _APPARENTHORIZON_IMPL_HPP_ + +#include "GRAMR.hpp" +#include "SmallDataIO.hpp" +#include "TensorAlgebra.hpp" +#include "UserVariables.hpp" + +// for 1D interpolation of restart centers +#include "Lagrange.hpp" +#include "SimpleArrayBox.hpp" +#include "SimpleInterpSource.hpp" + +#include // for 'setw' in stringstream +#include // infinity for min and max calculation + +template +ApparentHorizon::ApparentHorizon( + const AHInterpolation &a_interp, + double a_initial_guess, const AHFinder::params &a_params, + const std::string &a_stats, const std::string &a_coords, + bool solve_first_step) + : ApparentHorizon(a_interp, a_initial_guess, a_params, AHFunction::params(), + a_stats, a_coords, solve_first_step) +{ +} + +template +ApparentHorizon::ApparentHorizon( + const AHInterpolation &a_interp, + double a_initial_guess, const AHFinder::params &a_params, + const typename AHFunction::params &a_func_params, + const std::string &a_stats, const std::string &a_coords, + bool solve_first_step) + : m_params(a_params), + + m_stats(a_stats), m_coords(a_coords), + + m_func_params(a_func_params), + + m_printed_once(false), + + m_converged(0), + + m_has_been_found(false), + + m_num_failed_convergences(0), + + m_initial_guess(a_initial_guess), + + m_old_centers(3, a_interp.get_coord_system().get_origin()), + + m_max_F(0.), m_min_F(0.), m_ave_F(0.), m_std_F(0.), + + m_integration_methods({ +#if CH_SPACEDIM == 3 + IntegrationMethod::simpson, +#endif + IntegrationMethod::simpson + }), + + m_area(NAN), m_spin(NAN), m_mass(NAN), + + m_interp(a_interp), m_interp_plus(a_interp), m_interp_minus(a_interp), + + m_periodic_u(a_interp.is_u_periodic()), + m_num_global_u(a_params.num_points_u) + +#if CH_SPACEDIM == 3 + , + m_periodic_v(a_interp.is_v_periodic()), + m_num_global_v(a_params.num_points_v) +#endif +{ + initialise_PETSc(); + set_origin(a_interp.get_coord_system().get_origin()); + check_integration_methods(); + restart(solve_first_step); +} +template +ApparentHorizon::~ApparentHorizon() +{ + finalise_PETSc(); +} + +template +bool ApparentHorizon::good_to_go( + double a_dt, double a_time) const +{ + if (a_time < m_params.start_time - 1.e-7 /*just to be safe*/) + return false; + if (m_params.give_up_time >= 0. && a_time >= m_params.give_up_time && + !get_converged()) + return false; + + bool is_lost = + (m_params.max_fails_after_lost >= 0 + ? m_num_failed_convergences > m_params.max_fails_after_lost + : false); + + // stop if it has been found but was lost + return do_solve(a_dt, a_time) && !(has_been_found() && is_lost); +} + +template +bool ApparentHorizon::do_solve(double a_dt, + double a_time) const +{ + CH_assert(a_dt != 0); // Check if time was set! + return !(((int)(std::round(a_time / a_dt))) % m_params.solve_interval); +} +template +bool ApparentHorizon::do_print(double a_dt, + double a_time) const +{ + CH_assert(a_dt != 0); // Check if time was set! + return (get_converged() || !has_been_found()) && + !(((int)(std::round(a_time / a_dt))) % + (m_params.solve_interval * m_params.print_interval)); +} + +template +const std::array & +ApparentHorizon::get_origin() const +{ + return m_interp.get_origin(); +} + +template +const AHInterpolation & +ApparentHorizon::get_ah_interp() const +{ + return m_interp; +} + +template +void ApparentHorizon::set_origin( + const std::array &a_origin) +{ + m_interp.set_origin(a_origin); + m_interp_plus.set_origin(a_origin); + m_interp_minus.set_origin(a_origin); +} + +template +const std::array & +ApparentHorizon::get_center() const +{ + return m_old_centers[0]; +} +template +bool ApparentHorizon::get_converged() const +{ + return m_converged; +} + +template +int ApparentHorizon::get_failed_convergences() + const +{ + return m_num_failed_convergences; +} +template +bool ApparentHorizon::has_been_found() const +{ + return m_has_been_found; +} +template +double ApparentHorizon::get_max_F() const +{ + if (m_max_F <= 0.) + calculate_minmax_F(); + return m_max_F; +} +template +double ApparentHorizon::get_min_F() const +{ + if (m_min_F <= 0.) + calculate_minmax_F(); + return m_min_F; +} +template +double ApparentHorizon::get_ave_F() const +{ + if (m_ave_F <= 0.) + calculate_average_F(); + return m_ave_F; +} +template +double ApparentHorizon::get_std_F() const +{ + if (m_std_F <= 0.) + calculate_average_F(); + return m_std_F; +} + +template +double ApparentHorizon::get_initial_guess() const +{ + return m_initial_guess; +} + +template +void ApparentHorizon::set_initial_guess( + double a_initial_guess) +{ + m_initial_guess = a_initial_guess; +} +template +void ApparentHorizon::reset_initial_guess() +{ + CH_TIME("ApparentHorizon::reset_initial_guess"); + + if (!AHFinder::is_rank_active()) + return; + + auto origin = m_interp.get_origin(); + + // verify origin +- initial guess is inside the grid + bool out_of_grid = m_interp.is_in_grid(origin, m_initial_guess); + CH_assert(!out_of_grid); + + if (m_params.verbose > AHFinder::NONE) + { + pout() << "Setting Initial Guess to f=" << m_initial_guess + << " centered at (" << origin[0] << "," << origin[1] +#if CH_SPACEDIM == 3 + << "," << origin[2] +#endif + << ")" << std::endl; + } + + // read PETSc array to 'f' + dmda_arr_t f; + DMDAVecGetArray(m_dmda, m_snes_soln, &f); + +#if CH_SPACEDIM == 3 + for (int v = m_vmin; v < m_vmax; ++v) +#endif + { + for (int u = m_umin; u < m_umax; ++u) + { +#if CH_SPACEDIM == 3 + double &f_point = f[v][u]; +#else + double &f_point = f[u]; +#endif + + f_point = m_initial_guess; + } + } + + // write PETSc array back + DMDAVecRestoreArray(m_dmda, m_snes_soln, &f); +} +template +void ApparentHorizon::predict_next_origin() +{ + std::array new_center = m_old_centers[0]; + if (m_converged >= 3) // add 2nd derivative + { + FOR1(a) + { + new_center[a] += (m_old_centers[0][a] + m_old_centers[2][a] - + 2. * m_old_centers[1][a]); + } + if (m_params.verbose > AHFinder::SOME) + { + pout() << "OLD[-2]: (" << m_old_centers[2][0] << "," + << m_old_centers[2][1] +#if CH_SPACEDIM == 3 + << "," << m_old_centers[2][2] +#endif + << ")" << std::endl; + } + } + if (m_converged >= 2) // add 1st derivative + { + FOR1(a) + { + new_center[a] += (m_old_centers[0][a] - m_old_centers[1][a]); + } + + if (m_params.verbose > AHFinder::SOME) + { + pout() << "OLD[-1]: (" << m_old_centers[1][0] << "," + << m_old_centers[1][1] +#if CH_SPACEDIM == 3 + << "," << m_old_centers[1][2] +#endif + << ")" << std::endl; + pout() << "OLD[0]: (" << m_old_centers[0][0] << "," + << m_old_centers[0][1] +#if CH_SPACEDIM == 3 + << "," << m_old_centers[0][2] +#endif + << ")" << std::endl; + } + if (m_params.verbose > AHFinder::MIN) + { + pout() << "Estimated center: (" << new_center[0] << "," + << new_center[1] +#if CH_SPACEDIM == 3 + << "," << new_center[2] +#endif + << ")" << std::endl; + } + } + + set_origin(new_center); +} + +template +void ApparentHorizon::update_old_centers( + std::array new_center) +{ + FOR1(a) + { + if (!m_interp.get_interpolator()->get_boundary_reflective(Side::Lo, + a) && + !m_interp.get_interpolator()->get_boundary_reflective(Side::Hi, a)) + { + m_old_centers[2][a] = m_old_centers[1][a]; + m_old_centers[1][a] = m_old_centers[0][a]; + m_old_centers[0][a] = new_center[a]; + } + } +} + +template +void ApparentHorizon::solve(double a_dt, + double a_time, + double a_restart_time) +{ + CH_TIME("ApparentHorizon::solve"); + if (!good_to_go(a_dt, a_time)) + return; + + m_interp.refresh_interpolator(); //(ALL CHOMBO ranks do it!!) + + // estimate the next position where origin will be + if (get_converged()) + { + if (m_params.predict_origin) + predict_next_origin(); + else if (m_params.track_center) + set_origin(get_center()); + } + + // PETSc processes go inside 'if', others "wait" until 'if' gets to + // 'm_interp.break_interpolation_loop()' + if (m_interp.keep_interpolating_if_inactive()) + { + CH_TIME("ApparentHorizon::solve::solving"); + + if (!get_converged()) + reset_initial_guess(); // reset initial guess if diverged (or in + // first attempt) + + // actual solve happens here! + SNESSolve(m_snes, NULL, m_snes_soln); + + PetscInt its; + SNESGetIterationNumber(m_snes, &its); + if (m_params.verbose > AHFinder::MIN) + { + pout() << "SNES Iteration number " << its << endl; + } + SNESGetLinearSolveIterations(m_snes, &its); + if (m_params.verbose > AHFinder::MIN) + { + pout() << "KSP Iteration number " << its << endl; + } + m_interp.break_interpolation_loop(); + } + + { + if (m_params.verbose > AHFinder::SOME) + { + pout() << "In [ApparentHorizon::solve::post-solving]" << endl; + } + + CH_TIME("ApparentHorizon::solve::post-solving"); + + bool save_converged = get_converged(); + + // ask PETSc if it converged + check_convergence(); + + // retry with initial guess once if failed (and everything was ok + // before) + // on the 2nd iteration, 'save_converged' will be false and it will skip + // the 'if' + if (save_converged && !get_converged() && m_params.allow_re_attempt) + { + --m_num_failed_convergences; // reduce failed and try again + if (m_params.verbose > AHFinder::NONE) + { + pout() << "Re-attempting to solve using initial guess." + << std::endl; + } + solve(a_dt, a_time, a_restart_time); + return; // do nothing else + } + + m_area = calculate_area(); +#if CH_SPACEDIM == 3 + double spin_dimensionless = calculate_spin_dimensionless(m_area); + m_mass = calculate_mass(m_area, spin_dimensionless); + m_spin = spin_dimensionless * m_mass; +#endif + + // reset min and max F, to force re-calculation + m_max_F = 0.; + m_min_F = 0.; + m_ave_F = 0.; + m_std_F = 0.; + } + + // update center + // note that after this, the point that corresponds to the (u,v,r) + // coordinates is 'get_origin()' + if (m_params.track_center) + calculate_center(); + + write_outputs(a_dt, a_time, a_restart_time); + + // break if 'AH_stop_if_max_fails == true' + // only break after re-attempting + // (if max fails reached or give up time reached) + if (m_params.stop_if_max_fails && + ((m_params.max_fails_after_lost >= 0 && + m_num_failed_convergences > m_params.max_fails_after_lost) || + (m_params.give_up_time >= 0. && a_time >= m_params.give_up_time && + !get_converged()))) + MayDay::Error("Reached max fails. Stopping. Parameter " + "'stop_if_max_fails' is set to true."); + + if (m_params.verbose > AHFinder::SOME) + { + pout() << "ApparentHorizon::solve finished successfully!" << endl; + } +} + +template +void ApparentHorizon::write_outputs( + double a_dt, double a_time, double a_restart_time) +{ + // print stats (area, spin, origin, center) and coordinates + // stop printing if it stopped converging + // then in 'write_coords_file' nothing is printed if not converged + if (do_print(a_dt, a_time)) + { + CH_TIME("ApparentHorizon::solve::printing"); + if (m_params.verbose > AHFinder::MIN) + { + pout() << "Printing statistics and coordinates." << std::endl; + } + + // write stats + + // fake_dt is relevant for the 'remove_duplicate_time_data' part + double fake_dt = + a_dt * m_params.solve_interval * m_params.print_interval; + SmallDataIO file(m_stats, fake_dt, a_time, a_restart_time, + SmallDataIO::APPEND, !m_printed_once); + + file.remove_duplicate_time_data(); + + // std::string coords_filename = file.get_new_file_number(fake_dt, + // a_time); + + // first '1' corresponds to 'step' + // the first (CH_SPACEDIM==3 ? 3 : 1) corresponds to area+spin+mass or + // only area + CH_assert(CH_SPACEDIM == 3 || CH_SPACEDIM == 2); + std::vector values(1 + (CH_SPACEDIM == 3 ? 3 : 1) + + CH_SPACEDIM * (1 + m_params.track_center)); + + auto origin = get_origin(); + + int step = std::round(a_time / fake_dt); + + int idx = 0; + values[idx++] = step; + values[idx++] = m_area; +#if CH_SPACEDIM == 3 + values[idx++] = m_spin; + values[idx++] = m_mass; +#endif + for (int i = 0; i < CH_SPACEDIM; ++i) + values[idx++] = origin[i]; + if (m_params.track_center) + for (int i = 0; i < CH_SPACEDIM; ++i) + values[idx++] = m_old_centers[0][i]; + + // print headers to stats file in the beginning of evolution + if (!m_printed_once) + { + std::vector headers(1 + (CH_SPACEDIM == 3 ? 3 : 1) + + CH_SPACEDIM * + (1 + m_params.track_center)); + + idx = 0; + headers[idx++] = "file"; + headers[idx++] = "area"; +#if CH_SPACEDIM == 3 + headers[idx++] = "spin"; + headers[idx++] = "mass"; +#endif + headers[idx++] = "origin_x"; + headers[idx++] = "origin_y"; +#if CH_SPACEDIM == 3 + headers[idx++] = "origin_z"; +#endif + if (m_params.track_center) + { + headers[idx++] = "center_x"; + headers[idx++] = "center_y"; +#if CH_SPACEDIM == 3 + headers[idx++] = "center_z"; +#endif + } + + file.write_header_line(headers); + + m_printed_once = true; + } + + file.write_time_data_line(values); + + // write coordinates + m_interp.interpolate_extra_vars(m_params.extra_vars); + write_coords_file(a_dt, a_time, a_restart_time, m_coords, + m_params.print_geometry_data); + } +} +template +void ApparentHorizon::check_convergence() +{ + CH_TIME("ApparentHorizon::check_convergence"); + + // Check SNES convergence reasons in: + // https://www.mcs.anl.gov/petsc/petsc-current/docs/manualpages/SNES/SNESConvergedReason.html#SNESConvergedReason + + int result; + if (AHFinder::is_rank_active()) + { + SNESConvergedReason reason; + SNESGetConvergedReason(m_snes, &reason); + + // result will be 0 if any of the PETSc ranks says 'reason <=0' (PETSc + // convergence error) + int tmp = reason; // convert just to make sure it's 'int' +#ifdef CH_MPI + MPI_Allreduce(&tmp, &result, 1, MPI_INT, MPI_MIN, Chombo_MPI::comm); +#else + result = tmp; +#endif + } + else + { + int ALOT = 100; // smaller than all SNESConvergedReasons +#ifdef CH_MPI + MPI_Allreduce(&ALOT, &result, 1, MPI_INT, MPI_MIN, Chombo_MPI::comm); +#else + result = ALOT; +#endif + } + + bool converged = (result > 0); + + if ((bool)converged) + { + ++m_converged; + m_num_failed_convergences = 0; + } + else + { + m_converged = 0; // reset + ++m_num_failed_convergences; + } + + if (m_params.verbose > AHFinder::NONE) + { + pout() << (m_converged ? "Solver converged. Horizon FOUND." + : "Solver diverged. Horizon NOT found.") + << std::endl; + + if (m_params.verbose > AHFinder::MIN) + { + pout() << "SNESConvergedReason = " << result << std::endl; + } + } + + if (!m_has_been_found && m_converged) + m_has_been_found = true; // finally found :D +} + +template +void ApparentHorizon::restart( + bool solve_first_step) +{ + CH_TIME("ApparentHorizon::restart"); + + static double eps = 1.e-7; + + // GUIDING PRINCIPLE : restart should not depend on m_params, as these might + // have changed! We should be able to figure everything out without them + + const GRAMR &gramr = + (static_cast(m_interp.get_interpolator()->getAMR())); + + int current_step = AMR::s_step; + int restart_step = gramr.get_restart_step(); + + bool solve_time_0_now = (!m_params.re_solve_at_restart && + current_step == 0 && restart_step < 0); + + // solve if at t=0 and not a restart + if (solve_time_0_now) + { + // 'dt' doesn't matter for 1st time step, so set to 1 + if (solve_first_step) + solve(1., 0., 0.); + return; + } + + double current_time = gramr.getCurrentTime(); + double coarse_dt = (current_step == 0 ? 0. : current_time / current_step); + double level_dt = coarse_dt * pow(2., -m_params.level_to_run); + double current_solve_dt = level_dt * m_params.solve_interval; + + // READ STATS + + // get centre from stats file + std::string file = m_stats + ".dat"; + auto stats = SmallDataIO::read(file); + + int idx = 0; + double old_print_dt = 0.; + if (stats.size() == 0) + { // case when it never ran the AH or the file doesn't exist + if (m_params.verbose > AHFinder::NONE && current_step != 0) + { + pout() << "Empty stats file '" << file + << "'. Assuming AHFinder was not run yet for this AH." + << std::endl; + } + m_printed_once = false; + m_has_been_found = false; + m_converged = 0; + m_num_failed_convergences = 0; + } + else + { + // look for stats line with time 'current_time', as there may be + // further output messed up in the file after the last checkpoint was + // made + idx = stats[0].size(); + while ((--idx) >= 0 && stats[0][idx] - current_time > eps) + ; + + // figure out what was the old 'dt' used + // this may have changed if 'level_to_run' changed of if 'dt_multiplier' + // changed, or 'solve_interval', or 'print_interval' + if (idx > 0) + old_print_dt = stats[0][idx] - stats[0][idx - 1]; + else if (idx == 0 && stats[0].size() > 1) + old_print_dt = stats[0][idx + 1] - + stats[0][idx]; // there's still time steps forward + + if (m_params.verbose > AHFinder::SOME) + { + if (idx >= 0) + pout() << "stats[0][idx] = " << stats[0][idx] << std::endl; + pout() << "stats[0].size() = " << stats[0].size() << std::endl; + pout() << "current_time = " << current_time << std::endl; + pout() << "old_print_dt = " << old_print_dt << std::endl; + } + + m_printed_once = true; // don't delete old + + if (idx < 0) + { // case when job was restarted at a point before the AH was first ever + // found + if (m_params.verbose > AHFinder::NONE) + { + pout() + << "First time step is after restart in '" << file + << "'. Assuming AHFinder started after current step, but " + "before " + "PETSc had never converged." + << std::endl; + } + m_has_been_found = false; + m_converged = 0; + m_num_failed_convergences = 0; + } + else if (stats[0][idx] - current_time < + -(old_print_dt == 0. ? eps : old_print_dt - eps)) + { // case when the PETSc stopped converging and stopped printing + // so there is a mismatch with the last time in the file + if (m_params.verbose > AHFinder::NONE) + { + pout() << "Last time step not found in '" << file + << "'. Assuming AH stopped converging." << std::endl; + pout() << "(number of failed convergences will be set to 1)" + << std::endl; + } + m_has_been_found = true; + m_converged = 0; + m_num_failed_convergences = 1; + } + else if (std::isnan(stats[2][idx])) + { // case when the simulation was still going without ever + // having converged + // OR when job was restarted at a point before the AH was first ever + // found + if (m_params.verbose > AHFinder::NONE) + { + pout() << "Last time step is NAN in '" << file + << "'. Assuming AH wasn't found and PETSc didn't " + "converged." + << std::endl; + pout() << "(number of failed convergences will be set to 1)" + << std::endl; + } + m_has_been_found = false; + m_converged = 0; + m_num_failed_convergences = 1; + } + else + { // case when the AH was found and there is a coordinate file + if (m_params.verbose > AHFinder::NONE) + { + pout() << "Last time step found in '" << file + << "'. Reading coordinates file." << std::endl; + } + m_has_been_found = true; + m_converged = 1; + m_num_failed_convergences = 0; + } + } + + // force restart if interpolation was needed (because #points changed) + // or if time step found is not the current time + // 'int' in order to use MPI_INT later + int force_restart = false; + + // if not converged, leave + // origin as the initial guessed origin + // and 'solve' will reset coords to initial guess + if (m_converged) + { + //////////////////// + // READ ORIGIN + //////////////////// + + int cols = stats.size(); + + std::array origin; +#if CH_SPACEDIM == 3 + bool was_center_tracked = + (cols > 5 + CH_SPACEDIM); // 5 for time + file + area + spin + mass +#elif CH_SPACEDIM == 2 + bool was_center_tracked = + (cols > 3 + CH_SPACEDIM); // 3 for time + file + area +#endif + + int offset = CH_SPACEDIM + was_center_tracked * CH_SPACEDIM; + FOR1(a) { origin[a] = stats[cols - offset + a][idx]; } + + if (m_params.verbose > AHFinder::NONE) + { + pout() << "Setting origin from stats file '" << m_stats << "' at (" + << origin[0] << "," << origin[1] +#if CH_SPACEDIM == 3 + << "," << origin[2] +#endif + << ")" << std::endl; + } + + set_origin(origin); + + //////////////////// + // READ CENTERS + //////////////////// + + // this is only relevant if track_center was on + if (was_center_tracked) + { + // SET OLD CENTERS directly if old 'dt' matches current 'dt' + if (std::abs(old_print_dt - current_solve_dt) < eps) + { + if (m_params.verbose > AHFinder::NONE && + m_params.predict_origin) + { + pout() << "Reading old centers to predict next origin." + << std::endl; + } + + // skip i=0, this one is directly in the file and is done after + for (int i = m_old_centers.size() - 1; i > 0; --i) + { + if (idx >= i && !std::isnan(stats[2][idx - 1])) + { + m_converged++; + update_old_centers({ +#if CH_SPACEDIM == 3 + stats[cols - 3][idx - i], +#endif + stats[cols - 2][idx - i], + stats[cols - 1][idx - i] + }); + } + } + } + else + { + // calculate last NAN to make sure we don't calculate points of + // when AH didn't converge + int last_nan_idx = stats[0].size(); + while ((--last_nan_idx) >= 0 && + !std::isnan(stats[2][last_nan_idx])) + ; + + // interpolate old centers when dt has changed + std::vector old_centers_time_index; + // skip i=0, this one is directly in the file and is done after + for (int i = 1; i < m_old_centers.size(); ++i) + { + double time = current_time - current_solve_dt * i; + double index = idx - (current_time - time) / old_print_dt; + + if (index >= last_nan_idx) + old_centers_time_index.push_back(index); + } + + if (m_params.verbose > AHFinder::NONE && + m_params.predict_origin) + { + if (old_centers_time_index.size() == 0) + pout() << "AH time step changed and no old centers " + "able to " + "use for prediction." + << std::endl; + else + pout() + << "AH time step changed. Recovering " + << old_centers_time_index.size() + << " old centers from interpolation, for accurate " + "prediction of next origin." + << std::endl; + } + + int rows = stats[0].size(); + SimpleInterpSource<1> source({rows}); + SimpleArrayBox<1> box_x({rows}, stats[cols - CH_SPACEDIM]); + SimpleArrayBox<1> box_y({rows}, stats[cols - CH_SPACEDIM + 1]); +#if CH_SPACEDIM == 3 + SimpleArrayBox<1> box_z({rows}, stats[cols - CH_SPACEDIM + 2]); +#endif + + const int Order = 2; + Lagrange interpolator(source); + for (int i = old_centers_time_index.size() - 1; i >= 0; --i) + { + std::array old_center; + + interpolator.setup({0}, {old_print_dt}, + {old_centers_time_index[i]}); + old_center[0] = interpolator.interpData(box_x); + old_center[1] = interpolator.interpData(box_y); +#if CH_SPACEDIM == 3 + old_center[2] = interpolator.interpData(box_z); +#endif + + m_converged++; + update_old_centers(old_center); + + if (m_params.verbose > AHFinder::SOME) + { + pout() << "OLD[-" << i + 1 << "] = (" << old_center[0] + << "," << old_center[1] +#if CH_SPACEDIM == 3 + << "," << old_center[2] +#endif + << ")" << std::endl; + } + } + } + } + + // save current center no matter what + // place back OLD[0] - THE CENTER + update_old_centers({ +#if CH_SPACEDIM == 3 + stats[cols - 3][idx], +#endif + stats[cols - 2][idx], stats[cols - 1][idx] + }); + + //////////////////// + // READ COORDINATES + //////////////////// + + // determine last step in which there was an AH output + + // the lines commented below also work, but not if there's only one + // stats, as then 'old_print_dt' can't be defined std::string + // coords_filename = SmallDataIO::get_new_filename(m_coords, + // old_print_dt, current_time); + + int coords_file_number = stats[1][idx]; + std::string coords_filename = SmallDataIO::get_new_filename( + m_coords, 1. /*fake dt*/, coords_file_number); + + auto coords = SmallDataIO::read(coords_filename); + + if (m_params.verbose > AHFinder::NONE) + { + pout() << "Setting Initial Guess to previous file '" + << coords_filename << "' found." << std::endl; + } + + // now write local points based on file (or interpolated file) + if (AHFinder::is_rank_active()) + { + // read PETSc array to 'f' + dmda_arr_t f; + DMDAVecGetArray(m_dmda, m_snes_soln, &f); + + // doesn't change anything if number of points remained the same + force_restart |= interpolate_ah(f, coords); + + // write PETSc array back + DMDAVecRestoreArray(m_dmda, m_snes_soln, &f); + } + +#ifdef CH_MPI + int tmp = force_restart; + MPI_Allreduce(&tmp, &force_restart, 1, MPI_INT, MPI_LOR, + Chombo_MPI::comm); +#endif + } + + // if t=0, solve only if it's a restart and if solve_first_step = true (it + // may be false for example for a merger of a binary, for which we don't + // want to solve!) + bool solve_no_matter_what = + (m_params.re_solve_at_restart && + (current_step != 0 || + (current_step == 0 && solve_first_step && restart_step >= 0))); + + if (force_restart || solve_no_matter_what) + solve(current_time == 0. ? 1. : level_dt, current_time, current_time); +} + +template +void ApparentHorizon::write_coords_file( + double a_dt, double a_time, double a_restart_time, + const std::string &filename, bool write_geometry_data) const +{ + CH_TIME("ApparentHorizon::write_coords_file"); + + if (!get_converged()) + { + // old way: writing an empty file + // new way: just don't write any coords file if no AH was founds + // coords_file.write_header_line({"Horizon NOT found."}, ""); + return; + } + + if (m_params.verbose > AHFinder::NONE && write_geometry_data) + pout() << "Writing geometry data." << std::endl; + + CH_assert(a_dt != 0); // Check if time was set!! + + double fake_dt = a_dt * m_params.solve_interval * m_params.print_interval; + SmallDataIO file(filename, fake_dt, a_time, a_restart_time, + SmallDataIO::NEW, true); + + unsigned num_components; // only related to 'write_geometry_data' + if (write_geometry_data) + num_components = AHFunction::num_write_vars(); + else + num_components = 0; + + // Write headers + std::vector components(num_components + + m_params.num_extra_vars); + int el = 0; + + // extra vars headers + for (auto &var : m_params.extra_vars) + { + int var_enum = std::get<0>(var.second); + VariableType var_type = std::get<1>(var.second); + int der_type = std::get<2>(var.second); + + std::string var_name; + if (var_type == VariableType::evolution) + var_name = UserVariables::variable_names[var_enum]; + else + var_name = DiagnosticVariables::variable_names[var_enum]; + + static std::string xyz[CH_SPACEDIM] = { + "x", + "y" +#if CH_SPACEDIM == 3 + , + "z" +#endif + }; + + if (der_type == 0) + components[el++] = var_name; + else if (der_type == 1) + for (int i = 0; i < CH_SPACEDIM; ++i) + components[el++] = "d" + xyz[i] + "_" + var_name; + else // if (der_type == 2) + for (int i = 0; i < CH_SPACEDIM; ++i) + for (int j = i; j < CH_SPACEDIM; ++j) + components[el++] = + "d" + xyz[i] + "d" + xyz[j] + "_" + var_name; + } + + // geometry headers + if (write_geometry_data) + AHFunction::write_headers(&components[el]); + file.write_header_line(components, m_interp.get_labels()); + + ////////////////// + // method: + // 1) all PESc ranks send their coordinates to rank 0 + // (non-PETSc processes will do nothing) + // 2) rank 0 receives and stores all coordinates + // 3) non-blocking sends finish the communication ('MPI_Wait') + // 4) rank 0 writes all the information + // Note: this does NOT assume that rank 0 is part of the PETSc communicator! + // (even though for now the PETSc processes always include rank 0) + ////////////////// + +#ifdef CH_MPI + // not needed for non-PETSc ranks, but they will be waiting anyway + MPI_Request *requests = nullptr; // needed for non-blocking communication +#endif + double *output = nullptr; // needed for non-blocking communication + unsigned num_components_total = + num_components + m_params.num_extra_vars + CH_SPACEDIM; + + // step 1 + if (AHFinder::is_rank_active()) + { + +#ifdef CH_MPI + int rank_petsc; + MPI_Comm_rank(PETSC_COMM_WORLD, &rank_petsc); + + // make sure rank 0 of Chombo is rank 0 of PETSc + // this must be the case just because it is rank 0 of PETSc + // that will be writing in the file, and SmallDataIO assumes it's rank 0 + // of Chombo (!) the one writing + // Alternative 1: if rank 0 of PETSc is different than rank 0 of Chombo, + // transfer all the data to Chombo's rank 0 before writing + // Alternative 2: find a way to tell SmallDataIO to use another rank for + // writing + int rank = 0; + MPI_Comm_rank(Chombo_MPI::comm, &rank); + if ((rank != 0 && rank_petsc == 0) || (rank == 0 && rank_petsc != 0)) + MayDay::Error("PETSc's rank 0 should be Chombo's rank 0"); +#endif + +#if CH_SPACEDIM == 3 + int local_total = (m_vmax - m_vmin) * (m_umax - m_umin); +#elif CH_SPACEDIM == 2 + int local_total = (m_umax - m_umin); +#endif + +#ifdef CH_MPI + requests = new MPI_Request[local_total]; +#endif + output = new double[local_total * num_components_total]; + + int idx = 0; +#if CH_SPACEDIM == 3 + for (int v = m_vmin; v < m_vmax; ++v) +#endif + { + for (int u = m_umin; u < m_umax; ++u) + { +#if CH_SPACEDIM == 3 + output[idx * num_components_total] = m_u[idx]; + output[idx * num_components_total + 1] = m_v[idx]; + output[idx * num_components_total + 2] = m_F[idx]; +#elif CH_SPACEDIM == 2 + output[idx * num_components_total] = m_u[idx]; + output[idx * num_components_total + 1] = m_F[idx]; +#endif + + const auto data = m_interp.get_data(idx); + const auto coords = m_interp.get_cartesian_coords(idx); + AHFunction func(data, coords); + auto extra = m_interp.get_extra_data(idx); + + int el = 0; + + // extra vars + for (auto &var : m_params.extra_vars) + { + // int var_enum = std::get<0>(var.second); + // VariableType var_type = std::get<1>(var.second); + int der_type = std::get<2>(var.second); + + if (der_type == 0) + { + output[idx * num_components_total + CH_SPACEDIM + + el++] = extra.vars.at(var.first); + } + else if (der_type == 1) + { + for (int i = 0; i < CH_SPACEDIM; ++i) + output[idx * num_components_total + CH_SPACEDIM + + el++] = extra.d1.at(var.first)[i]; + } + else // if (der_type == 2) + { + for (int i = 0; i < CH_SPACEDIM * (CH_SPACEDIM + 1) / 2; + ++i) + output[idx * num_components_total + CH_SPACEDIM + + el++] = extra.d2.at(var.first)[i]; + } + } + + // geometry vars + if (write_geometry_data) + func.write_vars( + &output[idx * num_components_total + CH_SPACEDIM + el]); + +#ifdef CH_MPI + // all processes send their 'output' to rank 0, who receives + // and writes everything (to simplify, rank 0 also sends it + // to itself, otherwise we wouldn't know which tags rank 0 + // has) sends are tagged by global index, so that receives + // can be unique and writes indexed correctly +#if CH_SPACEDIM == 3 + int idx_global = v * m_num_global_u + u; +#elif CH_SPACEDIM == 2 + int idx_global = u; +#endif + MPI_Issend(&output[idx * num_components_total], + num_components_total, MPI_DOUBLE, 0, idx_global, + PETSC_COMM_WORLD, &(requests[idx])); +#else + // only rank 0 is active -> serial + file.write_data_line( + std::vector(output + idx * num_components_total + + CH_SPACEDIM, + output + idx * num_components_total + + num_components_total), + std::vector(output + idx * num_components_total, + output + idx * num_components_total + + CH_SPACEDIM)); +#endif + + idx++; + } + } + + // step 2 +#ifdef CH_MPI +#if CH_SPACEDIM == 3 + int total = m_num_global_u * m_num_global_v; +#elif CH_SPACEDIM == 2 + int total = m_num_global_u; +#endif + double temp[total * num_components_total]; + if (rank_petsc == 0) + { + for (unsigned i = 0; i < total; ++i) + { + MPI_Status status; + MPI_Recv(&temp[i * num_components_total], num_components_total, + MPI_DOUBLE, MPI_ANY_SOURCE, + i /* use i as tag to make them ordered*/, + PETSC_COMM_WORLD, &status); + } + } + + // step 3 + idx = 0; +#if CH_SPACEDIM == 3 + for (int v = m_vmin; v < m_vmax; ++v) +#endif + { + for (int u = m_umin; u < m_umax; ++u) + { + MPI_Wait(&(requests[idx]), MPI_STATUS_IGNORE); + idx++; + } + } + + delete[] requests; // free allocated memory + delete[] output; + + // step 4 + if (rank_petsc == 0) + { + for (unsigned i = 0; i < total; ++i) + { + file.write_data_line( + std::vector( + &temp[i * num_components_total] + CH_SPACEDIM, + &temp[i * num_components_total] + num_components_total), + std::vector(&temp[i * num_components_total], + &temp[i * num_components_total] + + CH_SPACEDIM)); + } + } +#endif + } +} + +template +void ApparentHorizon::check_integration_methods() +{ + // check if integration methods are valid given periodicity and number of + // points + bool valid_u = m_integration_methods[0].is_valid(m_params.num_points_u, + m_interp.is_u_periodic()); + + IntegrationMethod method_default = IntegrationMethod::trapezium; + if (!valid_u) + { + std::string warn = + "ApparentHorizon: Simpson IntegrationMethod for u is " + "not valid with this num_points_u.\n" + "Reverting to trapezium rule."; + MayDay::Warning(warn.c_str()); + pout() << warn << std::endl; + m_integration_methods[0] = method_default; + } + +#if CH_SPACEDIM == 3 + bool valid_v = m_integration_methods[1].is_valid(m_params.num_points_v, + m_interp.is_v_periodic()); + if (!valid_v) + { + std::string warn = + "ApparentHorizon: Simpson IntegrationMethod for v is " + "not valid with this num_points_v.\n" + "Reverting to trapezium rule."; + MayDay::Warning(warn.c_str()); + pout() << warn << std::endl; + m_integration_methods[1] = method_default; + } +#endif +} + +#if CH_SPACEDIM == 3 +// ONLY FOR 3D +template +double +ApparentHorizon::calculate_spin_dimensionless( + double a_area) +{ + CH_assert(CH_SPACEDIM == 3); + CH_TIME("ApparentHorizon::calculate_spin_dimensionless"); + + if (!get_converged()) + return NAN; + + double equator_length; + double integral = 0.; // temporary, but defined outside to be in scope for + // non-PETSc processes + + // PETSc processes go inside 'if', others "wait" until 'if' gets to + // 'm_interp.break_interpolation_loop()' + if (m_interp.keep_interpolating_if_inactive()) + { + + int idx = 0; + + Vec localF; + + DMGetLocalVector(m_dmda, &localF); + DMGlobalToLocalBegin(m_dmda, m_snes_soln, INSERT_VALUES, localF); + DMGlobalToLocalEnd(m_dmda, m_snes_soln, INSERT_VALUES, localF); + + dmda_arr_t in; + DMDAVecGetArray(m_dmda, localF, &in); + + // m_F is already set from solve + + int u_equator = floor(M_PI / 2.0 / m_du); + + for (int v = m_vmin; v < m_vmax; ++v) + { + for (int u = m_umin; u < m_umax; ++u) + { + AHDeriv deriv = diff(in, u, v); + + if (u_equator == u) + { + const auto geometry_data = m_interp.get_geometry_data(idx); + const auto data = m_interp.get_data(idx); + const auto coords = m_interp.get_cartesian_coords(idx); + AHFunction func(data, coords); + auto &g = func.get_metric(); + + double dxdv[3]; + dxdv[0] = geometry_data.dxdv[0] + + deriv.dvF * geometry_data.dxdf[0]; + dxdv[1] = geometry_data.dxdv[1] + + deriv.dvF * geometry_data.dxdf[1]; + dxdv[2] = geometry_data.dxdv[2] + + deriv.dvF * geometry_data.dxdf[2]; + + double norm2 = 0.; + FOR2(i, j) norm2 += g[i][j] * dxdv[i] * dxdv[j]; + CH_assert(norm2 >= 0.); + + double weight = m_integration_methods[1].weight( + v, m_params.num_points_v, m_interp.is_v_periodic()); + + integral += sqrt(norm2) * weight * m_dv; + } + + idx++; + } + } + + DMDAVecRestoreArray(m_dmda, localF, &in); + DMRestoreLocalVector(m_dmda, &localF); + + m_interp.break_interpolation_loop(); + } + + // reduction across all Chombo processes (note that 'integral' is 0 for + // non-PETSc processes) because SmallDataIO uses rank 0 to write this + // ensures rank 0 will have 'integral' (even though for now the PETSc + // processes always include rank 0) +#ifdef CH_MPI + MPI_Allreduce(&integral, &equator_length, 1, MPI_DOUBLE, MPI_SUM, + Chombo_MPI::comm); +#else // serial + equator_length = integral; +#endif + + // mass could be estimated based on "equator_length / (4. * M_PI)" + // but calculation using area as well is probably more precise + + double factor = + ((2. * M_PI * a_area / (equator_length * equator_length)) - 1.); + double spin_dimensionless = + (factor > 1. ? 0 + : sqrt(1. - factor * factor)); // factor>1 means numerical + // error with spin as 0 + + if (m_params.verbose > AHFinder::MIN) + { + pout() << "dimensionless spin = " << spin_dimensionless << endl; + } + + return spin_dimensionless; +} +#endif + +template +double ApparentHorizon::calculate_area() +{ + CH_TIME("ApparentHorizon::calculate_area"); + + if (!get_converged()) + return NAN; + + double area; + double integral = + 0.; // temporary, but defined outside to exist for non-PETSc processes + + // PETSc processes go inside 'if', others "wait" until 'if' gets to + // 'm_interp.break_interpolation_loop()' + if (m_interp.keep_interpolating_if_inactive()) + { + + int idx = 0; + + Vec localF; + + DMGetLocalVector(m_dmda, &localF); + DMGlobalToLocalBegin(m_dmda, m_snes_soln, INSERT_VALUES, localF); + DMGlobalToLocalEnd(m_dmda, m_snes_soln, INSERT_VALUES, localF); + + dmda_arr_t in; + DMDAVecGetArray(m_dmda, localF, &in); + + // m_F is already set from solve + +#if CH_SPACEDIM == 3 + for (int v = m_vmin; v < m_vmax; ++v) +#endif + { + double inner_integral = 0.; + for (int u = m_umin; u < m_umax; ++u) + { + const auto geometric_data = m_interp.get_geometry_data(idx); + const auto data = m_interp.get_data(idx); + const auto coords = m_interp.get_cartesian_coords(idx); + AHFunction func(data, coords); + auto &g = func.get_metric(); + + // Calculate Jacobian matrix for transformation from Cartesian + // to (f,u,v) coords + Tensor<2, double> Jac; + FOR1(k) + { + Jac[0][k] = geometric_data.dxdf[k]; + Jac[1][k] = geometric_data.dxdu[k]; +#if CH_SPACEDIM == 3 + Jac[2][k] = geometric_data.dxdv[k]; +#endif + } + // auto det = TensorAlgebra::compute_determinant(Jac); + AHDeriv deriv = diff(in, u +#if CH_SPACEDIM == 3 + , + v +#endif + ); + + // Now do the coordinate transformation + Tensor<2, double> g_spherical = {0.}; + FOR4(i, j, k, l) + { + g_spherical[i][j] += Jac[i][k] * Jac[j][l] * g[k][l]; + } + + // Construct the 2-metric on the horizon in (u,v) coords + // i.e. substitute df = (df/du)du + (df/dv)dv + // into the spherical metric + Tensor<2, double, CH_SPACEDIM - 1> g_horizon = {0.}; + g_horizon[0][0] = g_spherical[1][1] + + g_spherical[0][0] * deriv.duF * deriv.duF + + 2.0 * g_spherical[0][1] * deriv.duF; +#if CH_SPACEDIM == 3 + g_horizon[1][1] = g_spherical[2][2] + + g_spherical[0][0] * deriv.dvF * deriv.dvF + + 2.0 * g_spherical[0][2] * deriv.dvF; + g_horizon[0][1] = g_horizon[1][0] = + g_spherical[1][2] + + g_spherical[0][0] * deriv.duF * deriv.dvF + + g_spherical[0][1] * deriv.dvF + + g_spherical[0][2] * deriv.duF; +#endif + + double det = TensorAlgebra::compute_determinant(g_horizon); + + double weight = m_integration_methods[0].weight( + u, m_params.num_points_u, m_interp.is_u_periodic()); + + double element = sqrt(det) * weight * m_du; + +// assume a (GR_SPACEDIM - CH_SPACEDIM)-sphere leftover +#if GR_SPACEDIM != CH_SPACEDIM + double n_sphere = (GR_SPACEDIM - CH_SPACEDIM); + element *= pow(sqrt(func.get_metric_hd()) * m_F[idx], n_sphere); +#endif + + inner_integral += element; + idx++; + } +#if CH_SPACEDIM == 3 + double weight = m_integration_methods[1].weight( + v, m_params.num_points_v, m_interp.is_v_periodic()); + integral += weight * m_dv * inner_integral; +#elif CH_SPACEDIM == 2 + integral += inner_integral; +#endif + } + +// do this separate so that the gamma is only calculated once +#if GR_SPACEDIM != CH_SPACEDIM + double n_sphere = (GR_SPACEDIM - CH_SPACEDIM); + // this is 2pi for n=1, 4pi for n=2, 2pi^2 for n=3, ... + double n_sphere_coeff = 2. * std::pow(M_PI, (n_sphere + 1.) / 2.) / + std::tgamma((n_sphere + 1.) / 2.); + integral *= n_sphere_coeff; +#endif + + DMDAVecRestoreArray(m_dmda, localF, &in); + DMRestoreLocalVector(m_dmda, &localF); + + m_interp.break_interpolation_loop(); + } + +// reduction across all Chombo processes (note that 'area' is 0 for non-PETSc +// processes) because SmallDataIO uses rank 0 to write this ensures rank 0 will +// have the area (even though for now the PETSc processes always include rank 0) +#ifdef CH_MPI + // we want all the processes to have the area so that all use it in the + // spin calculation (which in reality is needed not for the output files, + // but only for the pout() prints) + MPI_Allreduce(&integral, &area, 1, MPI_DOUBLE, MPI_SUM, Chombo_MPI::comm); +#else // serial + area = integral; +#endif + + if (m_params.verbose > AHFinder::NONE) + { + pout() << "area = " << area << endl; + } + + return area; +} + +template +std::array +ApparentHorizon::calculate_center() +{ + CH_TIME("ApparentHorizon::calculate_center"); + + if (!get_converged()) + { +#if CH_SPACEDIM == 3 + return {NAN, NAN, NAN}; +#elif CH_SPACEDIM == 2 + return {NAN, NAN}; +#endif + } + + // [OLD] Method: + // Calculate centroid of all the points, by summing the coordinates {x,y,z} + // of all the points, which are spread across processors, and then reducing + // them all with MPI. Finally, divide by total number of points summed, to + // get the centroid + // Above is not good. Replace by calculating the maximum and minimum point + // of the surface and estimating the center by their average + + // std::array temp = {0., 0., 0.}; // old method + std::array max_temp = get_origin(); + std::array min_temp = max_temp; + + int idx = 0; + if (AHFinder::is_rank_active()) // in principle this wouldn't be needed, as + // non-PETSc processes wouldn't even enter + // the loop anyways + { +#if CH_SPACEDIM == 3 + for (int v = m_vmin; v < m_vmax; ++v) +#endif + { + for (int u = m_umin; u < m_umax; ++u) + { + for (unsigned i = 0; i < CH_SPACEDIM; ++i) + { + double coord_i = + m_interp.get_grid_coord(i, m_F[idx], m_u[idx] +#if CH_SPACEDIM == 3 + , + m_v[idx] +#endif + ); + + // temp[i] += point[i]; // old method + if (coord_i > max_temp[i]) + max_temp[i] = coord_i; + if (coord_i < min_temp[i]) + min_temp[i] = coord_i; + } + + idx++; + } + } + } + + std::array max_point; + std::array min_point; + std::array center; + // int idx_sum; + +#ifdef CH_MPI + MPI_Allreduce(&max_temp, &max_point, CH_SPACEDIM, MPI_DOUBLE, MPI_MAX, + Chombo_MPI::comm); + MPI_Allreduce(&min_temp, &min_point, CH_SPACEDIM, MPI_DOUBLE, MPI_MIN, + Chombo_MPI::comm); + // MPI_Allreduce(¢er, ¢er, CH_SPACEDIM, MPI_DOUBLE, MPI_SUM, + // Chombo_MPI::comm); // old method + // MPI_Allreduce(&idx, &idx_sum, 1, MPI_INT, MPI_MAX, Chombo_MPI::comm); +#else // serial + max_point = max_temp; + min_point = min_temp; + // idx_sum = idx; +#endif + + // CH_assert(idx_sum != 0); // old method + + for (unsigned i = 0; i < CH_SPACEDIM; ++i) + { + // center[i] /= idx_sum; // old method + center[i] = (max_point[i] + min_point[i]) / 2.; + } + + if (m_params.verbose > AHFinder::SOME) + { + pout() << "max_point: (" << max_point[0] << ", " << max_point[1] +#if CH_SPACEDIM == 3 + << "," << max_point[2] +#endif + << ")" << std::endl; + pout() << "min_point: (" << min_point[0] << ", " << min_point[1] +#if CH_SPACEDIM == 3 + << ", " << min_point[2] +#endif + << ")" << std::endl; + } + + update_old_centers(center); // set new + + if (m_params.verbose > AHFinder::MIN) + { + pout() << "center: (" << center[0] << ", " << center[1] +#if CH_SPACEDIM == 3 + << ", " << center[2] +#endif + << ")" << std::endl; + } + + return center; +} + +template +void ApparentHorizon::calculate_minmax_F() const +{ + if (!get_converged()) + return; + + double local_max = std::numeric_limits::min(); + double local_min = std::numeric_limits::max(); + if (AHFinder::is_rank_active()) + { + auto local_minmax = (std::minmax_element(m_F.begin(), m_F.end())); + local_min = *(local_minmax.first); + local_max = *(local_minmax.second); + } + double global_max = local_max; + double global_min = local_min; + +#ifdef CH_MPI + MPI_Allreduce(&local_min, &global_min, 1, MPI_DOUBLE, MPI_MIN, + Chombo_MPI::comm); + MPI_Allreduce(&local_max, &global_max, 1, MPI_DOUBLE, MPI_MAX, + Chombo_MPI::comm); +#endif + + m_max_F = global_max; + m_min_F = global_min; +} + +template +void ApparentHorizon::calculate_average_F() const +{ + if (!get_converged()) + return; + + int local_size = 0; + double local_sum = 0.; + double local_sum_sq = 0.; + if (AHFinder::is_rank_active()) + { + std::pair sums(0., 0.); + auto lambda_sum = [](std::pair sums, double r) { + sums.first += r; // radius + sums.second += r * r; // radius^2 + return std::move(sums); + }; + + auto local_sums = + std::accumulate(m_F.begin(), m_F.end(), sums, lambda_sum); + + local_size = m_F.size(); + local_sum = local_sums.first; + local_sum_sq = local_sums.second; + } + int global_size = local_size; + double global_sum = local_sum; + double global_sum_sq = local_sum_sq; + +#ifdef CH_MPI + MPI_Allreduce(&local_size, &global_size, 1, MPI_INT, MPI_SUM, + Chombo_MPI::comm); + MPI_Allreduce(&local_sum, &global_sum, 1, MPI_DOUBLE, MPI_SUM, + Chombo_MPI::comm); + MPI_Allreduce(&global_sum_sq, &global_sum_sq, 1, MPI_DOUBLE, MPI_SUM, + Chombo_MPI::comm); +#endif + + m_ave_F = global_sum / global_size; + m_std_F = sqrt(global_sum_sq / global_size - + m_ave_F * m_ave_F); // sqrt( variance ) +} + +#endif // _APPARENTHORIZON_IMPL_HPP_ \ No newline at end of file diff --git a/Source/ApparentHorizonFinder/ApparentHorizon_petsc.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon_petsc.impl.hpp new file mode 100644 index 000000000..2950d8185 --- /dev/null +++ b/Source/ApparentHorizonFinder/ApparentHorizon_petsc.impl.hpp @@ -0,0 +1,959 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef _APPARENTHORIZON_HPP_ +#error "This file should only be included through ApparentHorizon.hpp" +#endif + +#ifndef _APPARENTHORIZON_PETSC_IMPL_HPP_ +#define _APPARENTHORIZON_PETSC_IMPL_HPP_ + +// petsc libraries +#include "petscsys.h" +#include "petscviewerhdf5.h" + +// for interpolation at restart: +#include "Lagrange.hpp" +#include "SimpleArrayBox.hpp" +#include "SimpleInterpSource.hpp" + +///////////////////////////////////////////////////////////////////////// +///////////////////////////////////////////////////////////////////////// +/////////////////////////// PETSc stuff below /////////////////////////// +///////////////////////////////////////////////////////////////////////// +///////////////////////////////////////////////////////////////////////// + +template +void ApparentHorizon::initialise_PETSc() +{ + CH_TIME("ApparentHorizon::initialise_PETSc"); + + if (!AHFinder::is_rank_active()) + return; + + const double lo_u = m_interp.get_domain_u_min(); + const double hi_u = m_interp.get_domain_u_max(); +#if CH_SPACEDIM == 3 // lo_v, hi_v not used otherwise + const double lo_v = m_interp.get_domain_v_min(); + const double hi_v = m_interp.get_domain_v_max(); +#endif + +#if PETSC_VERSION_LT(3, 5, 0) +#define DM_BOUNDARY_PERIODIC DMDA_BOUNDARY_PERIODIC +#define DM_BOUNDARY_GHOSTED DMDA_BOUNDARY_PERIODIC +#endif + +#if CH_SPACEDIM == 3 + DMDACreate2d(PETSC_COMM_WORLD, + m_periodic_u ? DM_BOUNDARY_PERIODIC : DM_BOUNDARY_GHOSTED, + m_periodic_v ? DM_BOUNDARY_PERIODIC : DM_BOUNDARY_GHOSTED, + DMDA_STENCIL_BOX, m_num_global_u, + m_num_global_v, /* grid size (negative means that we can + override it using command line) */ + PETSC_DECIDE, PETSC_DECIDE, /* distribution */ + 1, /* number of degrees of freedom */ + 3, /* stencil width (each side from central point) */ + NULL, NULL, &m_dmda); +#elif CH_SPACEDIM == 2 + DMDACreate1d(PETSC_COMM_WORLD, + m_periodic_u ? DM_BOUNDARY_PERIODIC : DM_BOUNDARY_GHOSTED, + m_num_global_u, /* grid size (negative means that we can + override it using command line) */ + 1, /* number of degrees of freedom */ + 3, /* stencil width (each side from central point) */ + NULL, &m_dmda); +#endif + + DMSetUp(m_dmda); + + // reload -> is it the same value? +#if CH_SPACEDIM == 3 // lo_v, hi_v not used otherwise + DMDAGetInfo(m_dmda, NULL, &m_num_global_u, &m_num_global_v, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); +#elif CH_SPACEDIM == 2 + DMDAGetInfo(m_dmda, NULL, &m_num_global_u, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL); +#endif + + m_du = + (hi_u - lo_u) / (m_periodic_u ? m_num_global_u : (m_num_global_u - 1)); + +#if CH_SPACEDIM == 3 + m_dv = + (hi_v - lo_v) / (m_periodic_v ? m_num_global_v : (m_num_global_v - 1)); +#endif + +#if CH_SPACEDIM == 3 + DMDAGetCorners(m_dmda, &m_umin, &m_vmin, NULL, &m_nu, &m_nv, NULL); + m_vmax = m_vmin + m_nv; +#elif CH_SPACEDIM == 2 + DMDAGetCorners(m_dmda, &m_umin, NULL, NULL, &m_nu, NULL, NULL); +#endif + m_umax = m_umin + m_nu; + +#if CH_SPACEDIM == 3 + int vec_size = m_nu * m_nv; +#elif CH_SPACEDIM == 2 + int vec_size = m_nu; +#endif + + m_F.resize(vec_size); + m_u.resize(vec_size); +#if CH_SPACEDIM == 3 + m_v.resize(vec_size); +#endif + +#if CH_SPACEDIM == 3 + for (int v = m_vmin; v < m_vmax; ++v) +#endif + { + for (int u = m_umin; u < m_umax; ++u) + { +#if CH_SPACEDIM == 3 + m_u[(v - m_vmin) * m_nu + (u - m_umin)] = lo_u + u * m_du; + m_v[(v - m_vmin) * m_nu + (u - m_umin)] = lo_v + v * m_dv; +#elif CH_SPACEDIM == 2 + m_u[u - m_umin] = lo_u + u * m_du; +#endif + } + } + + DMCreateGlobalVector(m_dmda, &m_snes_soln); + PetscObjectSetName((PetscObject)m_snes_soln, "F"); + + VecDuplicate(m_snes_soln, &m_snes_rhs); + PetscObjectSetName((PetscObject)m_snes_rhs, "expansion"); + +#if PETSC_VERSION_GE(3, 5, 0) + DMSetMatType(m_dmda, MATAIJ); + DMCreateMatrix(m_dmda, &m_snes_jac); + MatSetOption(m_snes_jac, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE); +#else + DMCreateMatrix(m_dmda, MATAIJ, &m_snes_jac); + MatSetOption(m_snes_jac, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE); +#endif + + SNESCreate(PETSC_COMM_WORLD, &m_snes); + SNESSetFunction(m_snes, m_snes_rhs, &Petsc_form_function, this); + SNESSetJacobian(m_snes, m_snes_jac, m_snes_jac, &Petsc_form_jacobian, this); + SNESMonitorSet(m_snes, &Petsc_SNES_monitor, this, NULL); + + SNESSetFromOptions(m_snes); +} + +template +void ApparentHorizon::finalise_PETSc() +{ + if (!AHFinder::is_rank_active()) + return; + + SNESDestroy(&m_snes); + VecDestroy(&m_snes_soln); + VecDestroy(&m_snes_rhs); + MatDestroy(&m_snes_jac); + DMDestroy(&m_dmda); +} + +template +bool ApparentHorizon::interpolate_ah( + dmda_arr_t f, const std::vector> &old_coords) +{ + // check if number of points changed + bool points_changed = false; + const double du = old_coords[0][1] - old_coords[0][0]; + const double lo_u = m_interp.get_domain_u_min(); + +#if CH_SPACEDIM == 3 + // assumes ordering in file is done first by fixed 'v', varying 'u' + // such that number of equal 'v's will be the number of 'u' points + int old_number_of_u = 1; + while (abs(old_coords[1][old_number_of_u - 1] - + old_coords[1][old_number_of_u]) < 1e-7) + ++old_number_of_u; + int old_number_of_v = old_coords[0].size() / old_number_of_u; + const double dv = + old_coords[1][old_number_of_u] - old_coords[1][old_number_of_u - 1]; + const double lo_v = m_interp.get_domain_v_min(); + // int total_new_points = m_num_global_u * m_num_global_v; + + points_changed |= (old_number_of_v != m_num_global_v); +#elif CH_SPACEDIM == 2 + int old_number_of_u = old_coords[0].size(); + // int total_new_points = m_num_global_u; +#endif + points_changed |= (old_number_of_u != m_num_global_u); + + if (!points_changed) + { + // these should be equal if all is good + // remember that only PETSc ranks have m_du and m_dv defined + // but only those should enter here, so all good +#if CH_SPACEDIM == 3 + CH_assert(abs(m_dv - dv) < 1e-7); + CH_assert(old_coords[0].size() == m_num_global_u * m_num_global_v); +#elif CH_SPACEDIM == 2 + CH_assert(old_coords[0].size() == m_num_global_u); +#endif + CH_assert(abs(m_du - du) < 1e-7); + + // select that F's that matter to this ranking +#if CH_SPACEDIM == 3 + for (int v = m_vmin; v < m_vmax; ++v) +#endif + { + for (int u = m_umin; u < m_umax; ++u) + { +#if CH_SPACEDIM == 3 + int idx_global = v * m_num_global_u + u; + int idx_local = (v - m_vmin) * m_nu + (u - m_umin); + if (std::abs((lo_v + v * m_dv) - old_coords[1][idx_global]) > + 1.e-7) + MayDay::Error( + "'v' coordinates incompatible with restart data"); +#elif CH_SPACEDIM == 2 + int idx_global = u; + int idx_local = (u - m_umin); +#endif + + if (std::abs((lo_u + u * m_du) - old_coords[0][idx_global]) > + 1.e-7) + MayDay::Error( + "'u' coordinates incompatible with restart data"); + + m_F[idx_local] = old_coords[CH_SPACEDIM - 1][idx_global]; + +#if CH_SPACEDIM == 3 + // as 'write_coords_file' preserves order + f[v][u] = m_F[idx_local]; +#elif CH_SPACEDIM == 2 + f[u] = m_F[idx_local]; +#endif + } + } + + return false; + } + + if (m_params.verbose > AHFinder::NONE) + { + pout() << "Number of AH points changed. Interpolating old ones." + << std::endl; + } + + // start interpolating + +#if CH_SPACEDIM == 3 + SimpleArrayBox box( + {old_number_of_u, old_number_of_v}, old_coords[CH_SPACEDIM - 1], + {m_interp.is_u_periodic(), m_interp.is_v_periodic()}); + SimpleInterpSource source( + {old_number_of_u, old_number_of_v}, + {m_interp.is_u_periodic(), m_interp.is_v_periodic()}); +#elif CH_SPACEDIM == 2 + SimpleArrayBox box({old_number_of_u}, + old_coords[CH_SPACEDIM - 1], + {m_interp.is_u_periodic()}); + SimpleInterpSource source({old_number_of_u}, + {m_interp.is_u_periodic()}); +#endif + + const int Order = 4; + bool verbose = false; + Lagrange interpolator4(source, verbose); + +#if CH_SPACEDIM == 3 + // local, no derivative + std::array derivs = {0, 0}; + std::array dxs = {du, dv}; +#elif CH_SPACEDIM == 2 + std::array derivs = {0}; + std::array dxs = {du}; +#endif + +#if CH_SPACEDIM == 3 + for (int v = m_vmin; v < m_vmax; ++v) + { + double v_val = lo_v + v * m_dv; + double v_old_idx = v_val / dv; + // round such that we don't get AMRInterpolator errors + // (e.g. the last point idx=N-1 should be exact, and not being exact was + // generating this error in the Lagrange intrepolator trying to access + // cells beyond the last cell, and hence error... this fixed it) + if (v_old_idx - int(v_old_idx) < 1.e-7) + v_old_idx = int(v_old_idx); +#else + { +#endif + for (int u = m_umin; u < m_umax; ++u) + { + double u_val = lo_u + u * m_du; + double u_old_idx = u_val / du; + // round such that we don't get AMRInterpolator errors (same as for + // 'v') + if (u_old_idx - int(u_old_idx) < 1.e-7) + u_old_idx = int(u_old_idx); + +#if CH_SPACEDIM == 3 + // int idx_global = v * m_num_global_u + u; + int idx_local = (v - m_vmin) * m_nu + (u - m_umin); + + std::array evalCoord = {u_old_idx, + v_old_idx}; +#elif CH_SPACEDIM == 2 + // int idx_global = u; + int idx_local = (u - m_umin); + std::array evalCoord = {u_old_idx}; +#endif + + interpolator4.setup(derivs, dxs, evalCoord); + m_F[idx_local] = interpolator4.interpData(box); + +#if CH_SPACEDIM == 3 + // as 'write_coords_file' preserves order + f[v][u] = m_F[idx_local]; +#elif CH_SPACEDIM == 2 + f[u] = m_F[idx_local]; +#endif + } + } + + if (m_params.verbose > AHFinder::NONE) + { + pout() << "Interpolation Successfull." << std::endl; + } + + return true; +} + +template +void ApparentHorizon::set_stencils(AHDeriv &out, + int u +#if CH_SPACEDIM == 3 + , + int v +#endif +) +{ +#if CH_SPACEDIM == 3 + const bool periodic[CH_SPACEDIM - 1] = {m_periodic_u, m_periodic_v}; + + int coord[CH_SPACEDIM - 1] = {u, v}; + const int N[CH_SPACEDIM - 1] = {m_num_global_u, m_num_global_v}; + + int d_start[CH_SPACEDIM - 1]; + double *const d_weights[CH_SPACEDIM - 1] = {out.du_weights, out.dv_weights}; + + int dd_start[CH_SPACEDIM - 1]; + double *const dd_weights[CH_SPACEDIM - 1] = {out.dudu_weights, + out.dvdv_weights}; +#elif CH_SPACEDIM == 2 + const bool periodic[CH_SPACEDIM - 1] = {m_periodic_u}; + + int coord[CH_SPACEDIM - 1] = {u}; + const int N[CH_SPACEDIM - 1] = {m_num_global_u}; + + int d_start[CH_SPACEDIM - 1]; + double *const d_weights[CH_SPACEDIM - 1] = {out.du_weights}; + + int dd_start[CH_SPACEDIM - 1]; + double *const dd_weights[CH_SPACEDIM - 1] = {out.dudu_weights}; +#endif + + for (int i = 0; i < CH_SPACEDIM - 1; ++i) + { + + // Non-periodic boundaries + if (!periodic[i] && (coord[i] == 0 || coord[i] == 1 || + coord[i] == N[i] - 2 || coord[i] == N[i] - 1)) + { + + if (coord[i] == 0) + { + + d_start[i] = 0; + + d_weights[i][0] = -2.083333333333333333333; + d_weights[i][1] = +4.000000000000000000000; + d_weights[i][2] = -3.000000000000000000000; + d_weights[i][3] = +1.333333333333333333333; + d_weights[i][4] = -0.250000000000000000000; + + dd_start[i] = 0; + + dd_weights[i][0] = +3.75000000000000000000; + dd_weights[i][1] = -12.8333333333333333333; + dd_weights[i][2] = +17.8333333333333333333; + dd_weights[i][3] = -13.0000000000000000000; + dd_weights[i][4] = +5.08333333333333333333; + dd_weights[i][5] = -0.83333333333333333333; + } + else if (coord[i] == 1) + { + + d_start[i] = -1; + + d_weights[i][0] = -0.250000000000000000000; + d_weights[i][1] = -0.833333333333333333333; + d_weights[i][2] = +1.500000000000000000000; + d_weights[i][3] = -0.500000000000000000000; + d_weights[i][4] = +0.083333333333333333333; + + dd_start[i] = -1; + + dd_weights[i][0] = +0.83333333333333333333; + dd_weights[i][1] = -1.25000000000000000000; + dd_weights[i][2] = -0.33333333333333333333; + dd_weights[i][3] = +1.16666666666666666667; + dd_weights[i][4] = -0.50000000000000000000; + dd_weights[i][5] = +0.08333333333333333333; + } + else if (coord[i] == N[i] - 2) + { + + d_start[i] = -3; + + d_weights[i][0] = -0.083333333333333333333; + d_weights[i][1] = +0.500000000000000000000; + d_weights[i][2] = -1.500000000000000000000; + d_weights[i][3] = +0.833333333333333333333; + d_weights[i][4] = +0.250000000000000000000; + + dd_start[i] = -4; + + dd_weights[i][0] = +0.08333333333333333333; + dd_weights[i][1] = -0.50000000000000000000; + dd_weights[i][2] = +1.16666666666666666667; + dd_weights[i][3] = -0.33333333333333333333; + dd_weights[i][4] = -1.25000000000000000000; + dd_weights[i][5] = +0.83333333333333333333; + } + else if (coord[i] == N[i] - 1) + { + + d_start[i] = -4; + + d_weights[i][0] = +0.250000000000000000000; + d_weights[i][1] = -1.333333333333333333333; + d_weights[i][2] = +3.000000000000000000000; + d_weights[i][3] = -4.000000000000000000000; + d_weights[i][4] = +2.083333333333333333333; + + dd_start[i] = -5; + + dd_weights[i][0] = -0.83333333333333333333; + dd_weights[i][1] = +5.08333333333333333333; + dd_weights[i][2] = -13.0000000000000000000; + dd_weights[i][3] = +17.8333333333333333333; + dd_weights[i][4] = -12.8333333333333333333; + dd_weights[i][5] = +3.75000000000000000000; + } + } + // Standard points + else + { + + d_start[i] = -2; + + d_weights[i][0] = +0.083333333333333333333; + d_weights[i][1] = -0.666666666666666666666; + d_weights[i][2] = 0; + d_weights[i][3] = +0.666666666666666666666; + d_weights[i][4] = -0.083333333333333333333; + + if (coord[i] == N[i] - 3) + { + dd_start[i] = -3; + + dd_weights[i][0] = 0; + dd_weights[i][1] = -0.08333333333333333333; + dd_weights[i][2] = +1.33333333333333333333; + dd_weights[i][3] = -2.50000000000000000000; + dd_weights[i][4] = +1.33333333333333333333; + dd_weights[i][5] = -0.08333333333333333333; + } + else + { + dd_start[i] = -2; + + dd_weights[i][0] = -0.08333333333333333333; + dd_weights[i][1] = +1.33333333333333333333; + dd_weights[i][2] = -2.50000000000000000000; + dd_weights[i][3] = +1.33333333333333333333; + dd_weights[i][4] = -0.08333333333333333333; + dd_weights[i][5] = 0; + } + } + } + + out.du_stencil_start = d_start[0]; + out.dudu_stencil_start = dd_start[0]; + +#if CH_SPACEDIM == 3 + out.dv_stencil_start = d_start[1]; + out.dvdv_stencil_start = dd_start[1]; +#endif +} + +template +AHDeriv ApparentHorizon::diff(const dmda_arr_t in, + int u +#if CH_SPACEDIM == 3 + , + int v +#endif +) +{ + CH_TIME("ApparentHorizon::diff"); + + AHDeriv out; + set_stencils(out, u +#if CH_SPACEDIM == 3 + , + v +#endif + ); + +#if CH_SPACEDIM == 3 + // d/du + for (int j = 0; j < DWIDTH; ++j) + { + out.duF += + out.du_weights[j] * in[v][u + out.du_stencil_start + j] / m_du; + + // d2/dudv + for (int k = 0; k < DWIDTH; ++k) + { + out.dudvF += + out.du_weights[j] * out.dv_weights[k] * + in[v + out.dv_stencil_start + k][u + out.du_stencil_start + j] / + (m_du * m_dv); + } + } + + // d/dv + for (int j = 0; j < DWIDTH; ++j) + { + out.dvF += + out.dv_weights[j] * in[v + out.dv_stencil_start + j][u] / m_dv; + } + + // d2/du2 + for (int j = 0; j < DDWIDTH; ++j) + { + out.duduF += out.dudu_weights[j] * + in[v][u + out.dudu_stencil_start + j] / (m_du * m_du); + } + + // d2/dv2 + for (int j = 0; j < DDWIDTH; ++j) + { + out.dvdvF += out.dvdv_weights[j] * + in[v + out.dvdv_stencil_start + j][u] / (m_dv * m_dv); + } + +#elif CH_SPACEDIM == 2 + + // d/du + for (int j = 0; j < DWIDTH; ++j) + { + out.duF += out.du_weights[j] * in[u + out.du_stencil_start + j] / m_du; + } + + // d2/du2 + for (int j = 0; j < DDWIDTH; ++j) + { + out.duduF += out.dudu_weights[j] * in[u + out.dudu_stencil_start + j] / + (m_du * m_du); + } + +#endif + + return out; +} + +template +void ApparentHorizon::form_function(Vec F, Vec Rhs) +{ + CH_TIME("ApparentHorizon::form_function"); + + // Scatter ghost cells + Vec localF; + DMGetLocalVector(m_dmda, &localF); + DMGlobalToLocalBegin(m_dmda, F, INSERT_VALUES, localF); + DMGlobalToLocalEnd(m_dmda, F, INSERT_VALUES, localF); + + dmda_arr_t in; + DMDAVecGetArray(m_dmda, localF, &in); + + dmda_arr_t out; + DMDAVecGetArray(m_dmda, Rhs, &out); + +#if CH_SPACEDIM == 3 + for (int v = m_vmin; v < m_vmax; ++v) +#endif + { + for (int u = m_umin; u < m_umax; ++u) + { +#if CH_SPACEDIM == 3 + m_F[(v - m_vmin) * m_nu + (u - m_umin)] = in[v][u]; +#elif CH_SPACEDIM == 2 + m_F[u - m_umin] = in[u]; +#endif + } + } + + bool out_of_grid = m_interp.set_coordinates(m_F, m_u +#if CH_SPACEDIM == 3 + , + m_v +#endif + ); + // abort if out of grid - reduces the time in divergence dramatically! + if (out_of_grid) + SNESSetFunctionDomainError(m_snes); + + // must interpolate anyway, as other PETSc ranks do not know if this one is + // "out_of_grid" + m_interp.interpolate(); + + int idx = 0; + +#if CH_SPACEDIM == 3 + for (int v = m_vmin; v < m_vmax; ++v) +#endif + { + for (int u = m_umin; u < m_umax; ++u) + { + +#if CH_SPACEDIM == 3 + double &_out = out[v][u]; +#elif CH_SPACEDIM == 2 + double &_out = out[u]; +#endif + + AHDeriv deriv = diff(in, u +#if CH_SPACEDIM == 3 + , + v +#endif + ); + + if (out_of_grid) // no need to calculate expansion + _out = 0.; + else if (!m_periodic_u && (u == 0 || u == m_num_global_u - 1)) + _out = deriv.duF; + +#if CH_SPACEDIM == 3 + else if (!m_periodic_v && (v == 0 || v == m_num_global_v - 1)) + _out = deriv.dvF; +#endif + + else + { + const auto geometry_data = m_interp.get_geometry_data(idx); + const auto data = m_interp.get_data(idx); + const auto coords = m_interp.get_cartesian_coords(idx); + AHFunction func(data, coords); + _out = func.get(geometry_data, deriv, m_func_params); + } + + ++idx; + } + } + + DMDAVecRestoreArray(m_dmda, localF, &in); + DMDAVecRestoreArray(m_dmda, Rhs, &out); + DMRestoreLocalVector(m_dmda, &localF); +} + +template +void ApparentHorizon::form_jacobian(Vec F, Mat J) +{ + CH_TIME("ApparentHorizon::form_jacobian"); + + // Scatter ghost cells + Vec localF; + DMGetLocalVector(m_dmda, &localF); + DMGlobalToLocalBegin(m_dmda, F, INSERT_VALUES, localF); + DMGlobalToLocalEnd(m_dmda, F, INSERT_VALUES, localF); + + dmda_arr_t in; + DMDAVecGetArray(m_dmda, localF, &in); + + bool out_of_grid = m_interp_plus.set_coordinates(m_F, m_u, +#if CH_SPACEDIM == 3 + m_v, +#endif + eps); + out_of_grid |= m_interp_minus.set_coordinates(m_F, m_u, +#if CH_SPACEDIM == 3 + m_v, +#endif + -eps); + // abort if out of grid - reduces the time in divergence dramatically! + if (out_of_grid) + SNESSetFunctionDomainError(m_snes); + + // must interpolate anyway, as other PETSc ranks do not know if this one is + // "out_of_grid" + m_interp_plus.interpolate(); + m_interp_minus.interpolate(); + + int idx = 0; + +#if CH_SPACEDIM == 3 + for (int v = m_vmin; v < m_vmax; ++v) +#endif + { + for (int u = m_umin; u < m_umax; ++u) + { + MatStencil row[1] = {0}; + row[0].i = u; +#if CH_SPACEDIM == 3 + row[0].j = v; +#endif + + if (out_of_grid) // no need to calculate expansion + { + MatStencil col[DWIDTH] = {0}; + double val[DWIDTH] = {0}; + MatSetValuesStencil(J, 1, row, DWIDTH, col, val, INSERT_VALUES); + } + else if (!m_periodic_u && (u == 0 || u == m_num_global_u - 1)) + { + MatStencil col[DWIDTH] = {0}; + double val[DWIDTH] = {0}; + + const AHDeriv deriv = diff(in, u +#if CH_SPACEDIM == 3 + , + v +#endif + ); + + for (int a = 0; a < DWIDTH; ++a) + { + col[a].i = u + deriv.du_stencil_start + a; +#if CH_SPACEDIM == 3 + col[a].j = v; +#endif + val[a] = deriv.du_weights[a]; + } + + MatSetValuesStencil(J, 1, row, DWIDTH, col, val, INSERT_VALUES); + } + +#if CH_SPACEDIM == 3 + else if (!m_periodic_v && (v == 0 || v == m_num_global_v - 1)) + { + MatStencil col[DWIDTH] = {0}; + double val[DWIDTH] = {0}; + + const AHDeriv deriv = diff(in, u, v); + + for (int b = 0; b < DWIDTH; ++b) + { + col[b].i = u; + col[b].j = v + deriv.dv_stencil_start + b; + val[b] = deriv.dv_weights[b]; + } + + MatSetValuesStencil(J, 1, row, DWIDTH, col, val, INSERT_VALUES); + } +#endif + else + { + +#if CH_SPACEDIM == 3 + const int NVAL = DDWIDTH * DDWIDTH; +#elif CH_SPACEDIM == 2 + const int NVAL = DDWIDTH; +#endif + MatStencil col[NVAL] = {0}; + double val[NVAL] = {0}; + + const AHDeriv deriv_default = diff(in, u +#if CH_SPACEDIM == 3 + , + v +#endif + ); + + // "local" and "stencil" jacobian terms + // "local" (point {u,v}) corresponds to (a == + // -deriv_default.dudu_stencil_start + // && b == -deriv_default.dvdv_stencil_start) and has to use + // 'm_interp_plus' and 'm_interp_minus' + { +#if CH_SPACEDIM == 3 + for (int b = 0; b < DDWIDTH; ++b) + { +#elif CH_SPACEDIM == 2 + { + int b = 0; +#endif + for (int a = 0; a < DDWIDTH; ++a) + { + col[b * DDWIDTH + a].i = + u + deriv_default.dudu_stencil_start + a; + const int uu = col[b * DDWIDTH + a].i; + const int u_start = + -deriv_default.dudu_stencil_start; +#if CH_SPACEDIM == 3 + col[b * DDWIDTH + a].j = + v + deriv_default.dvdv_stencil_start + b; + const int vv = col[b * DDWIDTH + a].j; + const int v_start = + -deriv_default.dvdv_stencil_start; +#elif CH_SPACEDIM == 2 + const int v_start = 0; +#endif + + if (a == u_start && b == v_start) // <=> uu=u, vv=v + { + val[b * DDWIDTH + a] = point_jacobian( + u, uu, +#if CH_SPACEDIM == 3 + v, vv, +#endif + in, idx, m_interp_plus, m_interp_minus); + } + else + { + val[b * DDWIDTH + a] = + point_jacobian(u, uu, +#if CH_SPACEDIM == 3 + v, vv, +#endif + in, idx, m_interp, m_interp); + } + } + } + } + + MatSetValuesStencil(J, 1, row, NVAL, col, val, INSERT_VALUES); + } + + ++idx; + } + } + + DMDAVecRestoreArray(m_dmda, localF, &in); + DMRestoreLocalVector(m_dmda, &localF); + + MatAssemblyBegin(J, MAT_FINAL_ASSEMBLY); + MatAssemblyEnd(J, MAT_FINAL_ASSEMBLY); +} + +template +double ApparentHorizon::point_jacobian( + int u, int u_stencil, +#if CH_SPACEDIM == 3 + int v, int v_stencil, +#endif + dmda_arr_t in, int idx, + const AHInterpolation &interp_plus, + const AHInterpolation &interp_minus) +{ +#if CH_SPACEDIM == 3 + double &_in = in[v_stencil][u_stencil]; +#elif CH_SPACEDIM == 2 + double &_in = in[u_stencil]; +#endif + + // "plus" perturbation + double expansionPlus; + { + double in_old = _in; + _in += eps; // perturb just the point {u,v} + + const AHDeriv deriv = diff(in, u +#if CH_SPACEDIM == 3 + , + v +#endif + ); + + const auto geometry_data = interp_plus.get_geometry_data(idx); + const auto data = interp_plus.get_data(idx); + const auto coords = interp_plus.get_cartesian_coords(idx); + AHFunction func(data, coords); + expansionPlus = func.get(geometry_data, deriv, m_func_params); + + _in = in_old; + } + + // "minus" perturbation + double expansionMinus; + { + double in_old = _in; + _in -= eps; // perturb just the point {u,v} + + const AHDeriv deriv = diff(in, u +#if CH_SPACEDIM == 3 + , + v +#endif + ); + + const auto geometry_data = interp_minus.get_geometry_data(idx); + const auto data = interp_minus.get_data(idx); + const auto coords = interp_minus.get_cartesian_coords(idx); + AHFunction func(data, coords); + expansionMinus = func.get(geometry_data, deriv, m_func_params); + + _in = in_old; + } + + return (expansionPlus - expansionMinus) / (2. * eps); +} + +//! functions used by PETSc based on 'form_function' and 'form_jacobian' +template +PetscErrorCode +ApparentHorizon::Petsc_form_function(SNES snes, + Vec F, + Vec Rhs, + void *ptr) +{ + ApparentHorizon &ah = *reinterpret_cast(ptr); + CH_assert(ah.m_snes == snes); + ah.form_function(F, Rhs); + return 0; +} + +template +PetscErrorCode +#if PETSC_VERSION_GE(3, 5, 0) +ApparentHorizon::Petsc_form_jacobian( + SNES snes, Vec F, Mat Amat, Mat Pmat, void *ptr) +#else +ApparentHorizon::Petsc_form_jacobian( + SNES snes, Vec F, Mat *Amat, Mat *Pmat, MatStructure *flag, void *ptr) +#endif +{ + ApparentHorizon &ah = *reinterpret_cast(ptr); + +#if PETSC_VERSION_GE(3, 5, 0) + CH_assert(ah.m_snes == snes && Amat == Pmat); + ah.form_jacobian(F, Amat); +#else + CH_assert(ah.m_snes == snes && *Amat == *Pmat); + ah.form_jacobian(F, *Amat); +#endif + + return 0; +} + +template +PetscErrorCode ApparentHorizon::Petsc_SNES_monitor( + SNES snes, PetscInt its, PetscReal norm, void *ptr) +{ + ApparentHorizon &ah = *reinterpret_cast(ptr); + CH_assert(ah.m_snes == snes); + return 0; +} + +#endif // _APPARENTHORIZON_PETSC_IMPL_HPP_ diff --git a/Source/BlackHoles/BHAMR.hpp b/Source/BlackHoles/BHAMR.hpp index f5f33e79e..9febe1ddc 100644 --- a/Source/BlackHoles/BHAMR.hpp +++ b/Source/BlackHoles/BHAMR.hpp @@ -7,7 +7,13 @@ #define BHAMR_HPP_ #include "GRAMR.hpp" +#if CH_SPACEDIM == 3 #include "PunctureTracker.hpp" +#endif + +#ifdef USE_AHFINDER +#include "AHFinder.hpp" +#endif /// A child of Chombo's AMR class to interface with tools which require /// access to the whole AMR hierarchy, and those of GRAMR @@ -17,14 +23,25 @@ class BHAMR : public GRAMR { public: +#if CH_SPACEDIM == 3 PunctureTracker m_puncture_tracker; +#endif + +#ifdef USE_AHFINDER + AHFinder m_ah_finder; +#endif BHAMR() {} void set_interpolator(AMRInterpolator> *a_interpolator) override { GRAMR::set_interpolator(a_interpolator); +#if CH_SPACEDIM == 3 m_puncture_tracker.set_interpolator(a_interpolator); +#endif +#ifdef USE_AHFINDER + m_ah_finder.set_interpolator(a_interpolator); +#endif } }; diff --git a/Source/BlackHoles/PunctureTracker.cpp b/Source/BlackHoles/PunctureTracker.cpp index 55b45b1f0..542ea2d0e 100644 --- a/Source/BlackHoles/PunctureTracker.cpp +++ b/Source/BlackHoles/PunctureTracker.cpp @@ -3,6 +3,8 @@ * Please refer to LICENSE in GRChombo's root directory. */ +#if CH_SPACEDIM == 3 + #include "PunctureTracker.hpp" #include "ChomboParameters.hpp" // for writing data #include "DimensionDefinitions.hpp" @@ -230,3 +232,5 @@ std::vector PunctureTracker::get_puncture_vector() const } return puncture_vector; } + +#endif // #if CH_SPACEDIM == 3 diff --git a/Source/BoxUtils/BoxLoops.impl.hpp b/Source/BoxUtils/BoxLoops.impl.hpp index 1ce7baba6..e8767df61 100644 --- a/Source/BoxUtils/BoxLoops.impl.hpp +++ b/Source/BoxUtils/BoxLoops.impl.hpp @@ -34,14 +34,14 @@ void BoxLoops::innermost_loop(const ComputePack &compute_pack, for (int ix = loop_lo_x; ix <= x_simd_max; ix += simd_width) { compute_pack.call_compute( - Cell>(IntVect(ix, iy, iz), box_pointers)); + Cell>(IntVect(D_DECL(ix, iy, iz)), box_pointers)); } // REMAINDER LOOP for (int ix = x_simd_max + simd::simd_len; ix <= loop_hi_x; ++ix) { compute_pack.call_compute( - Cell(IntVect(ix, iy, iz), box_pointers)); + Cell(IntVect(D_DECL(ix, iy, iz)), box_pointers)); } } @@ -54,7 +54,7 @@ void BoxLoops::innermost_loop(const ComputePack &compute_pack, for (int ix = loop_lo_x; ix <= loop_hi_x; ++ix) { compute_pack.call_compute( - Cell(IntVect(ix, iy, iz), box_pointers)); + Cell(IntVect(D_DECL(ix, iy, iz)), box_pointers)); } } @@ -71,6 +71,9 @@ void BoxLoops::loop(const ComputePack &compute_pack, const int *loop_lo = loop_box.loVect(); const int *loop_hi = loop_box.hiVect(); +#if CH_SPACEDIM == 2 + int iz = 0; +#endif #pragma omp parallel for default(shared) collapse(CH_SPACEDIM - 1) #if CH_SPACEDIM >= 3 for (int iz = loop_lo[2]; iz <= loop_hi[2]; ++iz) diff --git a/Source/BoxUtils/BoxPointers.hpp b/Source/BoxUtils/BoxPointers.hpp index 120b156c2..d2c2cdf36 100644 --- a/Source/BoxUtils/BoxPointers.hpp +++ b/Source/BoxUtils/BoxPointers.hpp @@ -74,16 +74,16 @@ class BoxPointers CellIndexIn get_in_index(IntVect integer_coords) const { - return (m_in_stride[2] * (integer_coords[2] - m_in_lo[2]) + - m_in_stride[1] * (integer_coords[1] - m_in_lo[1]) + - (integer_coords[0] - m_in_lo[0])); + return (D_TERM((integer_coords[0] - m_in_lo[0]), + +m_in_stride[1] * (integer_coords[1] - m_in_lo[1]), + +m_in_stride[2] * (integer_coords[2] - m_in_lo[2]))); } CellIndexOut get_out_index(IntVect integer_coords) const { - return (m_out_stride[2] * (integer_coords[2] - m_out_lo[2]) + - m_out_stride[1] * (integer_coords[1] - m_out_lo[1]) + - (integer_coords[0] - m_out_lo[0])); + return (D_TERM((integer_coords[0] - m_out_lo[0]), + +m_out_stride[1] * (integer_coords[1] - m_out_lo[1]), + +m_out_stride[2] * (integer_coords[2] - m_out_lo[2]))); } }; diff --git a/Source/BoxUtils/NanCheck.hpp b/Source/BoxUtils/NanCheck.hpp index 2e814c6fe..2d700c5d4 100644 --- a/Source/BoxUtils/NanCheck.hpp +++ b/Source/BoxUtils/NanCheck.hpp @@ -7,6 +7,7 @@ #define NANCHECK_HPP_ #include "Cell.hpp" +#include "Coordinates.hpp" #include "UserVariables.hpp" /// This compute class checks for nans or very large values and aborts the @@ -14,17 +15,23 @@ class NanCheck { protected: - const std::string m_error_info = "NanCheck"; - const double m_max_abs = 1e20; + const double m_dx; + const std::array &m_center; + const std::string m_error_info; + const double m_max_abs; public: - NanCheck() {} - - /// This constructor takes a string which will be displayed when nans happen - NanCheck(const std::string a_error_info) : m_error_info(a_error_info) {} + NanCheck(const std::string a_error_info = "NanCheck", + const double a_max_abs = 1e20) + : m_dx(-1), m_center{}, m_error_info(a_error_info), m_max_abs(a_max_abs) + { + } - NanCheck(const std::string a_error_info, const double a_max_abs) - : m_error_info(a_error_info), m_max_abs(a_max_abs) + NanCheck(const double a_dx, const std::array &a_center, + const std::string a_error_info = "NanCheck", + const double a_max_abs = 1e20) + : m_dx(a_dx), m_center(a_center), m_error_info(a_error_info), + m_max_abs(a_max_abs) { } @@ -52,6 +59,11 @@ class NanCheck } pout() << "Integer coordinates: " << current_cell.get_int_vect() << std::endl; + if (m_dx > 0) + { + Coordinates coords{current_cell, m_dx, m_center}; + pout() << coords << std::endl; + } } MayDay::Error("Values have become nan."); } diff --git a/Source/CCZ4/ADMConformalVars.hpp b/Source/CCZ4/ADMConformalVars.hpp index a45a98d07..8b51a9930 100644 --- a/Source/CCZ4/ADMConformalVars.hpp +++ b/Source/CCZ4/ADMConformalVars.hpp @@ -39,10 +39,21 @@ template struct VarsNoGauge define_enum_mapping(mapping_function, c_K, K); // Symmetric 2-tensors +#if CH_SPACEDIM == 3 define_symmetric_enum_mapping(mapping_function, GRInterval(), h); define_symmetric_enum_mapping(mapping_function, GRInterval(), A); +#elif CH_SPACEDIM == 2 + define_symmetric_enum_mapping(mapping_function, + GRInterval(), h); + define_symmetric_enum_mapping(mapping_function, + GRInterval(), A); +#else +#ifdef CH_SPACEDIM +#error define_enum_mapping() has not got your dimension combination implemented. +#endif +#endif } }; @@ -60,8 +71,17 @@ template struct VarsWithGauge : public VarsNoGauge using namespace VarsTools; // define_enum_mapping is part of VarsTools VarsNoGauge::enum_mapping(mapping_function); define_enum_mapping(mapping_function, c_lapse, lapse); +#if CH_SPACEDIM == 3 define_enum_mapping(mapping_function, GRInterval(), shift); +#elif CH_SPACEDIM == 2 + define_enum_mapping(mapping_function, GRInterval(), + shift); +#else +#ifdef CH_SPACEDIM +#error define_enum_mapping() has not got your dimension combination implemented. +#endif +#endif } }; @@ -76,8 +96,17 @@ template struct Diff2VarsNoGauge { using namespace VarsTools; // define_enum_mapping is part of VarsTools define_enum_mapping(mapping_function, c_chi, chi); +#if CH_SPACEDIM == 3 define_symmetric_enum_mapping(mapping_function, GRInterval(), h); +#elif CH_SPACEDIM == 2 + define_symmetric_enum_mapping(mapping_function, + GRInterval(), h); +#else +#ifdef CH_SPACEDIM +#error define_enum_mapping() has not got your dimension combination implemented. +#endif +#endif } }; @@ -96,8 +125,17 @@ struct Diff2VarsWithGauge : public Diff2VarsNoGauge using namespace VarsTools; // define_enum_mapping is part of VarsTools Diff2VarsNoGauge::enum_mapping(mapping_function); define_enum_mapping(mapping_function, c_lapse, lapse); +#if CH_SPACEDIM == 3 define_enum_mapping(mapping_function, GRInterval(), shift); +#elif CH_SPACEDIM == 2 + define_enum_mapping(mapping_function, GRInterval(), + shift); +#else +#ifdef CH_SPACEDIM +#error define_enum_mapping() has not got your dimension combination implemented. +#endif +#endif } }; } // namespace ADMConformalVars diff --git a/Source/CCZ4/BSSNVars.hpp b/Source/CCZ4/BSSNVars.hpp index e5f563dad..b7c322a3b 100644 --- a/Source/CCZ4/BSSNVars.hpp +++ b/Source/CCZ4/BSSNVars.hpp @@ -31,8 +31,17 @@ struct VarsNoGauge : public ADMConformalVars::VarsNoGauge { using namespace VarsTools; // define_enum_mapping is part of VarsTools ADMConformalVars::VarsNoGauge::enum_mapping(mapping_function); +#if CH_SPACEDIM == 3 define_enum_mapping(mapping_function, GRInterval(), Gamma); //!< The auxilliary variable Gamma^i +#elif CH_SPACEDIM == 2 + define_enum_mapping(mapping_function, GRInterval(), + Gamma); //!< The auxilliary variable Gamma^i +#else +#ifdef CH_SPACEDIM +#error define_enum_mapping() has not got your dimension combination implemented. +#endif +#endif } }; @@ -52,9 +61,19 @@ template struct VarsWithGauge : public VarsNoGauge using namespace VarsTools; // define_enum_mapping is part of VarsTools VarsNoGauge::enum_mapping(mapping_function); define_enum_mapping(mapping_function, c_lapse, lapse); +#if CH_SPACEDIM == 3 define_enum_mapping(mapping_function, GRInterval(), shift); define_enum_mapping(mapping_function, GRInterval(), B); +#elif CH_SPACEDIM == 2 + define_enum_mapping(mapping_function, GRInterval(), + shift); + define_enum_mapping(mapping_function, GRInterval(), B); +#else +#ifdef CH_SPACEDIM +#error define_enum_mapping() has not got your dimension combination implemented. +#endif +#endif } }; diff --git a/Source/GRChomboCore/BoundaryConditions.cpp b/Source/GRChomboCore/BoundaryConditions.cpp index ce9244326..8ab7c88f1 100644 --- a/Source/GRChomboCore/BoundaryConditions.cpp +++ b/Source/GRChomboCore/BoundaryConditions.cpp @@ -681,8 +681,13 @@ void BoundaryConditions::fill_extrapolating_cell( for (int icomp : extrapolating_comps) { // current radius - double radius = Coordinates::get_radius( - iv, m_dx, {m_center[0], m_center[1], m_center[2]}); + double radius = Coordinates::get_radius(iv, m_dx, { + m_center[0], m_center[1] +#if CH_SPACEDIM == 3 + , + m_center[2] +#endif + }); // vector of 2 nearest values and radii within the grid std::array value_at_point; @@ -710,8 +715,13 @@ void BoundaryConditions::fill_extrapolating_cell( } } value_at_point[i] = out_box(iv_tmp, icomp); - r_at_point[i] = Coordinates::get_radius( - iv_tmp, m_dx, {m_center[0], m_center[1], m_center[2]}); + r_at_point[i] = Coordinates::get_radius(iv_tmp, m_dx, { + m_center[0], m_center[1] +#if CH_SPACEDIM == 3 + , + m_center[2] +#endif + }); } } else // Lo side @@ -735,8 +745,13 @@ void BoundaryConditions::fill_extrapolating_cell( } } value_at_point[i] = out_box(iv_tmp, icomp); - r_at_point[i] = Coordinates::get_radius( - iv_tmp, m_dx, {m_center[0], m_center[1], m_center[2]}); + r_at_point[i] = Coordinates::get_radius(iv_tmp, m_dx, { + m_center[0], m_center[1] +#if CH_SPACEDIM == 3 + , + m_center[2] +#endif + }); } } diff --git a/Source/GRChomboCore/GRAMR.cpp b/Source/GRChomboCore/GRAMR.cpp index 07f9d4d60..fc95bae0a 100644 --- a/Source/GRChomboCore/GRAMR.cpp +++ b/Source/GRChomboCore/GRAMR.cpp @@ -11,6 +11,8 @@ GRAMR::GRAMR() : m_interpolator(nullptr) {} // Called after AMR object set up void GRAMR::set_interpolator(AMRInterpolator> *a_interpolator) { + // check AMRInterpolator was set + CH_assert(a_interpolator != nullptr); m_interpolator = a_interpolator; } diff --git a/Source/GRChomboCore/GRAMR.hpp b/Source/GRChomboCore/GRAMR.hpp index ae911493f..41a65fdc7 100644 --- a/Source/GRChomboCore/GRAMR.hpp +++ b/Source/GRChomboCore/GRAMR.hpp @@ -60,6 +60,9 @@ class GRAMR : public AMR // const version of above std::vector get_gramrlevels() const; + + int get_max_level() const { return m_max_level; } + int get_restart_step() const { return m_restart_step; } }; #endif /* GRAMR_HPP_ */ diff --git a/Source/GRChomboCore/GRAMRLevel.cpp b/Source/GRChomboCore/GRAMRLevel.cpp index f5387341c..a7b9d7aa3 100644 --- a/Source/GRChomboCore/GRAMRLevel.cpp +++ b/Source/GRChomboCore/GRAMRLevel.cpp @@ -221,30 +221,27 @@ void GRAMRLevel::tagCells(IntVectSet &a_tags) const IntVect &smallEnd = b.smallEnd(); const IntVect &bigEnd = b.bigEnd(); - const int xmin = smallEnd[0]; - const int ymin = smallEnd[1]; - const int zmin = smallEnd[2]; - - const int xmax = bigEnd[0]; - const int ymax = bigEnd[1]; - const int zmax = bigEnd[2]; - -#pragma omp parallel for collapse(3) schedule(static) default(shared) - for (int iz = zmin; iz <= zmax; ++iz) - for (int iy = ymin; iy <= ymax; ++iy) - for (int ix = xmin; ix <= xmax; ++ix) - { - IntVect iv(ix, iy, iz); - if (tagging_criterion(iv, 0) >= - m_p.regrid_thresholds[m_level]) - { + D_TERM(const int xmin = smallEnd[0];, const int ymin = smallEnd[1]; + , const int zmin = smallEnd[2];) + + D_TERM(const int xmax = bigEnd[0];, const int ymax = bigEnd[1]; + , const int zmax = bigEnd[2];) + +#pragma omp parallel for collapse(CH_SPACEDIM) schedule(static) default(shared) + D_INVTERM(for (int ix = xmin; ix <= xmax; ++ix), + for (int iy = ymin; iy <= ymax; ++iy), + for (int iz = zmin; iz <= zmax; ++iz)) + { + IntVect iv(D_DECL(ix, iy, iz)); + if (tagging_criterion(iv, 0) >= m_p.regrid_thresholds[m_level]) + { // local_tags |= is not thread safe. #pragma omp critical - { - local_tags |= iv; - } - } + { + local_tags |= iv; } + } + } } local_tags.grow(m_p.tag_buffer_size); diff --git a/Source/GRChomboCore/GRAMRLevel.hpp b/Source/GRChomboCore/GRAMRLevel.hpp index b7709fd0d..9e3acb55f 100644 --- a/Source/GRChomboCore/GRAMRLevel.hpp +++ b/Source/GRChomboCore/GRAMRLevel.hpp @@ -27,7 +27,7 @@ // Chombo namespace #include "UsingNamespace.H" -class GRAMRLevel : public AMRLevel, public InterpSource +class GRAMRLevel : public AMRLevel, public InterpSource<> { public: GRAMRLevel(GRAMR &gr_amr, const SimulationParameters &a_p, int a_verbosity); diff --git a/Source/GRChomboCore/SimulationParametersBase.hpp b/Source/GRChomboCore/SimulationParametersBase.hpp index bc8e0331b..06ddbf129 100644 --- a/Source/GRChomboCore/SimulationParametersBase.hpp +++ b/Source/GRChomboCore/SimulationParametersBase.hpp @@ -11,11 +11,17 @@ #include "CCZ4.hpp" #include "ChomboParameters.hpp" #include "GRParmParse.hpp" -#include "SphericalExtraction.hpp" #include +#ifdef USE_AHFINDER +#include "AHFinder.hpp" +#endif + +#if CH_SPACEDIM == 3 +#include "SphericalExtraction.hpp" // add this type alias here for backwards compatibility using extraction_params_t = SphericalExtraction::params_t; +#endif class SimulationParametersBase : public ChomboParameters { @@ -53,6 +59,7 @@ class SimulationParametersBase : public ChomboParameters pp.load("min_chi", min_chi, 1e-4); pp.load("min_lapse", min_lapse, 1e-4); +#if CH_SPACEDIM == 3 // Extraction params pp.load("num_extraction_radii", extraction_params.num_extraction_radii, 1); @@ -117,6 +124,14 @@ class SimulationParametersBase : public ChomboParameters } pp.load("write_extraction", extraction_params.write_extraction, false); +#endif + +#ifdef USE_AHFINDER + // Apparent horizon parameters + pp.load("AH_activate", AH_activate, false); + if (AH_activate) + AH_params.read_params(pp, *this); +#endif } void check_params() @@ -197,6 +212,7 @@ class SimulationParametersBase : public ChomboParameters std::numeric_limits::epsilon(), "set to 2.0 for 1+log slicing"); +#if CH_SPACEDIM == 3 // Now extraction parameters FOR1(idir) { @@ -236,6 +252,7 @@ class SimulationParametersBase : public ChomboParameters check_parameter(mode_name, value_str, (l >= 2) && (abs(m) <= l), "l must be >= 2 and m must satisfy -l <= m <= l"); } +#endif } public: @@ -249,7 +266,14 @@ class SimulationParametersBase : public ChomboParameters // Collection of parameters necessary for the CCZ4 RHS and extraction CCZ4::params_t ccz4_params; +#if CH_SPACEDIM == 3 SphericalExtraction::params_t extraction_params; +#endif + +#ifdef USE_AHFINDER + bool AH_activate; + AHFinder::params AH_params; +#endif }; #endif /* SIMULATIONPARAMETERSBASE_HPP_ */ diff --git a/Source/InitialConditions/BlackHoles/SingleBH.hpp b/Source/InitialConditions/BlackHoles/SingleBH.hpp new file mode 100644 index 000000000..293fa6d2e --- /dev/null +++ b/Source/InitialConditions/BlackHoles/SingleBH.hpp @@ -0,0 +1,50 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef SINGLEBH_HPP_ +#define SINGLEBH_HPP_ + +#include "BoostedBH.hpp" +#include "Cell.hpp" +#include "Coordinates.hpp" +#include "Tensor.hpp" +#include "UserVariables.hpp" //This files needs NUM_VARS - total number of components +#include "simd.hpp" +#include + +enum Lapse +{ + ONE, + PRE_COLLAPSED, + CHI +}; + +class SingleBH +{ + protected: + double m_dx; + BoostedBH bh; + int m_initial_lapse; + + public: + SingleBH(BoostedBH::params_t a_bh_params, double a_dx, + int a_initial_lapse = Lapse::PRE_COLLAPSED) + : m_dx(a_dx), bh(a_bh_params), m_initial_lapse(a_initial_lapse) + { + } + + template void compute(Cell current_cell) const; + + protected: + template + data_t compute_chi(Coordinates coords) const; + + template + Tensor<2, data_t> compute_A(data_t chi, Coordinates coords) const; +}; + +#include "SingleBH.impl.hpp" + +#endif /* SINGLEBH_HPP_ */ diff --git a/Source/InitialConditions/BlackHoles/SingleBH.impl.hpp b/Source/InitialConditions/BlackHoles/SingleBH.impl.hpp new file mode 100644 index 000000000..9a59cf777 --- /dev/null +++ b/Source/InitialConditions/BlackHoles/SingleBH.impl.hpp @@ -0,0 +1,71 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#if !defined(SINGLEBH_HPP_) +#error "This file should only be included through SingleBH.hpp" +#endif + +#ifndef SINGLEBH_IMPL_HPP_ +#define SINGLEBH_IMPL_HPP_ + +#include "BSSNVars.hpp" +#include "SingleBH.hpp" +#include "VarsTools.hpp" +#include "simd.hpp" + +template void SingleBH::compute(Cell current_cell) const +{ + BSSNVars::VarsWithGauge vars; + VarsTools::assign(vars, + 0.); // Set only the non-zero components explicitly below + Coordinates coords(current_cell, m_dx); + + vars.chi = compute_chi(coords); + + // Conformal metric is flat + FOR1(i) vars.h[i][i] = 1.; + + vars.A = compute_A(vars.chi, coords); + + switch (m_initial_lapse) + { + case Lapse::ONE: + vars.lapse = 1.; + break; + case Lapse::PRE_COLLAPSED: + vars.lapse = sqrt(vars.chi); + break; + case Lapse::CHI: + vars.lapse = vars.chi; + break; + default: + MayDay::Error("SingleBH::Supplied initial lapse not supported."); + } + + current_cell.store_vars(vars); +} + +template +data_t SingleBH::compute_chi(Coordinates coords) const +{ + const data_t psi = 1. + bh.psi_minus_one(coords); + return pow(psi, -4); +} + +template +Tensor<2, data_t> SingleBH::compute_A(data_t chi, + Coordinates coords) const +{ + + Tensor<2, data_t> Aij = bh.Aij(coords); + Tensor<2, data_t> out; + + // Aij(CCZ4) = psi^(-6) * Aij(Baumgarte&Shapiro book) + FOR2(i, j) out[i][j] = pow(chi, 3 / 2.) * Aij[i][j]; + + return out; +} + +#endif /* SINGLEBH_IMPL_HPP_ */ diff --git a/Source/utils/Coordinates.hpp b/Source/utils/Coordinates.hpp index 47d5f0f1e..593cc2c48 100644 --- a/Source/utils/Coordinates.hpp +++ b/Source/utils/Coordinates.hpp @@ -22,7 +22,10 @@ template class Coordinates public: data_t x; // We vectorise over x so we must allow x to be a vector double y; +#if !(DEFAULT_TENSOR_DIM == CH_SPACEDIM && CH_SPACEDIM == 2) + // don't even add a 'z' double z; +#endif std::array m_center; Coordinates(IntVect integer_coords, double dx, @@ -38,6 +41,8 @@ template class Coordinates #elif DEFAULT_TENSOR_DIM == CH_SPACEDIM + 1 && CH_SPACEDIM == 2 y = 0; compute_coord(z, integer_coords[1], dx, center[1]); +#elif DEFAULT_TENSOR_DIM == CH_SPACEDIM && CH_SPACEDIM == 2 + compute_coord(y, integer_coords[1], dx, center[1]); #else #ifdef CH_SPACEDIM #error compute_coord has not got your dimension combination implemented. @@ -69,7 +74,7 @@ template class Coordinates data_t get_radius() const { // Note that this is not currently dimension independent - data_t r = sqrt(pow(x, 2) + pow(y, 2) + pow(z, 2)); + data_t r = sqrt(D_TERM(x * x, +y * y, +z * z)); const double minimum_r = 1e-6; return simd_max(r, minimum_r); @@ -82,14 +87,17 @@ template class Coordinates { data_t xx; double yy; - double zz; // Note that this is not currently dimension independent compute_coord(xx, integer_coords[0], dx, center[0]); compute_coord(yy, integer_coords[1], dx, center[1]); + +#if CH_SPACEDIM == 3 + double zz; compute_coord(zz, integer_coords[2], dx, center[2]); +#endif - data_t r = sqrt(pow(xx, 2) + pow(yy, 2) + pow(zz, 2)); + data_t r = sqrt(D_TERM(xx * xx, +yy * yy, +zz * zz)); const double minimum_r = 1e-6; return simd_max(r, minimum_r); @@ -100,9 +108,11 @@ template ALWAYS_INLINE ostream &operator<<(ostream &os, const Coordinates &in_coords) { - os << "(x,y,z) = (" << in_coords.x << "," << in_coords.y << "," - << in_coords.z << ")" - << " r = " << in_coords.get_radius(); + os << "(x,y,z) = (" << in_coords.x << "," << in_coords.y << +#if CH_SPACEDIM == 3 + "," << in_coords.z << +#endif + ") r = " << in_coords.get_radius(); return os; } #endif /* COORDINATES_HPP_ */ diff --git a/Source/utils/SmallDataIO.cpp b/Source/utils/SmallDataIO.cpp index f11848d97..87820297c 100644 --- a/Source/utils/SmallDataIO.cpp +++ b/Source/utils/SmallDataIO.cpp @@ -57,7 +57,7 @@ SmallDataIO::SmallDataIO(std::string a_filename_prefix, double a_dt, // overwrite any existing file if this is the first step file_openmode = std::ios::out; } - else if (m_restart_time > 0. && + else if (m_restart_time >= 0. && m_time < m_restart_time + m_dt + m_coords_epsilon) { // allow reading in the restart case so that duplicate time @@ -205,7 +205,7 @@ void SmallDataIO::line_break() void SmallDataIO::remove_duplicate_time_data(const bool keep_m_time_data) { - if (m_rank == 0 && m_restart_time > 0. && m_mode == APPEND && + if (m_rank == 0 && m_restart_time >= 0. && m_mode == APPEND && m_time < m_restart_time + m_dt + m_coords_epsilon) { // copy lines with time < m_time into a temporary file @@ -316,6 +316,12 @@ std::string SmallDataIO::get_new_filename(const std::string &a_file_prefix, CH_assert(a_dt > 0); const int step = std::round(a_time / a_dt); + // append step number to filename if in NEW mode + return a_file_prefix + pad_number(step) + a_file_extension; +} + +std::string SmallDataIO::pad_number(int step, int a_filename_steps_width) +{ // append step number to filename (pad to make it // a_filename_steps_width digits). std::string step_string = std::to_string(step); @@ -327,8 +333,7 @@ std::string SmallDataIO::get_new_filename(const std::string &a_file_prefix, std::string step_string_padded = std::string(a_filename_steps_width - step_string.length(), '0') + step_string; - // append step number to filename if in NEW mode - return a_file_prefix + step_string_padded + a_file_extension; + return step_string_padded; } // returns m_data_epsilon @@ -348,3 +353,120 @@ double SmallDataIO::get_default_coords_epsilon() { return pow(10.0, -s_default_coords_precision); } + +std::vector> SmallDataIO::read(std::string a_filename, + bool verbose) +{ + int rank = 0; +#ifdef CH_MPI + MPI_Comm_rank(Chombo_MPI::comm, &rank); +#endif + + int Nrows = 0, Ncols = 0; + + std::fstream file; + + if (rank == 0) + { + file.open(a_filename, std::ios::in); + if (!file) + { + if (verbose) + pout() << "File '" << a_filename << "' not found." << std::endl; + } + else + { + // run through file to get number of rows and cols + std::string line, save = ""; + while (std::getline(file, line)) + { + if (line.substr(0, 2) == "//" || line.substr(0, 1) == "#") + continue; + else if (save == "") + save = line; + ++Nrows; + } + + // run through line and count number of columns + std::stringstream str(save); + + std::string x_str; + double x; + str >> x_str; + if (x_str == "nan") + x = NAN; + else + (std::stringstream(x_str) >> x); + while (!str.fail()) + { + str >> x_str; + if (x_str == "nan") + x = NAN; + else + (std::stringstream(x_str) >> x); + ++Ncols; + } + } + } + +#ifdef CH_MPI + MPI_Bcast(&Nrows, 1, MPI_INT, 0, Chombo_MPI::comm); + MPI_Bcast(&Ncols, 1, MPI_INT, 0, Chombo_MPI::comm); +#endif + + if (rank == 0 && Nrows != 0 && verbose) + std::cout << "Found " << Ncols << " columns and " << Nrows + << " rows in file '" << a_filename << "'." << std::endl; + + std::vector> out(Ncols, std::vector(Nrows)); + + if (rank == 0 && Nrows != 0) + { + file.clear(); + file.seekg(0, file.beg); + + int j = 0; + std::string line; + while (std::getline(file, line)) + { + if (line.substr(0, 2) == "//" || line.substr(0, 1) == "#") + continue; + + std::stringstream ss(line); + std::string x_str; + double x; + ss >> x_str; + if (x_str == "nan") + x = NAN; + else + (std::stringstream(x_str) >> x); + + int i = 0; + while (!ss.fail() && i < Ncols) + { + out[i++][j] = x; + ss >> x_str; + if (x_str == "nan") + x = NAN; + else + (std::stringstream(x_str) >> x); + } + ++j; + } + + file.close(); + } + +#ifdef CH_MPI + for (int i = 0; i < Ncols; ++i) + MPI_Bcast(&out[i][0], Nrows, MPI_DOUBLE, 0, Chombo_MPI::comm); +#endif + + /* if(rank==0) + for(int i=0; i Begin: Helper for the assign function diff --git a/Tests/AMRInterpolatorTest/AMRInterpolatorTest.cpp b/Tests/AMRInterpolatorTest/AMRInterpolatorTest.cpp index c0ea26aa3..69a95a435 100644 --- a/Tests/AMRInterpolatorTest/AMRInterpolatorTest.cpp +++ b/Tests/AMRInterpolatorTest/AMRInterpolatorTest.cpp @@ -123,9 +123,9 @@ int main(int argc, char *argv[]) int status = runInterpolatorTest(argc, argv); if (status == 0) - pout() << "BasicAMRInterpolator test passed." << endl; + pout() << "AMRInterpolator test passed." << endl; else - pout() << "BasicAMRInterpolator test failed with return code " << status + pout() << "AMRInterpolator test failed with return code " << status << endl; mainFinalize(); diff --git a/Tests/ApparentHorizonFinderTest2D/AHTest2DFunction.hpp b/Tests/ApparentHorizonFinderTest2D/AHTest2DFunction.hpp new file mode 100644 index 000000000..385ba9781 --- /dev/null +++ b/Tests/ApparentHorizonFinderTest2D/AHTest2DFunction.hpp @@ -0,0 +1,38 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef _AHTEST2DFUNCTION_HPP_ +#define _AHTEST2DFUNCTION_HPP_ + +#include "AHFunctionDefault.hpp" + +#include "AHData.hpp" +#include "AHDeriv.hpp" +#include "AHGeometryData.hpp" + +#include "UserVariables.hpp" + +// to look for x == y +struct AHTest2DFunction : AHFunctionDefault +{ + double v; + + static int vars_min() { return c_V; } + static int vars_max() { return c_V; } + + AHTest2DFunction(const AHData &a_data, + const Tensor<1, double> &a_coords) + { + v = a_data.vars.at(c_V); + } + + double get(const AHGeometryData &geo_data, const AHDeriv &deriv, + const params &a_params) const + { + return v; + } +}; + +#endif /* _AHTEST2DFUNCTION_HPP_ */ diff --git a/Tests/ApparentHorizonFinderTest2D/ApparentHorizonTest2D.cpp b/Tests/ApparentHorizonFinderTest2D/ApparentHorizonTest2D.cpp new file mode 100644 index 000000000..e92f39034 --- /dev/null +++ b/Tests/ApparentHorizonFinderTest2D/ApparentHorizonTest2D.cpp @@ -0,0 +1,127 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifdef CH_LANG_CC +/* + * _______ __ + * / ___/ / ___ __ _ / / ___ + * / /__/ _ \/ _ \/ V \/ _ \/ _ \ + * \___/_//_/\___/_/_/_/_.__/\___/ + * Please refer to LICENSE, in Chombo's root directory. + */ +#endif +/* +The purpose of this test is to test: + 1) if the AHFinder code remains compatible with 2D + 2) if GRChombo is properly setup also to work with 2D (Chombo should +recompile again for 2D when compiling this example) +*/ + +// General includes: +#include + +#include "BHAMR.hpp" +#include "parstream.H" //Gives us pout() + +#include "SetupFunctions.hpp" +#include "SimulationParameters.hpp" + +// Problem specific includes: +#include "AMRInterpolator.hpp" +#include "ApparentHorizonTest2DLevel.hpp" +#include "Lagrange.hpp" +#include "SmallDataIO.hpp" + +#ifdef USE_AHFINDER +#include "ApparentHorizon.hpp" +#endif + +int runApparentHorizonTest2D(int argc, char *argv[]) +{ + // Load the parameter file and construct the SimulationParameters class + // To add more parameters edit the SimulationParameters file. + std::string in_string = argv[argc - 1]; + pout() << in_string << std::endl; + GRParmParse pp(0, argv + argc, NULL, in_string.c_str()); + SimulationParameters sim_params(pp); + + BHAMR bh_amr; + DefaultLevelFactory ah_test_level_fact( + bh_amr, sim_params); + setupAMRObject(bh_amr, ah_test_level_fact); + + int status = 0; + +#ifdef USE_AHFINDER + AHFinder::params AH_params = {1, 20, 1, 1, false, false, 0, 0., -1.}; + AH_params.verbose = 3; + + // Set up interpolator and PETSc subcommunicator when AH extraction is + // active + AMRInterpolator> interpolator( + bh_amr, sim_params.origin, sim_params.dx, sim_params.boundary_params, + sim_params.verbosity); + bh_amr.set_interpolator(&interpolator); + + AHStringGeometry sph(sim_params.L); + // use 2. as initial guess, doesn't matter much (as long as it converges to + // the correct solution) + bh_amr.m_ah_finder.add_ah(sph, 2., AH_params); + + if (!bh_amr.m_ah_finder.get(0)->get_converged()) + status = 1; + else + { + // get area from file to determine status + auto stats = SmallDataIO::read("stats_AH1.dat"); + if (stats.size() == 0) + status = 1; + else + { + double area = stats[2][0]; + + // Exact value from Mathematica + /* + Solve[y + A * Sin[2 Pi x / L] == Pi, y][[1]] + NIntegrate[ Sqrt[ 1 + D[y/.%,x]^2 ], {x, 0, L} ] + + For A = 1 and L = 16 + */ + double area_exact = 16.600072; + + double error_perc = fabs(1. - area / area_exact) * 100; + pout() << "error = " << error_perc << "%" << std::endl; + status |= (error_perc > 0.1) || + +std::isnan( + error_perc); // accept 0.1% error in area calculation + } + } +#endif + + return status; +} + +int main(int argc, char *argv[]) +{ + mainSetup(argc, argv); + + int status = runApparentHorizonTest2D(argc, argv); + + if (status == 0) + { + std::cout << "ApparentHorizon2D test passed." << endl; + pout() << "ApparentHorizon2D test passed." << endl; + } + else + { + std::cout << "ApparentHorizon2D test FAILED with return code " << status + << endl; + pout() << "ApparentHorizon2D test FAILED with return code " << status + << endl; + } + + mainFinalize(); + return status; +} diff --git a/Tests/ApparentHorizonFinderTest2D/ApparentHorizonTest2D.inputs b/Tests/ApparentHorizonFinderTest2D/ApparentHorizonTest2D.inputs new file mode 100644 index 000000000..4c0106cf0 --- /dev/null +++ b/Tests/ApparentHorizonFinderTest2D/ApparentHorizonTest2D.inputs @@ -0,0 +1,25 @@ +chk_prefix = AHTest2D +plot_prefix = AHTest2DPlot +verbosity = 0 +N_full = 32 +L_full = 16 + +center = 0 0 + +num_ghosts = 3 +max_level = 1 +regrid_interval = 1 1 +ref_ratio = 2 2 + +isPeriodic = 1 0 +hi_boundary = 1 1 +lo_boundary = 1 2 + +vars_parity = 2 # V +vars_asymptotic_values = 0. # V + +#Max and min box sizes +max_grid_size = 32 +block_factor = 4 +tag_buffer_size = 3 +checkpoint_interval = 1 diff --git a/Tests/ApparentHorizonFinderTest2D/ApparentHorizonTest2DLevel.hpp b/Tests/ApparentHorizonFinderTest2D/ApparentHorizonTest2DLevel.hpp new file mode 100644 index 000000000..76f13c356 --- /dev/null +++ b/Tests/ApparentHorizonFinderTest2D/ApparentHorizonTest2DLevel.hpp @@ -0,0 +1,68 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef APPARENT_HORIZON_TEST2DLEVEL_HPP_ +#define APPARENT_HORIZON_TEST2DLEVEL_HPP_ + +#include "DefaultLevelFactory.hpp" +#include "GRAMRLevel.hpp" +#include "GRLevelData.hpp" + +class ApparentHorizonTest2DLevel : public GRAMRLevel +{ + friend class DefaultLevelFactory; + // Inherit the contructors from GRAMRLevel + using GRAMRLevel::GRAMRLevel; + + // initialize data + virtual void initialData() + { + + const DisjointBoxLayout &level_domain = m_state_new.disjointBoxLayout(); + + DataIterator dit = level_domain.dataIterator(); + + for (dit.begin(); dit.ok(); ++dit) + { + FArrayBox &fab = m_state_new[dit]; + const Box &b = level_domain[dit]; + + const IntVect &smallEnd = b.smallEnd(); + const IntVect &bigEnd = b.bigEnd(); + + const int xmin = smallEnd[0]; + const int ymin = smallEnd[1]; + + const int xmax = bigEnd[0]; + const int ymax = bigEnd[1]; + + // assigning values of 'chi' to a Schwarzschild BH in isotropic + // coordinates + for (int iy = ymin - 3; iy <= ymax + 3; ++iy) + for (int ix = xmin - 3; ix <= xmax + 3; ++ix) + { + const double x = (ix + 0.5) * m_dx - m_p.center[0]; + const double y = (iy + 0.5) * m_dx - m_p.center[1]; + const IntVect iv(ix, iy); + static const double perturb = 1; + const double y_perturbed = + abs(y) + perturb * sin(2. * M_PI * x / m_p.L); + // this is basically a decaying wave (sin(y)/y) slightly + // deformed in the 'x' direction + fab(iv, c_V) = sin(y_perturbed) / y_perturbed; + } + } + } + + virtual void specificEvalRHS(GRLevelData &a_soln, GRLevelData &a_rhs, + const double a_time) + { + } + + virtual void computeTaggingCriterion(FArrayBox &tagging_criterion, + const FArrayBox ¤t_state){}; +}; + +#endif /* APPARENT_HORIZON_TEST2DLEVEL_HPP_ */ diff --git a/Tests/ApparentHorizonFinderTest2D/GNUmakefile b/Tests/ApparentHorizonFinderTest2D/GNUmakefile new file mode 100644 index 000000000..655f421cd --- /dev/null +++ b/Tests/ApparentHorizonFinderTest2D/GNUmakefile @@ -0,0 +1,28 @@ +# -*- Mode: Makefile -*- + +### This makefile produces an executable for each name in the `ebase' +### variable using the libraries named in the `LibNames' variable. + +# Included makefiles need an absolute path to the Chombo installation +# CHOMBO_HOME := Please set the CHOMBO_HOME locally (e.g. export CHOMBO_HOME=... in bash) +# + +DIM = 2 +cxxcppflags := ${cxxcppflags} -DGR_SPACEDIM=2 -DDEFAULT_TENSOR_DIM=2 + +GRCHOMBO_SOURCE = ../../Source + +ebase := ApparentHorizonTest2D + +LibNames := AMRTimeDependent AMRTools BoxTools + +src_dirs := $(GRCHOMBO_SOURCE)/utils \ + $(GRCHOMBO_SOURCE)/simd \ + $(GRCHOMBO_SOURCE)/BoxUtils \ + $(GRCHOMBO_SOURCE)/GRChomboCore \ + $(GRCHOMBO_SOURCE)/AMRInterpolator \ + $(GRCHOMBO_SOURCE)/ApparentHorizonFinder \ + $(GRCHOMBO_SOURCE)/InitialConditions/BlackHoles \ + $(GRCHOMBO_SOURCE)/BlackHoles + +include $(CHOMBO_HOME)/mk/Make.test diff --git a/Tests/ApparentHorizonFinderTest2D/SimulationParameters.hpp b/Tests/ApparentHorizonFinderTest2D/SimulationParameters.hpp new file mode 100755 index 000000000..adc9824b2 --- /dev/null +++ b/Tests/ApparentHorizonFinderTest2D/SimulationParameters.hpp @@ -0,0 +1,18 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef SIMULATIONPARAMETERS_HPP_ +#define SIMULATIONPARAMETERS_HPP_ + +// General includes +#include "ChomboParameters.hpp" + +class SimulationParameters : public ChomboParameters +{ + public: + SimulationParameters(GRParmParse &pp) : ChomboParameters(pp) {} +}; + +#endif /* SIMULATIONPARAMETERS_HPP_ */ diff --git a/Tests/ApparentHorizonFinderTest2D/UserVariables.hpp b/Tests/ApparentHorizonFinderTest2D/UserVariables.hpp new file mode 100644 index 000000000..8d3433bdd --- /dev/null +++ b/Tests/ApparentHorizonFinderTest2D/UserVariables.hpp @@ -0,0 +1,35 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef USERVARIABLES_HPP +#define USERVARIABLES_HPP + +#include "EmptyDiagnosticVariables.hpp" +#include +#include + +// assign enum to each variable +enum +{ + c_V, + + NUM_VARS +}; + +namespace UserVariables +{ +static const std::array variable_names = {"V"}; +} + +#include "UserVariables.inc.hpp" + +// already the 2D default +// #include "AHStringGeometry.hpp" +// #define AHSurfaceGeometry AHStringGeometry + +#include "AHTest2DFunction.hpp" +#define AHFunction AHTest2DFunction + +#endif /* USERVARIABLES_HPP */ diff --git a/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.cpp b/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.cpp new file mode 100755 index 000000000..0f1bf27f1 --- /dev/null +++ b/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.cpp @@ -0,0 +1,127 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifdef CH_LANG_CC +/* + * _______ __ + * / ___/ / ___ __ _ / / ___ + * / /__/ _ \/ _ \/ V \/ _ \/ _ \ + * \___/_//_/\___/_/_/_/_.__/\___/ + * Please refer to LICENSE, in Chombo's root directory. + */ +#endif + +// General includes: +#include +#include +#include +#include +#include + +#include "parstream.H" //Gives us pout() +using std::endl; +#include "BHAMR.hpp" + +#include "SetupFunctions.hpp" +#include "SimulationParameters.hpp" + +// Problem specific includes: +#include "AMRInterpolator.hpp" +#include "ApparentHorizonTest3DLevel.hpp" +#include "InterpolationQuery.hpp" +#include "Lagrange.hpp" +#include "SmallDataIO.hpp" +#include "UserVariables.hpp" + +#ifdef _OPENMP +#include +#endif + +#ifdef USE_AHFINDER +#include "ApparentHorizon.hpp" +#endif + +int runApparentHorizonTest3D(int argc, char *argv[]) +{ + // Load the parameter file and construct the SimulationParameter class + // To add more parameters edit the SimulationParameters file. + std::string in_string = argv[argc - 1]; + pout() << in_string << std::endl; + GRParmParse pp(0, argv + argc, NULL, in_string.c_str()); + SimulationParameters sim_params(pp); + + BHAMR bh_amr; + DefaultLevelFactory ah_test_level_fact( + bh_amr, sim_params); + setupAMRObject(bh_amr, ah_test_level_fact); + + int status = 0; + +#ifdef USE_AHFINDER + AHFinder::params AH_params = {1, 15, 20, 1, 1, false, false, 0, 0., -1.}; + AH_params.verbose = 3; + + // Set up interpolator and PETSc subcommunicator when AH extraction is + // active + AMRInterpolator> interpolator( + bh_amr, sim_params.origin, sim_params.dx, sim_params.boundary_params, + sim_params.verbosity); + bh_amr.set_interpolator(&interpolator); + + AHSphericalGeometry sph(sim_params.center); + bh_amr.m_ah_finder.add_ah(sph, sim_params.initial_guess, AH_params); + + if (!bh_amr.m_ah_finder.get(0)->get_converged()) + status = 1; + else + { + + // get area from file to determine status + auto stats = SmallDataIO::read("stats_AH1.dat"); + if (stats.size() == 0) + status = 1; + else + { + double spin = stats[3][0]; + double mass = stats[4][0]; + + double error_perc = + fabs(1. - mass / sim_params.kerr_params.mass) * 100; + if (sim_params.kerr_params.spin != 0.) + error_perc += + fabs(1. - spin / sim_params.kerr_params.spin) * 100; + pout() << "error = " << error_perc << "%" << std::endl; + status |= (error_perc > 0.1) || + +std::isnan( + error_perc); // accept 0.1% error in area calculation + } + } +#endif + + return status; +} + +int main(int argc, char *argv[]) +{ + mainSetup(argc, argv); + + int status = runApparentHorizonTest3D(argc, argv); + + if (status == 0) + { + std::cout << "ApparentHorizon3D test passed." << endl; + pout() << "ApparentHorizon3D test passed." << endl; + } + else + { + std::cout << "ApparentHorizon3D test FAILED with return code " << status + << endl; + pout() << "ApparentHorizon3D test FAILED with return code " << status + << endl; + } + + mainFinalize(); + return status; +} diff --git a/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.inputs b/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.inputs new file mode 100755 index 000000000..a374c32cf --- /dev/null +++ b/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.inputs @@ -0,0 +1,36 @@ +chk_prefix = AHTest3D +plot_prefix = AHTest3DPlot +verbosity = 0 +N_full = 32 +L_full = 16 + +num_ghosts = 3 +max_level = 1 +regrid_interval = 1 1 1 1 0 0 0 0 0 +ref_ratio = 2 2 2 2 2 2 2 2 2 + +isPeriodic = 0 0 0 +hi_boundary = 1 1 1 +lo_boundary = 1 1 2 + +vars_parity = 0 0 4 6 0 5 0 #chi and hij + 0 0 4 6 0 5 0 #K and Aij + 0 1 2 3 #Theta and Gamma + 0 1 2 3 1 2 3 #lapse shift and B + 0 1 2 3 # Ham and Mom + +vars_asymptotic_values = 1. 1. 0. 0. 1. 0. 1. #chi and hij + 0. 0. 0. 0. 0. 0. 0. #K and Aij + 0. 0. 0. 0. #Theta and Gamma + 1. 0. 0. 0. 0. 0. 0. #lapse shift and B + 0. 0. 0. 0. # Ham and Mom + +#Max and min box sizes +max_grid_size = 32 +block_factor = 4 +tag_buffer_size = 3 +checkpoint_interval = 1 + +kerr_mass = 10. +kerr_spin = 5. +#initial_guess = 5 diff --git a/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3DLevel.hpp b/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3DLevel.hpp new file mode 100755 index 000000000..30fb581de --- /dev/null +++ b/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3DLevel.hpp @@ -0,0 +1,97 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef APPARENT_HORIZON_TEST3DLEVEL_HPP_ +#define APPARENT_HORIZON_TEST3DLEVEL_HPP_ + +#include "AMRLevel.H" +#include "BoxLoops.hpp" +#include "CoarseAverage.H" +#include "DefaultLevelFactory.hpp" +#include "FourthOrderFillPatch.H" +#include "GRAMR.hpp" +#include "GRAMRLevel.hpp" +#include "GRLevelData.hpp" +#include "GammaCalculator.hpp" +#include "InterpSource.hpp" +#include "KerrBH.hpp" +#include "LevelFluxRegister.H" //We don't actually use flux conservation but Chombo assumes we do +#include "LevelRK4.H" +#include "SetValue.hpp" + +class ApparentHorizonTest3DLevel : public GRAMRLevel +{ + friend class DefaultLevelFactory; + // Inherit the contructors from GRAMRLevel + using GRAMRLevel::GRAMRLevel; + + // initialize data + virtual void initialData() + { + /* + m_state_new.setVal(0); + m_state_new.setVal(1, c_h11); + m_state_new.setVal(1, c_h22); + m_state_new.setVal(1, c_h33); + const DisjointBoxLayout &level_domain = m_state_new.disjointBoxLayout(); + + DataIterator dit = level_domain.dataIterator(); + double mass = m_p.mass; + + for (dit.begin(); dit.ok(); ++dit) + { + FArrayBox &fab = m_state_new[dit]; + const Box &b = level_domain[dit]; + + const IntVect &smallEnd = b.smallEnd(); + const IntVect &bigEnd = b.bigEnd(); + + const int xmin = smallEnd[0]; + const int ymin = smallEnd[1]; + const int zmin = smallEnd[2]; + + const int xmax = bigEnd[0]; + const int ymax = bigEnd[1]; + const int zmax = bigEnd[2]; + + // assigning values of 'chi' to a Schwarzschild BH in isotropic + // coordinates + for (int iz = zmin - 3; iz <= zmax + 3; ++iz) + for (int iy = ymin - 3; iy <= ymax + 3; ++iy) + for (int ix = xmin - 3; ix <= xmax + 3; ++ix) + { + const double x = (ix + 0.5) * m_dx - m_p.center[0]; + const double y = (iy + 0.5) * m_dx - m_p.center[1]; + const double z = (iz + 0.5) * m_dx - m_p.center[2]; + const double r = sqrt(x * x + y * y + z * z); + const IntVect iv(ix, iy, iz); + fab(iv, c_chi) = pow((1 + mass / (2 * r)), -4.0); + } + } + */ + + // First set everything to zero then calculate initial data + // Get the Kerr solution in the variables, then no need to calculate the + // \tilde\Gamma^i numerically (not calculated in the Kerr ICs) as not + // needed in AHFinder + BoxLoops::loop( + make_compute_pack(SetValue(0.), KerrBH(m_p.kerr_params, m_dx)), + m_state_new, m_state_new, INCLUDE_GHOST_CELLS); + + // fillAllGhosts(); + // BoxLoops::loop(GammaCalculator(m_dx), m_state_new, m_state_new, + // EXCLUDE_GHOST_CELLS); + } + + virtual void specificEvalRHS(GRLevelData &a_soln, GRLevelData &a_rhs, + const double a_time) + { + } + + virtual void computeTaggingCriterion(FArrayBox &tagging_criterion, + const FArrayBox ¤t_state){}; +}; + +#endif /* APPARENT_HORIZON_TEST3DLEVEL_HPP_ */ diff --git a/Tests/ApparentHorizonFinderTest3D/GNUmakefile b/Tests/ApparentHorizonFinderTest3D/GNUmakefile new file mode 100755 index 000000000..c01c5b935 --- /dev/null +++ b/Tests/ApparentHorizonFinderTest3D/GNUmakefile @@ -0,0 +1,27 @@ +# -*- Mode: Makefile -*- + +### This makefile produces an executable for each name in the `ebase' +### variable using the libraries named in the `LibNames' variable. + +# Included makefiles need an absolute path to the Chombo installation +# CHOMBO_HOME := Please set the CHOMBO_HOME locally (e.g. export CHOMBO_HOME=... in bash) +# + +GRCHOMBO_SOURCE = $(shell pwd)/../../Source + +ebase := ApparentHorizonTest3D + +LibNames := AMRTimeDependent AMRTools BoxTools + +src_dirs := $(GRCHOMBO_SOURCE)/utils \ + $(GRCHOMBO_SOURCE)/simd \ + $(GRCHOMBO_SOURCE)/BoxUtils \ + $(GRCHOMBO_SOURCE)/GRChomboCore \ + $(GRCHOMBO_SOURCE)/AMRInterpolator \ + $(GRCHOMBO_SOURCE)/ApparentHorizonFinder \ + $(GRCHOMBO_SOURCE)/InitialConditions/BlackHoles \ + $(GRCHOMBO_SOURCE)/BlackHoles \ + $(GRCHOMBO_SOURCE)/CCZ4 + +include $(CHOMBO_HOME)/mk/Make.test + diff --git a/Tests/ApparentHorizonFinderTest3D/SimulationParameters.hpp b/Tests/ApparentHorizonFinderTest3D/SimulationParameters.hpp new file mode 100755 index 000000000..34e6a3373 --- /dev/null +++ b/Tests/ApparentHorizonFinderTest3D/SimulationParameters.hpp @@ -0,0 +1,41 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef SIMULATIONPARAMETERS_HPP_ +#define SIMULATIONPARAMETERS_HPP_ + +// General includes +#include "ChomboParameters.hpp" +#include "GRParmParse.hpp" + +#include "KerrBH.hpp" + +class SimulationParameters : public ChomboParameters +{ + public: + SimulationParameters(GRParmParse &pp) : ChomboParameters(pp) + { + readParams(pp); + } + + void readParams(GRParmParse &pp) + { + // Initial Kerr data + pp.load("kerr_mass", kerr_params.mass); + pp.load("kerr_spin", kerr_params.spin); + pp.load("kerr_center", kerr_params.center, center); + +#ifdef USE_AHFINDER + pp.load("initial_guess", initial_guess, kerr_params.mass * 0.5); +#endif + } + KerrBH::params_t kerr_params; + +#ifdef USE_AHFINDER + double initial_guess; +#endif +}; + +#endif /* SIMULATIONPARAMETERS_HPP_ */ diff --git a/Tests/ApparentHorizonFinderTest3D/UserVariables.hpp b/Tests/ApparentHorizonFinderTest3D/UserVariables.hpp new file mode 100755 index 000000000..1b59197a2 --- /dev/null +++ b/Tests/ApparentHorizonFinderTest3D/UserVariables.hpp @@ -0,0 +1,29 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef USERVARIABLES_HPP +#define USERVARIABLES_HPP + +#include "ArrayTools.hpp" +#include "CCZ4UserVariables.hpp" +#include "EmptyDiagnosticVariables.hpp" + +/// This enum gives the index of every variable stored in the grid +enum +{ + // Note that it is important that the first enum value is set to 1 more than + // the last CCZ4 var enum + NUM_VARS = NUM_CCZ4_VARS, +}; + +namespace UserVariables +{ +static const std::array variable_names = + ccz4_variable_names; +} // namespace UserVariables + +#include "UserVariables.inc.hpp" + +#endif /* USERVARIABLES_HPP */ diff --git a/Tests/InterpolatorTest/GNUmakefile b/Tests/InterpolatorTest/GNUmakefile new file mode 100644 index 000000000..d89500efb --- /dev/null +++ b/Tests/InterpolatorTest/GNUmakefile @@ -0,0 +1,20 @@ +# -*- Mode: Makefile -*- + +### This makefile produces an executable for each name in the `ebase' +### variable using the libraries named in the `LibNames' variable. + +# Included makefiles need an absolute path to the Chombo installation +# CHOMBO_HOME := Please set the CHOMBO_HOME locally (e.g. export CHOMBO_HOME=... in bash) + +GRCHOMBO_SOURCE = $(shell pwd)/../../Source + +ebase := InterpolatorTest + +LibNames := AMRTimeDependent AMRTools BoxTools + +src_dirs := $(GRCHOMBO_SOURCE)/GRChomboCore \ + $(GRCHOMBO_SOURCE)/utils \ + $(GRCHOMBO_SOURCE)/simd \ + $(GRCHOMBO_SOURCE)/AMRInterpolator + +include $(CHOMBO_HOME)/mk/Make.test diff --git a/Tests/InterpolatorTest/InterpolatorTest.cpp b/Tests/InterpolatorTest/InterpolatorTest.cpp new file mode 100644 index 000000000..34dcd7143 --- /dev/null +++ b/Tests/InterpolatorTest/InterpolatorTest.cpp @@ -0,0 +1,115 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifdef CH_LANG_CC +/* + * _______ __ + * / ___/ / ___ __ _ / / ___ + * / /__/ _ \/ _ \/ V \/ _ \/ _ \ + * \___/_//_/\___/_/_/_/_.__/\___/ + * Please refer to LICENSE, in Chombo's root directory. + */ +#endif + +// General includes: +#include +#include +#include +#include +#include + +#include "parstream.H" //Gives us pout() + +#include "Lagrange.hpp" +#include "QuinticConvolution.hpp" +#include "SetupFunctions.hpp" +#include "SimpleArrayBox.hpp" +#include "SimpleInterpSource.hpp" + +// cell centered grid +// double get_dx(double L, int num_points_u) { return L / num_points_u; } +// double get_x(int i, double m_dx) { return m_dx * (i + 0.5); } +// double get_idx(double x, double m_dx) { return x / m_dx - 0.5; } + +// node centered grid +double get_dx(double L, int num_points_u) { return L / (num_points_u - 1.); } +double get_x(int i, double m_dx) { return m_dx * i; } +double get_idx(double x, double m_dx) { return x / m_dx; } + +double func(double x, double L) +{ + // cubic + // maximum is 1 at x = L / 6 * ( 3 +- sqrt(3) ) + // return x * (x - L / 2.) * (x - L) / (L * L * L) * 12. * sqrt(3.); + + // quadratic + // maximum is 1 at x = L / 2 + // return x * (x - L) / (L * L) * 4.; + + // non-linear + return sin(2. * M_PI * x / L); +} + +int runInterpolatorTest(int argc, char *argv[]) +{ + + const int num_points_u = 200; + const double L = 10.; + const double m_dx = get_dx(L, num_points_u); + + std::vector f(num_points_u); + + for (int i = 0; i < num_points_u; ++i) + { + double x = get_x(i, m_dx); + f[i] = func(x, L); + } + + SimpleInterpSource<1> source({num_points_u}); + SimpleArrayBox<1> box({num_points_u}, f); + + double test_point = + M_PI; // just because it's irrational, hence for sure not in the grid + + Lagrange<4, 1> interpolator1(source, true); + interpolator1.setup({0}, {m_dx}, {get_idx(test_point, m_dx)}); + double val1 = interpolator1.interpData(box); + + QuinticConvolution<1> interpolator2(source, true); + interpolator2.setup({0}, {m_dx}, {get_idx(test_point, m_dx)}); + double val2 = interpolator2.interpData(box); + + double exact = func(test_point, L); + + double err1 = abs(val1 / exact - 1.) * 100.; + double err2 = abs(val2 / exact - 1.) * 100.; + + pout() << std::setprecision(9); + pout() << "Exact value is: " << exact << std::endl; + pout() << "Lagrange Interpolator gave: " << val1 << " (" << err1 + << "% error)" << std::endl; + pout() << "Quintic Convolution Interpolator gave: " << val2 << " (" << err2 + << "% error)" << std::endl; + + bool wrong = (err1 + err2 > 1e-5); + + return wrong; +} + +int main(int argc, char *argv[]) +{ + mainSetup(argc, argv); + + int status = runInterpolatorTest(argc, argv); + + if (status == 0) + pout() << "Interpolator test passed." << endl; + else + pout() << "Interpolator test failed with return code " << status + << endl; + + mainFinalize(); + return status; +} diff --git a/Tests/InterpolatorTest/InterpolatorTest.inputs b/Tests/InterpolatorTest/InterpolatorTest.inputs new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/Tests/InterpolatorTest/InterpolatorTest.inputs @@ -0,0 +1 @@ + diff --git a/Tests/InterpolatorTest/SimulationParameters.hpp b/Tests/InterpolatorTest/SimulationParameters.hpp new file mode 100644 index 000000000..5e00a7de6 --- /dev/null +++ b/Tests/InterpolatorTest/SimulationParameters.hpp @@ -0,0 +1,19 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef SIMULATIONPARAMETERS_HPP_ +#define SIMULATIONPARAMETERS_HPP_ + +// General includes +#include "ChomboParameters.hpp" +#include "GRParmParse.hpp" + +class SimulationParameters : public ChomboParameters +{ + public: + SimulationParameters(GRParmParse &pp) : ChomboParameters(pp) {} +}; + +#endif /* SIMULATIONPARAMETERS_HPP_ */ diff --git a/Tests/InterpolatorTest/UserVariables.hpp b/Tests/InterpolatorTest/UserVariables.hpp new file mode 100644 index 000000000..f7642667d --- /dev/null +++ b/Tests/InterpolatorTest/UserVariables.hpp @@ -0,0 +1,26 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef USERVARIABLES_HPP +#define USERVARIABLES_HPP + +#include "EmptyDiagnosticVariables.hpp" +#include +#include + +// assign enum to each variable +enum +{ + NUM_VARS +}; + +namespace UserVariables +{ +static const std::array variable_names = {}; +} + +#include "UserVariables.inc.hpp" + +#endif /* USERVARIABLES_HPP */ From a3a6fc04847005ddccb3203c1f76b2d6874ad625 Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Tue, 19 Jan 2021 11:29:02 +0100 Subject: [PATCH 02/92] Improve convergence by change Expansion to (r * Expansion) --- Examples/KerrBH/SimulationParameters.hpp | 2 +- .../cosmos.Make.defs.local | 2 +- .../docker_hpc-base-stretch.Make.defs.local | 6 +-- .../AHFunctionDefault.hpp | 15 ++++-- Source/ApparentHorizonFinder/AHFunctions.hpp | 24 ++++++--- .../ApparentHorizonFinder/AHInterpolation.hpp | 1 + .../AHInterpolation.impl.hpp | 14 +++++ .../ApparentHorizon.impl.hpp | 29 +++++----- .../ApparentHorizon_petsc.impl.hpp | 54 ++++++++++++++++--- .../AHTest2DFunction.hpp | 3 +- 10 files changed, 116 insertions(+), 34 deletions(-) diff --git a/Examples/KerrBH/SimulationParameters.hpp b/Examples/KerrBH/SimulationParameters.hpp index f9d7a7708..f56b671a6 100644 --- a/Examples/KerrBH/SimulationParameters.hpp +++ b/Examples/KerrBH/SimulationParameters.hpp @@ -35,7 +35,7 @@ class SimulationParameters : public SimulationParametersBase pp.load("AH_initial_guess", AH_initial_guess, 0.5 * kerr_params.mass); #ifdef USE_CHI_CONTOURS pp.load("look_for_chi_contour", look_for_chi_contour); - CH_assert(sim_params.look_for_chi_contour > 0.); + CH_assert(look_for_chi_contour > 0.); #endif #endif } diff --git a/InstallNotes/MakeDefsLocalExamples/cosmos.Make.defs.local b/InstallNotes/MakeDefsLocalExamples/cosmos.Make.defs.local index b5851c2f7..56bb6e702 100644 --- a/InstallNotes/MakeDefsLocalExamples/cosmos.Make.defs.local +++ b/InstallNotes/MakeDefsLocalExamples/cosmos.Make.defs.local @@ -20,5 +20,5 @@ cxxdbgflags = -g -Wl,--eh-frame-hdr cxxoptflags = -O3 -override-limits -xSSE4.2 -axAVX fdbgflags = -g -Wl,--eh-frame-hdr foptflags = -O3 -override-limits -xSSE4.2 -axAVX -syslibflags = -L/nfs/software/apps/papi/5.4.1/lib -lpapi +syslibflags = -L/nfs/software/apps/papi/5.4.1/lib -lpapi cxxcppflags = -DUSE_PAPI -UCH_USE_MEMORY_TRACKING diff --git a/InstallNotes/MakeDefsLocalExamples/docker_hpc-base-stretch.Make.defs.local b/InstallNotes/MakeDefsLocalExamples/docker_hpc-base-stretch.Make.defs.local index 805d0308d..82dc68418 100644 --- a/InstallNotes/MakeDefsLocalExamples/docker_hpc-base-stretch.Make.defs.local +++ b/InstallNotes/MakeDefsLocalExamples/docker_hpc-base-stretch.Make.defs.local @@ -21,6 +21,6 @@ cxxdbgflags = -g -Wl,--eh-frame-hdr -std=c++14 cxxoptflags = -Ofast -march=native -std=c++14 fdbgflags = -g -Wl,--eh-frame-hdr foptflags = -Ofast -march=native -syslibflags = -lgfortran -cxxcppflags = -ftemplate-depth=150 -DUSE_PAPI -XTRALIBFLAGS = -L/usr/lib/openblas-base/ -lopenblas +syslibflags = -lgfortran +cxxcppflags = -ftemplate-depth=150 -DUSE_PAPI +XTRALIBFLAGS = -L/usr/lib/openblas-base/ -lopenblas diff --git a/Source/ApparentHorizonFinder/AHFunctionDefault.hpp b/Source/ApparentHorizonFinder/AHFunctionDefault.hpp index e5f626eaf..53df9f56f 100644 --- a/Source/ApparentHorizonFinder/AHFunctionDefault.hpp +++ b/Source/ApparentHorizonFinder/AHFunctionDefault.hpp @@ -3,8 +3,8 @@ * Please refer to LICENSE in GRChombo's root directory. */ -#ifndef _AHFUNCTION_HPP_ -#define _AHFUNCTION_HPP_ +#ifndef _AHFUNCTIONDEFAULT_HPP_ +#define _AHFUNCTIONDEFAULT_HPP_ #include "AlwaysInline.hpp" #include "Tensor.hpp" @@ -51,6 +51,15 @@ struct AHFunctionDefault struct params // no params needed { }; + + // some constructor with these arguments: + // AHFunctionDefault(const AHData &a_data, + // const Tensor<1, double> &a_coords, + // const Tensor<1, double> &a_coords_cartesian); + + // and some 'get' + // double get(const AHGeometryData &geo_data, const AHDeriv &deriv, + // const params &a_params) const; }; -#endif /* _AHFUNCTION_HPP_ */ +#endif /* _AHFUNCTIONDEFAULT_HPP_ */ diff --git a/Source/ApparentHorizonFinder/AHFunctions.hpp b/Source/ApparentHorizonFinder/AHFunctions.hpp index 420bcd769..94a95e6c4 100644 --- a/Source/ApparentHorizonFinder/AHFunctions.hpp +++ b/Source/ApparentHorizonFinder/AHFunctions.hpp @@ -41,8 +41,11 @@ struct ExpansionFunction : AHFunctionDefault Tensor<2, double> K; double trK; -// case of reduced Cartoon methods that have an extra metric component and -// require the coordinates to calculate the expansion + // target coordinate (e.g. radius) + double f; + + // case of reduced Cartoon methods that have an extra metric component and + // require the coordinates to calculate the expansion #if GR_SPACEDIM != CH_SPACEDIM // hd - higher dimensions Tensor<1, double> x; // coordinates @@ -110,10 +113,13 @@ struct ExpansionFunction : AHFunctionDefault } ExpansionFunction(const AHData &a_data, - const Tensor<1, double> &a_coords) + const Tensor<1, double> &a_coords, + const Tensor<1, double> &a_coords_cartesian) { CH_TIME("ExpansionFunction::calculate_data"); + f = a_coords[CH_SPACEDIM - 1]; + // * --------------------------- // * GR-RELATED DATA // * --------------------------- @@ -185,8 +191,8 @@ struct ExpansionFunction : AHFunctionDefault } } -// part for higher dimensions that use Cartoon Method -// Uli's paper does it for 3+1D (from 5D) - arxiv 1808.05834 + // part for higher dimensions that use Cartoon Method + // Uli's paper does it for 3+1D (from 5D) - arxiv 1808.05834 #if GR_SPACEDIM != CH_SPACEDIM int comp_hww = c_K - 1; // c_K-1 expected to be c_hww (is c_hww direct better?) @@ -297,7 +303,10 @@ struct ExpansionFunction : AHFunctionDefault } #endif - return expansion; + // using "r * Expansion" significantly improves the convergence + // (making a Schw. BH converge for any radius >~ 0.5*r_AH instead of + // only up to ~ 3 * r_AH as it happens just with the expansion) + return expansion * f; } }; @@ -310,7 +319,8 @@ struct ChiContourFunction : AHFunctionDefault static int vars_max() { return c_chi; } ChiContourFunction(const AHData &a_data, - const Tensor<1, double> &a_coords) + const Tensor<1, double> &a_coords, + const Tensor<1, double> &a_coords_cartesian) { chi = a_data.vars.at(c_chi); } diff --git a/Source/ApparentHorizonFinder/AHInterpolation.hpp b/Source/ApparentHorizonFinder/AHInterpolation.hpp index 273902a21..9418746df 100644 --- a/Source/ApparentHorizonFinder/AHInterpolation.hpp +++ b/Source/ApparentHorizonFinder/AHInterpolation.hpp @@ -97,6 +97,7 @@ template class AHInterpolation double add_epsilon = 0.); const AHGeometryData get_geometry_data(int idx) const; const Tensor<1, double> get_cartesian_coords(int idx) const; + const Tensor<1, double> get_coords(int idx) const; const AHData get_data(int idx) const; //! verify origin +- initial guess is inside the grid diff --git a/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp b/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp index 4406769f4..9dfc99e4a 100644 --- a/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp +++ b/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp @@ -466,6 +466,20 @@ AHInterpolation::get_cartesian_coords( }; } +template +const Tensor<1, double> +AHInterpolation::get_coords(int idx) const +{ + return + { + m_u[idx], +#if CH_SPACEDIM == 3 + m_v[idx], +#endif + m_f[idx] + }; +} + template const AHData AHInterpolation::get_data(int idx) const diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp index 90b037589..c57c46b3a 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp @@ -1108,9 +1108,6 @@ void ApparentHorizon::write_coords_file( output[idx * num_components_total + 1] = m_F[idx]; #endif - const auto data = m_interp.get_data(idx); - const auto coords = m_interp.get_cartesian_coords(idx); - AHFunction func(data, coords); auto extra = m_interp.get_extra_data(idx); int el = 0; @@ -1144,15 +1141,21 @@ void ApparentHorizon::write_coords_file( // geometry vars if (write_geometry_data) + { + const auto data = m_interp.get_data(idx); + const auto coords = m_interp.get_coords(idx); + const auto coords_cart = m_interp.get_cartesian_coords(idx); + AHFunction func(data, coords, coords_cart); func.write_vars( &output[idx * num_components_total + CH_SPACEDIM + el]); + } #ifdef CH_MPI - // all processes send their 'output' to rank 0, who receives - // and writes everything (to simplify, rank 0 also sends it - // to itself, otherwise we wouldn't know which tags rank 0 - // has) sends are tagged by global index, so that receives - // can be unique and writes indexed correctly + // all processes send their 'output' to rank 0, who receives + // and writes everything (to simplify, rank 0 also sends it + // to itself, otherwise we wouldn't know which tags rank 0 + // has) sends are tagged by global index, so that receives + // can be unique and writes indexed correctly #if CH_SPACEDIM == 3 int idx_global = v * m_num_global_u + u; #elif CH_SPACEDIM == 2 @@ -1314,8 +1317,9 @@ ApparentHorizon::calculate_spin_dimensionless( { const auto geometry_data = m_interp.get_geometry_data(idx); const auto data = m_interp.get_data(idx); - const auto coords = m_interp.get_cartesian_coords(idx); - AHFunction func(data, coords); + const auto coords = m_interp.get_coords(idx); + const auto coords_cart = m_interp.get_cartesian_coords(idx); + AHFunction func(data, coords, coords_cart); auto &g = func.get_metric(); double dxdv[3]; @@ -1415,8 +1419,9 @@ double ApparentHorizon::calculate_area() { const auto geometric_data = m_interp.get_geometry_data(idx); const auto data = m_interp.get_data(idx); - const auto coords = m_interp.get_cartesian_coords(idx); - AHFunction func(data, coords); + const auto coords = m_interp.get_coords(idx); + const auto coords_cart = m_interp.get_cartesian_coords(idx); + AHFunction func(data, coords, coords_cart); auto &g = func.get_metric(); // Calculate Jacobian matrix for transformation from Cartesian diff --git a/Source/ApparentHorizonFinder/ApparentHorizon_petsc.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon_petsc.impl.hpp index 2950d8185..f27faad42 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon_petsc.impl.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon_petsc.impl.hpp @@ -141,6 +141,45 @@ void ApparentHorizon::initialise_PETSc() SNESMonitorSet(m_snes, &Petsc_SNES_monitor, this, NULL); SNESSetFromOptions(m_snes); + + if (m_params.verbose > AHFinder::MIN) + { + SNESType snes_type; + SNESGetType(m_snes, &snes_type); + double snes_atol, snes_rtol, snes_stol; + int snes_maxit, snes_maxf; + SNESGetTolerances(m_snes, &snes_atol, &snes_rtol, &snes_stol, + &snes_maxit, &snes_maxf); + + double snes_divtol; + SNESGetDivergenceTolerance(m_snes, &snes_divtol); + + KSP snes_ksp; + SNESGetKSP(m_snes, &snes_ksp); + KSPType ksp_type; + KSPGetType(snes_ksp, &ksp_type); + double ksp_rtol, ksp_abstol, ksp_dtol; + int ksp_maxits; + KSPGetTolerances(snes_ksp, &ksp_rtol, &ksp_abstol, &ksp_dtol, + &ksp_maxits); + + pout() << "-------------------------------------\n"; + pout() << "AHFinder Options:\n"; + pout() << "-------------------------------------\n"; + pout() << "PETSc SNES Options:\n"; + pout() << "Type: " << snes_type << "\n"; + pout() << "atol = " << snes_atol << ", rtol = " << snes_rtol + << ", stol = " << snes_stol << ",\n"; + pout() << "maxit = " << snes_maxit << ", maxf = " << snes_maxf << "\n"; + pout() << "divtol = " << snes_divtol << "\n"; + pout() << "-------------------------------------\n"; + pout() << "PETSc KSP Options:\n"; + pout() << "Type: " << ksp_type << "\n"; + pout() << "rtol = " << ksp_rtol << ", abstol = " << ksp_abstol + << ", dtol = " << ksp_dtol << ", maxits = " << ksp_maxits + << "\n"; + pout() << "-------------------------------------" << std::endl; + } } template @@ -654,8 +693,9 @@ void ApparentHorizon::form_function(Vec F, Vec Rhs) { const auto geometry_data = m_interp.get_geometry_data(idx); const auto data = m_interp.get_data(idx); - const auto coords = m_interp.get_cartesian_coords(idx); - AHFunction func(data, coords); + const auto coords = m_interp.get_coords(idx); + const auto coords_cart = m_interp.get_cartesian_coords(idx); + AHFunction func(data, coords, coords_cart); _out = func.get(geometry_data, deriv, m_func_params); } @@ -878,8 +918,9 @@ double ApparentHorizon::point_jacobian( const auto geometry_data = interp_plus.get_geometry_data(idx); const auto data = interp_plus.get_data(idx); - const auto coords = interp_plus.get_cartesian_coords(idx); - AHFunction func(data, coords); + const auto coords = interp_plus.get_coords(idx); + const auto coords_cart = interp_plus.get_cartesian_coords(idx); + AHFunction func(data, coords, coords_cart); expansionPlus = func.get(geometry_data, deriv, m_func_params); _in = in_old; @@ -900,8 +941,9 @@ double ApparentHorizon::point_jacobian( const auto geometry_data = interp_minus.get_geometry_data(idx); const auto data = interp_minus.get_data(idx); - const auto coords = interp_minus.get_cartesian_coords(idx); - AHFunction func(data, coords); + const auto coords = interp_minus.get_coords(idx); + const auto coords_cart = interp_minus.get_cartesian_coords(idx); + AHFunction func(data, coords, coords_cart); expansionMinus = func.get(geometry_data, deriv, m_func_params); _in = in_old; diff --git a/Tests/ApparentHorizonFinderTest2D/AHTest2DFunction.hpp b/Tests/ApparentHorizonFinderTest2D/AHTest2DFunction.hpp index 385ba9781..e98ae3721 100644 --- a/Tests/ApparentHorizonFinderTest2D/AHTest2DFunction.hpp +++ b/Tests/ApparentHorizonFinderTest2D/AHTest2DFunction.hpp @@ -23,7 +23,8 @@ struct AHTest2DFunction : AHFunctionDefault static int vars_max() { return c_V; } AHTest2DFunction(const AHData &a_data, - const Tensor<1, double> &a_coords) + const Tensor<1, double> &a_coords, + const Tensor<1, double> &a_cart_coords) { v = a_data.vars.at(c_V); } From e55e1e27a0510166801f20d89db3da794049b7b2 Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Tue, 19 Jan 2021 18:09:21 +0100 Subject: [PATCH 03/92] Fix Make.defs.local for github action Allow radius power in expansion output to be user-changeable --- .../intel-no-hdf5-minimal.Make.defs.local | 2 +- Source/ApparentHorizonFinder/AHFunctions.hpp | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/InstallNotes/MakeDefsLocalExamples/intel-no-hdf5-minimal.Make.defs.local b/InstallNotes/MakeDefsLocalExamples/intel-no-hdf5-minimal.Make.defs.local index 2203aa8b0..e77a8f446 100644 --- a/InstallNotes/MakeDefsLocalExamples/intel-no-hdf5-minimal.Make.defs.local +++ b/InstallNotes/MakeDefsLocalExamples/intel-no-hdf5-minimal.Make.defs.local @@ -2,7 +2,7 @@ # This is used for the github action that builds and runs the tests # with the Intel C++ compiler and MPI implementation -DIM = 3 +DIM ?= 3 DEBUG = TRUE PRECISION = DOUBLE OPT = TRUE diff --git a/Source/ApparentHorizonFinder/AHFunctions.hpp b/Source/ApparentHorizonFinder/AHFunctions.hpp index 94a95e6c4..2785496ba 100644 --- a/Source/ApparentHorizonFinder/AHFunctions.hpp +++ b/Source/ApparentHorizonFinder/AHFunctions.hpp @@ -212,6 +212,10 @@ struct ExpansionFunction : AHFunctionDefault #endif } + struct params // no params needed + { + double expansion_radius_power = 1.; + }; double get(const AHGeometryData &geo_data, const AHDeriv &deriv, const params &a_params) const { @@ -306,7 +310,7 @@ struct ExpansionFunction : AHFunctionDefault // using "r * Expansion" significantly improves the convergence // (making a Schw. BH converge for any radius >~ 0.5*r_AH instead of // only up to ~ 3 * r_AH as it happens just with the expansion) - return expansion * f; + return expansion * pow(f, a_params.expansion_radius_power); } }; From da40de2952a488b1d7a6c9f24e4f15ee30f80e22 Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Wed, 27 Jan 2021 16:31:10 +0000 Subject: [PATCH 04/92] Minor bugs --- Examples/KerrBH/params.txt | 50 +++++++++++-------- .../ApparentHorizon.impl.hpp | 7 ++- .../ApparentHorizon_petsc.impl.hpp | 10 ++-- 3 files changed, 40 insertions(+), 27 deletions(-) diff --git a/Examples/KerrBH/params.txt b/Examples/KerrBH/params.txt index 6b3108fa9..01a38f13b 100644 --- a/Examples/KerrBH/params.txt +++ b/Examples/KerrBH/params.txt @@ -3,7 +3,7 @@ verbosity = 0 chk_prefix = KerrBH_ plot_prefix = KerrBHp_ -#restart_file = KerrBH_000060.3d.hdf5 +#restart_file = KerrBH_000400.3d.hdf5 # 'N' is the number of subdivisions in each direction of a cubic box # 'L' is the length of the longest side of the box, dx_coarsest = L/N @@ -12,7 +12,7 @@ plot_prefix = KerrBHp_ # NB - if you have a non-cubic grid, you can specify 'N1' or 'N1_full', # 'N2' or 'N2_full' and 'N3' or 'N3_full' ( then dx_coarsest = L/N(max) ) # NB - the N values need to be multiples of the block_factor -N_full = 64 +N_full = 128 L_full = 128 # Params for Kerr BH @@ -32,8 +32,8 @@ max_level = 6 # There are (max_level+1) grids, so min is zero # Need one for each level except the top one, ie max_level items # Generally you do not need to regrid frequently on every level # Level Regridding: 0 1 2 3 4 5 -regrid_interval = 50 25 5 5 5 5 -regrid_thresholds = 0.3 0.3 0.3 0.3 0.3 0.3 +regrid_interval = 0 0 0 0 0 0 +regrid_threshold = 0.0065 # Max box sizes max_grid_size = 16 @@ -72,12 +72,12 @@ nonzero_asymptotic_values = 1.0 1.0 1.0 1.0 1.0 # Set up time steps # dt will be dx*dt_multiplier on each grid level # HDF5files are written every dt = L/N*dt_multiplier*checkpoint_interval -checkpoint_interval = 50 -plot_interval = 10 -num_plot_vars = 0 +checkpoint_interval = 0 +plot_interval = 20 +num_plot_vars = 2 +plot_vars = chi Ham dt_multiplier = 0.25 -stop_time = 20.0 -max_steps = 10000000 +stop_time = 100.0 #Lapse evolution lapse_power = 1.0 @@ -91,37 +91,45 @@ eta = 1.0 # This is gamma driver, usually of order 1/M_ADM of spacetime # CCZ4 parameters # if using BSSN the kappa values should be zero -formulation = 1 # 1 for BSSN, 0 for CCZ4 -kappa1 = 0 +formulation = 0 # 1 for BSSN, 0 for CCZ4 +kappa1 = 0.1 kappa2 = 0 -kappa3 = 0 -covariantZ4 = 0 # 0: default. 1: dampk1 -> dampk1/lapse +kappa3 = 1 +covariantZ4 = 1 # 0: default. 1: dampk1 -> dampk1/lapse # coefficient for KO numerical dissipation sigma = 0.3 #extraction params -activate_extraction = 0 +activate_extraction = 1 num_extraction_radii = 3 -extraction_radii = 30. 40. 50. -extraction_levels = 2 1 0 +extraction_radii = 20. 25. 30. +extraction_levels = 1 1 0 num_points_phi = 50 num_points_theta = 52 #Apparent Horizon finder AH_activate = 1 AH_num_ranks = 20 -AH_num_points_u = 30 -AH_num_points_v = 50 +AH_num_points_u = 49 +AH_num_points_v = 48 AH_solve_interval = 1 -AH_print_interval = 2 +AH_print_interval = 1 AH_track_center = false AH_predict_origin = false #AH_level_to_run = 0 #AH_allow_re_attempt = 0 -#AH_start_time = 0.0 +AH_stop_if_max_fails = 1 +#AH_start_time = 0. #AH_give_up_time = -1. # -1 to never #AH_max_fails_after_lost = 0 # -1 to never -#AH_verbose = 1 +#AH_print_geometry_data = 0 +#AH_re_solve_at_restart = 0 +AH_verbose = 3 #AH_initial_guess = 0.5 + +#look_for_chi_contour = 0.2 + +AH_num_write_vars = 1 +AH_write_vars = chi diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp index c57c46b3a..51f09ebfa 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp @@ -417,6 +417,11 @@ void ApparentHorizon::solve(double a_dt, double spin_dimensionless = calculate_spin_dimensionless(m_area); m_mass = calculate_mass(m_area, spin_dimensionless); m_spin = spin_dimensionless * m_mass; + + if (m_params.verbose > AHFinder::MIN) + { + pout() << "mass = " << m_mass << endl; + } #endif // reset min and max F, to force re-calculation @@ -1305,7 +1310,7 @@ ApparentHorizon::calculate_spin_dimensionless( // m_F is already set from solve - int u_equator = floor(M_PI / 2.0 / m_du); + int u_equator = std::round(M_PI / 2.0 / m_du); for (int v = m_vmin; v < m_vmax; ++v) { diff --git a/Source/ApparentHorizonFinder/ApparentHorizon_petsc.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon_petsc.impl.hpp index f27faad42..74fb8faab 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon_petsc.impl.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon_petsc.impl.hpp @@ -146,20 +146,20 @@ void ApparentHorizon::initialise_PETSc() { SNESType snes_type; SNESGetType(m_snes, &snes_type); - double snes_atol, snes_rtol, snes_stol; - int snes_maxit, snes_maxf; + PetscReal snes_atol, snes_rtol, snes_stol; + PetscInt snes_maxit, snes_maxf; SNESGetTolerances(m_snes, &snes_atol, &snes_rtol, &snes_stol, &snes_maxit, &snes_maxf); - double snes_divtol; + PetscReal snes_divtol; SNESGetDivergenceTolerance(m_snes, &snes_divtol); KSP snes_ksp; SNESGetKSP(m_snes, &snes_ksp); KSPType ksp_type; KSPGetType(snes_ksp, &ksp_type); - double ksp_rtol, ksp_abstol, ksp_dtol; - int ksp_maxits; + PetscReal ksp_rtol, ksp_abstol, ksp_dtol; + PetscInt ksp_maxits; KSPGetTolerances(snes_ksp, &ksp_rtol, &ksp_abstol, &ksp_dtol, &ksp_maxits); From 2539b08f4a13d8204e5bb7d511b909a6517a9b09 Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Thu, 18 Feb 2021 19:51:26 +0000 Subject: [PATCH 05/92] Fill only necessary ghosts in AHFinder Change default prints --- Source/ApparentHorizonFinder/AHData.hpp | 2 +- Source/ApparentHorizonFinder/AHFinder.cpp | 4 +- Source/ApparentHorizonFinder/AHFinder.hpp | 3 +- .../ApparentHorizonFinder/AHInterpolation.hpp | 8 ++- .../AHInterpolation.impl.hpp | 63 ++++++++++++++++++- .../ApparentHorizon.impl.hpp | 10 +-- 6 files changed, 77 insertions(+), 13 deletions(-) diff --git a/Source/ApparentHorizonFinder/AHData.hpp b/Source/ApparentHorizonFinder/AHData.hpp index c273ad82b..8585bfa2e 100644 --- a/Source/ApparentHorizonFinder/AHData.hpp +++ b/Source/ApparentHorizonFinder/AHData.hpp @@ -61,7 +61,7 @@ template struct AHData template AHData -get_AHData_idx(int idx, const AHData> a_data) +get_AHData_idx(int idx, const AHData> &a_data) { AHData data; diff --git a/Source/ApparentHorizonFinder/AHFinder.cpp b/Source/ApparentHorizonFinder/AHFinder.cpp index a2a9214e9..221ece9b8 100644 --- a/Source/ApparentHorizonFinder/AHFinder.cpp +++ b/Source/ApparentHorizonFinder/AHFinder.cpp @@ -358,7 +358,7 @@ void AHFinder::params::read_params(GRParmParse &pp, const ChomboParameters &a_p) // load vars to write to coord files num_extra_vars = 0; - extra_contain_diagnostic = false; + extra_contain_diagnostic = 0; int AH_num_write_vars; pp.load("AH_num_write_vars", AH_num_write_vars, 0); @@ -401,7 +401,7 @@ void AHFinder::params::read_params(GRParmParse &pp, const ChomboParameters &a_p) else { var_type = VariableType::diagnostic; - extra_contain_diagnostic = true; + ++extra_contain_diagnostic; } } if (var >= 0) diff --git a/Source/ApparentHorizonFinder/AHFinder.hpp b/Source/ApparentHorizonFinder/AHFinder.hpp index 54011f721..d5c7fc263 100644 --- a/Source/ApparentHorizonFinder/AHFinder.hpp +++ b/Source/ApparentHorizonFinder/AHFinder.hpp @@ -122,7 +122,8 @@ class AHFinder int num_extra_vars; // total number of extra vars (!=extra_vars.size() // as derivative count for multiple vars) - bool extra_contain_diagnostic; // not a parameter (set internally) + int extra_contain_diagnostic; // not a parameter (set internally); + // counts how many void read_params(GRParmParse &pp, const ChomboParameters &a_p); }; diff --git a/Source/ApparentHorizonFinder/AHInterpolation.hpp b/Source/ApparentHorizonFinder/AHInterpolation.hpp index 9418746df..a6ae0576c 100644 --- a/Source/ApparentHorizonFinder/AHInterpolation.hpp +++ b/Source/ApparentHorizonFinder/AHInterpolation.hpp @@ -79,7 +79,10 @@ template class AHInterpolation const std::array & get_origin() const; //!< get origin of CoordSystem - void refresh_interpolator(); //!< refresh AMRInterpolator 'm_interpolator' + void refresh_interpolator( + bool printing_step, + const std::map> + &extra_vars); //!< refresh AMRInterpolator 'm_interpolator' double get_grid_coord(int a_dir, double f, double u #if CH_SPACEDIM == 3 @@ -130,7 +133,8 @@ template class AHInterpolation int interpolate(); void interpolate_extra_vars( - std::map> extra_vars); + const std::map> + &extra_vars); const AHData get_extra_data(int idx) const; }; diff --git a/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp b/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp index 9dfc99e4a..6f3d094ed 100644 --- a/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp +++ b/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp @@ -366,10 +366,65 @@ int AHInterpolation::interpolate() } template -void AHInterpolation::refresh_interpolator() +void AHInterpolation::refresh_interpolator( + bool printing_step, + const std::map> &extra_vars) { CH_TIME("AHInterpolation::refresh_interpolator"); - m_interpolator->refresh(); + + // brute force filling of all ghosts: + // m_interpolator->refresh(); + + // OR try to only interpolate the variables needed: + + int min_evolution_var = + std::min(AHFunction::vars_min(), std::min(AHFunction::d1_vars_min(), + AHFunction::d2_vars_min())); + int max_evolution_var = + std::max(AHFunction::vars_max(), std::max(AHFunction::d1_vars_max(), + AHFunction::d2_vars_max())); + + int min_diagnostic_var = -1; + int max_diagnostic_var = -1; + + for (auto &var : extra_vars) + { + int var_enum = std::get<0>(var.second); + VariableType var_type = std::get<1>(var.second); + + if (var_type == VariableType::evolution) + { + if (var_enum < min_evolution_var) + min_evolution_var = var_enum; + else if (var_enum > max_evolution_var) + max_evolution_var = var_enum; + } + else if (printing_step) // diagnostic + { + if (min_diagnostic_var == -1) + { + min_diagnostic_var = var_enum; + max_diagnostic_var = var_enum; + } + else if (var_enum < min_diagnostic_var) + min_diagnostic_var = var_enum; + else if (var_enum > max_diagnostic_var) + max_diagnostic_var = var_enum; + } + } + + // fill ghosts manually to minimise communication + bool fill_ghosts = false; + m_interpolator->refresh(fill_ghosts); + m_interpolator->fill_multilevel_ghosts( + VariableType::evolution, + Interval(min_evolution_var, max_evolution_var)); + if (printing_step && min_diagnostic_var != -1) + { + m_interpolator->fill_multilevel_ghosts( + VariableType::evolution, + Interval(min_diagnostic_var, max_diagnostic_var)); + } } // 'set_coordinates' calls 'interpolate'. All Chombo_MPI:comm need to run @@ -489,7 +544,7 @@ AHInterpolation::get_data(int idx) const template void AHInterpolation::interpolate_extra_vars( - std::map> extra_vars) + const std::map> &extra_vars) { CH_TIME("AHInterpolation::interpolate_extra_vars"); @@ -518,6 +573,8 @@ void AHInterpolation::interpolate_extra_vars( m_extra.set_d2(query, var_enum, var.first, var_type, n); } + // ghosts for extra evolution or diagnostic vars already filled in + // 'refresh_interpolator' m_interpolator->interp(query); } diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp index 51f09ebfa..214fea388 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp @@ -345,7 +345,9 @@ void ApparentHorizon::solve(double a_dt, if (!good_to_go(a_dt, a_time)) return; - m_interp.refresh_interpolator(); //(ALL CHOMBO ranks do it!!) + m_interp.refresh_interpolator( + do_print(a_dt, a_time), + m_params.extra_vars); // (ALL CHOMBO ranks do it!!) // estimate the next position where origin will be if (get_converged()) @@ -418,7 +420,7 @@ void ApparentHorizon::solve(double a_dt, m_mass = calculate_mass(m_area, spin_dimensionless); m_spin = spin_dimensionless * m_mass; - if (m_params.verbose > AHFinder::MIN) + if (m_params.verbose > AHFinder::NONE) { pout() << "mass = " << m_mass << endl; } @@ -1376,7 +1378,7 @@ ApparentHorizon::calculate_spin_dimensionless( : sqrt(1. - factor * factor)); // factor>1 means numerical // error with spin as 0 - if (m_params.verbose > AHFinder::MIN) + if (m_params.verbose > AHFinder::NONE) { pout() << "dimensionless spin = " << spin_dimensionless << endl; } @@ -1525,7 +1527,7 @@ double ApparentHorizon::calculate_area() area = integral; #endif - if (m_params.verbose > AHFinder::NONE) + if (m_params.verbose > AHFinder::MIN) { pout() << "area = " << area << endl; } From 4f5831adbaf380ab16d101270d0e11fedf5887c5 Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Thu, 18 Feb 2021 19:51:49 +0000 Subject: [PATCH 06/92] Bug fixes and generalization of extraction to 2D (preparation for head-on 2D code) --- Examples/BinaryBH/params_very_cheap.txt | 2 +- .../AMRInterpolator/SphericalExtraction.hpp | 23 +- Source/AMRInterpolator/SphericalGeometry.hpp | 35 ++-- Source/AMRInterpolator/SurfaceExtraction.hpp | 9 + .../SurfaceExtraction.impl.hpp | 36 +++- Source/ApparentHorizonFinder/AHFinder.hpp | 5 - Source/ApparentHorizonFinder/AHFunctions.hpp | 4 +- .../AHSphericalGeometry.hpp | 145 ++++++++----- .../ApparentHorizon.impl.hpp | 44 ++-- .../GRChomboCore/SimulationParametersBase.hpp | 197 +++++++++--------- .../FixedGridsTaggingCriterion.hpp | 2 +- Source/utils/DimensionDefinitions.hpp | 2 +- Tests/ApparentHorizonFinderTest2D/GNUmakefile | 2 +- .../UserVariables.hpp | 5 +- 14 files changed, 310 insertions(+), 201 deletions(-) diff --git a/Examples/BinaryBH/params_very_cheap.txt b/Examples/BinaryBH/params_very_cheap.txt index 6cff37641..3d7de9334 100644 --- a/Examples/BinaryBH/params_very_cheap.txt +++ b/Examples/BinaryBH/params_very_cheap.txt @@ -106,7 +106,7 @@ modes = 2 0 # l m for spherical harmonics #Apparent Horizon finder AH_activate = 1 -AH_num_ranks = 2 +AH_num_ranks = 4 AH_num_points_u = 11 AH_num_points_v = 10 AH_solve_interval = 2 diff --git a/Source/AMRInterpolator/SphericalExtraction.hpp b/Source/AMRInterpolator/SphericalExtraction.hpp index 52ab7e040..348317cc6 100644 --- a/Source/AMRInterpolator/SphericalExtraction.hpp +++ b/Source/AMRInterpolator/SphericalExtraction.hpp @@ -85,14 +85,21 @@ class SphericalExtraction : public SurfaceExtraction const IntegrationMethod &a_method_phi = IntegrationMethod::trapezium, const bool a_broadcast_integral = false) { - auto integrand_re = [center = m_center, &geom = m_geom, es, el, em, + // {x, y, z} + SphericalGeometry::UP_DIR up_dir = m_geom.get_up_dir(); + std::array dirs = {(up_dir + 1) % 3, (up_dir + 2) % 3, up_dir}; + std::array center; + for (int i = 0; i < 3; ++i) + center[i] = (dirs[i] < CH_SPACEDIM ? m_center[dirs[i]] : 0.); + + auto integrand_re = [dirs, center, &geom = m_geom, es, el, em, &a_function](std::vector &a_data_here, double r, double theta, double phi) { // note that spin_Y_lm requires the coordinates with the center // at the origin - double x = geom.get_grid_coord(0, r, theta, phi) - center[0]; - double y = geom.get_grid_coord(1, r, theta, phi) - center[1]; - double z = geom.get_grid_coord(2, r, theta, phi) - center[2]; + double x = geom.get_grid_coord(dirs[0], r, theta, phi) - center[0]; + double y = geom.get_grid_coord(dirs[1], r, theta, phi) - center[1]; + double z = geom.get_grid_coord(dirs[2], r, theta, phi) - center[2]; SphericalHarmonics::Y_lm_t Y_lm = SphericalHarmonics::spin_Y_lm(x, y, z, es, el, em); auto function_here = a_function(a_data_here, r, theta, phi); @@ -103,14 +110,14 @@ class SphericalExtraction : public SurfaceExtraction add_integrand(integrand_re, out_integrals.first, a_method_theta, a_method_phi, a_broadcast_integral); - auto integrand_im = [center = m_center, &geom = m_geom, es, el, em, + auto integrand_im = [dirs, center, &geom = m_geom, es, el, em, &a_function](std::vector &a_data_here, double r, double theta, double phi) { // note that spin_Y_lm requires the coordinates with the center // at the origin - double x = geom.get_grid_coord(0, r, theta, phi) - center[0]; - double y = geom.get_grid_coord(1, r, theta, phi) - center[1]; - double z = geom.get_grid_coord(2, r, theta, phi) - center[2]; + double x = geom.get_grid_coord(dirs[0], r, theta, phi) - center[0]; + double y = geom.get_grid_coord(dirs[0], r, theta, phi) - center[1]; + double z = geom.get_grid_coord(dirs[0], r, theta, phi) - center[2]; SphericalHarmonics::Y_lm_t Y_lm = SphericalHarmonics::spin_Y_lm(x, y, z, es, el, em); auto function_here = a_function(a_data_here, r, theta, phi); diff --git a/Source/AMRInterpolator/SphericalGeometry.hpp b/Source/AMRInterpolator/SphericalGeometry.hpp index 8293dcd85..aa35d4ab6 100644 --- a/Source/AMRInterpolator/SphericalGeometry.hpp +++ b/Source/AMRInterpolator/SphericalGeometry.hpp @@ -6,10 +6,6 @@ #ifndef SPHERICALGEOMETRY_HPP_ #define SPHERICALGEOMETRY_HPP_ -#if CH_SPACEDIM != 3 -#error "This file should only be included for 3+1D simulations" -#endif - // Chombo includes #include "AlwaysInline.hpp" #include "MayDay.H" @@ -29,15 +25,25 @@ //! u = theta, v = phi class SphericalGeometry { - protected: - std::array m_center; - public: + enum UP_DIR + { + X, + Y, + Z + }; + SphericalGeometry(const std::array &a_center) : m_center(a_center) { +#if CH_SPACEDIM == 3 // for now force 'Z', could be an argument later on + m_up_dir = Z; +#elif CH_SPACEDIM == 2 // in Cartoon method, assume 'X' as 'z' axis + m_up_dir = X; +#endif } + ALWAYS_INLINE UP_DIR get_up_dir() const { return m_up_dir; } ALWAYS_INLINE double get_domain_u_min() const { return 0.; } ALWAYS_INLINE double get_domain_u_max() const { return M_PI; } ALWAYS_INLINE double get_domain_v_min() const { return 0.; } @@ -89,20 +95,25 @@ class SphericalGeometry //! returns the Cartesian coordinate in direction a_dir with specified //! radius, theta and phi. ALWAYS_INLINE double get_grid_coord(int a_dir, double a_radius, - double a_theta, double a_phi) const + double a_theta, double a_phi = 0.) const { - switch (a_dir) + double center = (a_dir < CH_SPACEDIM ? m_center[a_dir] : 0.); + switch ((a_dir + 2 - m_up_dir) % 3) { case (0): - return m_center[0] + a_radius * sin(a_theta) * cos(a_phi); + return center + a_radius * sin(a_theta) * cos(a_phi); case (1): - return m_center[1] + a_radius * sin(a_theta) * sin(a_phi); + return center + a_radius * sin(a_theta) * sin(a_phi); case (2): - return m_center[2] + a_radius * cos(a_theta); + return center + a_radius * cos(a_theta); default: MayDay::Error("SphericalGeometry: Direction not supported"); } } + + protected: + std::array m_center; + UP_DIR m_up_dir; }; #endif /* SPHERICALGEOMETRY_HPP_ */ diff --git a/Source/AMRInterpolator/SurfaceExtraction.hpp b/Source/AMRInterpolator/SurfaceExtraction.hpp index c46dbf636..eb444f130 100644 --- a/Source/AMRInterpolator/SurfaceExtraction.hpp +++ b/Source/AMRInterpolator/SurfaceExtraction.hpp @@ -50,6 +50,8 @@ template class SurfaceExtraction //!< extraction for each surface bool write_extraction; //!< whether or not to write the extracted data + std::string extraction_prefix; + int min_extraction_level() { return *(std::min_element(extraction_levels.begin(), @@ -92,11 +94,18 @@ template class SurfaceExtraction //! returns the flattened index for m_interp_data and m_interp_coords //! associated to given surface, u and v indices +#if CH_SPACEDIM == 3 int index(int a_isurface, int a_iu, int a_iv) const { return a_isurface * m_params.num_points_u * m_params.num_points_v + a_iu * m_params.num_points_v + a_iv; } +#elif CH_SPACEDIM == 2 + int index(int a_isurface, int a_iu) const + { + return a_isurface * m_params.num_points_u + a_iu; + } +#endif public: //! Normal constructor which requires vars to be added after construction diff --git a/Source/AMRInterpolator/SurfaceExtraction.impl.hpp b/Source/AMRInterpolator/SurfaceExtraction.impl.hpp index 16eb7e951..ccfe05ffb 100644 --- a/Source/AMRInterpolator/SurfaceExtraction.impl.hpp +++ b/Source/AMRInterpolator/SurfaceExtraction.impl.hpp @@ -19,8 +19,10 @@ SurfaceExtraction::SurfaceExtraction( : m_geom(a_geom), m_params(a_params), m_dt(a_dt), m_time(a_time), m_first_step(a_first_step), m_restart_time(a_restart_time), m_num_interp_points((procID() == 0) - ? m_params.num_surfaces * m_params.num_points_u * - m_params.num_points_v + ? m_params.num_surfaces * m_params.num_points_u +#if CH_SPACEDIM == 3 + * m_params.num_points_v +#endif : 0), m_du(m_geom.du(m_params.num_points_u)), m_dv(m_geom.dv(m_params.num_points_v)), m_done_extraction(false) @@ -37,6 +39,7 @@ SurfaceExtraction::SurfaceExtraction( for (int iu = 0; iu < m_params.num_points_u; ++iu) { double u = m_geom.u(iu, m_params.num_points_u); +#if CH_SPACEDIM == 3 for (int iv = 0; iv < m_params.num_points_v; ++iv) { double v = m_geom.v(iv, m_params.num_points_v); @@ -47,6 +50,14 @@ SurfaceExtraction::SurfaceExtraction( idir, surface_param_value, u, v); } } +#elif CH_SPACEDIM == 2 + FOR1(idir) + { + int idx = index(isurface, iu); + m_interp_coords[idir][idx] = + m_geom.get_grid_coord(idir, surface_param_value, u); + } +#endif } } } @@ -254,7 +265,11 @@ void SurfaceExtraction::integrate() for (int ivar = 0; ivar < m_vars.size(); ++ivar) { data_here[ivar] = +#if CH_SPACEDIM == 3 m_interp_data[ivar][index(isurface, iu, iv)]; +#elif CH_SPACEDIM == 2 + m_interp_data[ivar][index(isurface, iu)]; +#endif } for (int iintegral = 0; iintegral < num_integrals; ++iintegral) @@ -330,7 +345,8 @@ void SurfaceExtraction::write_extraction( CH_assert(m_done_extraction); if (procID() == 0) { - SmallDataIO extraction_file(a_file_prefix, m_dt, m_time, m_restart_time, + SmallDataIO extraction_file(m_params.extraction_prefix + a_file_prefix, + m_dt, m_time, m_restart_time, SmallDataIO::NEW, m_first_step); for (int isurface = 0; isurface < m_params.num_surfaces; ++isurface) @@ -374,17 +390,26 @@ void SurfaceExtraction::write_extraction( for (int iu = 0; iu < m_params.num_points_u; ++iu) { double u = m_geom.u(iu, m_params.num_points_u); +#if CH_SPACEDIM == 3 for (int iv = 0; iv < m_params.num_points_v; ++iv) { double v = m_geom.v(iv, m_params.num_points_v); int idx = index(isurface, iu, iv); +#elif CH_SPACEDIM == 2 + { + int idx = index(isurface, iu); +#endif std::vector data(m_vars.size()); for (int ivar = 0; ivar < m_vars.size(); ++ivar) { data[ivar] = m_interp_data[ivar][idx]; } +#if CH_SPACEDIM == 3 extraction_file.write_data_line(data, {u, v}); +#elif CH_SPACEDIM == 2 + extraction_file.write_data_line(data, {u}); +#endif } } extraction_file.line_break(); @@ -415,8 +440,9 @@ void SurfaceExtraction::write_integrals( CH_assert(vect.size() == m_params.num_surfaces); } // open file for writing - SmallDataIO integral_file(a_filename, m_dt, m_time, m_restart_time, - SmallDataIO::APPEND, m_first_step); + SmallDataIO integral_file(m_params.extraction_prefix + a_filename, m_dt, + m_time, m_restart_time, SmallDataIO::APPEND, + m_first_step); // remove any duplicate data if this is a restart integral_file.remove_duplicate_time_data(); diff --git a/Source/ApparentHorizonFinder/AHFinder.hpp b/Source/ApparentHorizonFinder/AHFinder.hpp index d5c7fc263..6c202a41a 100644 --- a/Source/ApparentHorizonFinder/AHFinder.hpp +++ b/Source/ApparentHorizonFinder/AHFinder.hpp @@ -16,13 +16,8 @@ // define SurfaceGeometry of AHFinder #ifndef AHSurfaceGeometry -#if CH_SPACEDIM == 3 #include "AHSphericalGeometry.hpp" #define AHSurfaceGeometry AHSphericalGeometry -#elif CH_SPACEDIM == 2 -#include "AHStringGeometry.hpp" -#define AHSurfaceGeometry AHStringGeometry -#endif #endif // default to expansion diff --git a/Source/ApparentHorizonFinder/AHFunctions.hpp b/Source/ApparentHorizonFinder/AHFunctions.hpp index 2785496ba..dd04ef99c 100644 --- a/Source/ApparentHorizonFinder/AHFunctions.hpp +++ b/Source/ApparentHorizonFinder/AHFunctions.hpp @@ -48,7 +48,7 @@ struct ExpansionFunction : AHFunctionDefault // require the coordinates to calculate the expansion #if GR_SPACEDIM != CH_SPACEDIM // hd - higher dimensions - Tensor<1, double> x; // coordinates + Tensor<1, double> x; // cartesian coordinates double g_hd; Tensor<1, double> dg_hd; @@ -199,7 +199,7 @@ struct ExpansionFunction : AHFunctionDefault int comp_Aww = c_Aww; // is (c_Theta-1) more general? Tensor<1, double, CH_SPACEDIM> dhww; - x = a_coords; + x = a_coords_cartesian; double hww = a_data.vars.at(comp_hww); FOR1(a) { dhww[a] = a_data.d1.at(comp_hww)[a]; } diff --git a/Source/ApparentHorizonFinder/AHSphericalGeometry.hpp b/Source/ApparentHorizonFinder/AHSphericalGeometry.hpp index 42de6ca54..8f905d5bc 100644 --- a/Source/ApparentHorizonFinder/AHSphericalGeometry.hpp +++ b/Source/ApparentHorizonFinder/AHSphericalGeometry.hpp @@ -6,10 +6,6 @@ #ifndef AHSPHERICALGEOMETRY_HPP_ #define AHSPHERICALGEOMETRY_HPP_ -#if CH_SPACEDIM != 3 -#error "This file should only be included for 3+1D simulations" -#endif - // Chombo includes #include "CH_Timer.H" @@ -44,7 +40,8 @@ class AHSphericalGeometry : public SphericalGeometry return m_center; } - AHGeometryData get_geometry_data(double r, double theta, double phi) const + AHGeometryData get_geometry_data(double r, double theta, + double phi = 0.) const { CH_TIME("AHSphericalGeometry::get_geometry_data"); @@ -60,76 +57,110 @@ class AHSphericalGeometry : public SphericalGeometry double cos2phi = cos(2. * phi); double sin2phi = sin(2. * phi); + // in 3D, (x,y,z) are normal + // in 2D Cartoon method, the spherical 'z' axis is the grid 'x' axis, so + // the 2D coordinates (x,y) are (z,x) spherical coordinates + // (that means commenting out everything related to 'y' and to 'v' + // [=phi]) + int iz = m_up_dir; + int ix = (m_up_dir + 1) % 3; +#if CH_SPACEDIM == 3 + int iy = (m_up_dir + 2) % 3; +#endif + AHGeometryData out; - out.du[0] = (costheta * cosphi) / r; - out.du[1] = (costheta * sinphi) / r; - out.du[2] = -sintheta / r; + out.du[ix] = (costheta * cosphi) / r; +#if CH_SPACEDIM == 3 + out.du[iy] = (costheta * sinphi) / r; +#endif + out.du[iz] = -sintheta / r; - out.dv[0] = (-sinphi) / (r * sintheta); - out.dv[1] = (cosphi) / (r * sintheta); - out.dv[2] = 0; +#if CH_SPACEDIM == 3 + out.dv[ix] = (-sinphi) / (r * sintheta); + out.dv[iy] = (cosphi) / (r * sintheta); + out.dv[iz] = 0; +#endif double dfdx = sintheta * cosphi; +#if CH_SPACEDIM == 3 double dfdy = sintheta * sinphi; +#endif double dfdz = costheta; - out.ddu[0][0] = + out.ddu[ix][ix] = (cos2theta * cosphi * cosphi - cos2phi) / (r * r * tantheta); - out.ddu[0][1] = ((cos2theta - 2.) * sin2phi) / (2. * r * r * tantheta); - out.ddu[0][2] = -(cos2theta * cosphi) / (r * r); - out.ddu[1][1] = + out.ddu[ix][iz] = -(cos2theta * cosphi) / (r * r); + out.ddu[iz][ix] = out.ddu[ix][iz]; + out.ddu[iz][iz] = (sin2theta) / (r * r); +#if CH_SPACEDIM == 3 + out.ddu[ix][iy] = + ((cos2theta - 2.) * sin2phi) / (2. * r * r * tantheta); + out.ddu[iy][iy] = (cos2theta * sinphi * sinphi + cos2phi) / (r * r * tantheta); - out.ddu[1][2] = -(cos2theta * sinphi) / (r * r); - out.ddu[2][2] = (sin2theta) / (r * r); - out.ddu[1][0] = out.ddu[0][1]; - out.ddu[2][0] = out.ddu[0][2]; - out.ddu[2][1] = out.ddu[1][2]; - - out.ddv[0][0] = sin2phi / (r * r * sintheta * sintheta); - out.ddv[0][1] = -cos2phi / (r * r * sintheta * sintheta); - out.ddv[0][2] = 0.; - out.ddv[1][1] = -sin2phi / (r * r * sintheta * sintheta); - out.ddv[1][2] = 0.; - out.ddv[2][2] = 0.; - out.ddv[1][0] = out.ddv[0][1]; - out.ddv[2][0] = out.ddv[0][2]; - out.ddv[2][1] = out.ddv[1][2]; + out.ddu[iy][iz] = -(cos2theta * sinphi) / (r * r); + out.ddu[iy][ix] = out.ddu[ix][iy]; + out.ddu[iz][iy] = out.ddu[iy][iz]; +#endif + +#if CH_SPACEDIM == 3 + out.ddv[ix][ix] = sin2phi / (r * r * sintheta * sintheta); + out.ddv[ix][iz] = 0.; + out.ddv[iz][ix] = out.ddv[ix][iz]; + out.ddv[iz][iz] = 0.; + out.ddv[ix][iy] = -cos2phi / (r * r * sintheta * sintheta); + out.ddv[iy][iy] = -sin2phi / (r * r * sintheta * sintheta); + out.ddv[iy][iz] = 0.; + out.ddv[iy][ix] = out.ddv[ix][iy]; + out.ddv[iz][iy] = out.ddv[iy][iz]; +#endif double ddfdxdx = (costheta * costheta + sintheta * sintheta * sinphi * sinphi) / r; - double ddfdxdy = -(sintheta * sintheta * sinphi * cosphi) / r; double ddfdxdz = -(sintheta * costheta * cosphi) / r; + double ddfdzdz = (sintheta * sintheta) / r; +#if CH_SPACEDIM == 3 + double ddfdxdy = -(sintheta * sintheta * sinphi * cosphi) / r; double ddfdydy = (costheta * costheta + sintheta * sintheta * cosphi * cosphi) / r; double ddfdydz = -(sintheta * costheta * sinphi) / r; - double ddfdzdz = (sintheta * sintheta) / r; +#endif + + out.df[ix] = dfdx; +#if CH_SPACEDIM == 3 + out.df[iy] = dfdy; +#endif + out.df[iz] = dfdz; + + out.ddf[ix][ix] = ddfdxdx; + out.ddf[ix][iz] = ddfdxdz; + out.ddf[iz][ix] = out.ddf[ix][iz]; + out.ddf[iz][iz] = ddfdzdz; +#if CH_SPACEDIM == 3 + out.ddf[ix][iy] = ddfdxdy; + out.ddf[iy][iy] = ddfdydy; + out.ddf[iy][iz] = ddfdydz; + out.ddf[iy][ix] = out.ddf[ix][iy]; + out.ddf[iz][iy] = out.ddf[iy][iz]; +#endif - out.df[0] = dfdx; - out.df[1] = dfdy; - out.df[2] = dfdz; - - out.ddf[0][0] = ddfdxdx; - out.ddf[0][1] = ddfdxdy; - out.ddf[0][2] = ddfdxdz; - out.ddf[1][1] = ddfdydy; - out.ddf[1][2] = ddfdydz; - out.ddf[2][2] = ddfdzdz; - out.ddf[1][0] = out.ddf[0][1]; - out.ddf[2][0] = out.ddf[0][2]; - out.ddf[2][1] = out.ddf[1][2]; - - out.dxdu[0] = r * costheta * cosphi; - out.dxdu[1] = r * costheta * sinphi; - out.dxdu[2] = -r * sintheta; - - out.dxdv[0] = -r * sintheta * sinphi; - out.dxdv[1] = r * sintheta * cosphi; - out.dxdv[2] = 0.; - - out.dxdf[0] = sintheta * cosphi; - out.dxdf[1] = sintheta * sinphi; - out.dxdf[2] = costheta; + out.dxdu[ix] = r * costheta * cosphi; +#if CH_SPACEDIM == 3 + out.dxdu[iy] = r * costheta * sinphi; +#endif + out.dxdu[iz] = -r * sintheta; + +#if CH_SPACEDIM == 3 + out.dxdv[ix] = -r * sintheta * sinphi; + out.dxdv[iy] = r * sintheta * cosphi; + out.dxdv[iz] = 0.; +#endif + + out.dxdf[ix] = sintheta * cosphi; +#if CH_SPACEDIM == 3 + out.dxdf[iy] = sintheta * sinphi; +#endif + out.dxdf[iz] = costheta; return out; } diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp index 214fea388..93e69d236 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp @@ -415,12 +415,21 @@ void ApparentHorizon::solve(double a_dt, } m_area = calculate_area(); +#if GR_SPACEDIM == 3 // GR_SPACEDIM, not CH_SPACEDIM !!! + #if CH_SPACEDIM == 3 double spin_dimensionless = calculate_spin_dimensionless(m_area); +#elif CH_SPACEDIM == 2 + double spin_dimensionless = 0.; +#endif + m_mass = calculate_mass(m_area, spin_dimensionless); + +#if CH_SPACEDIM == 3 m_spin = spin_dimensionless * m_mass; +#endif - if (m_params.verbose > AHFinder::NONE) + if (m_params.verbose > AHFinder::MIN) { pout() << "mass = " << m_mass << endl; } @@ -486,11 +495,12 @@ void ApparentHorizon::write_outputs( // std::string coords_filename = file.get_new_file_number(fake_dt, // a_time); - // first '1' corresponds to 'step' - // the first (CH_SPACEDIM==3 ? 3 : 1) corresponds to area+spin+mass or - // only area CH_assert(CH_SPACEDIM == 3 || CH_SPACEDIM == 2); - std::vector values(1 + (CH_SPACEDIM == 3 ? 3 : 1) + + // first '1' corresponds to 'step' + // area+spin+mass OR area+mass in 2D Cartoon OR only area for other + // cases (e.g. 4D -> 2D cartoon) + int stats = (GR_SPACEDIM == 3 ? (CH_SPACEDIM == 3 ? 3 : 2) : 1); + std::vector values(1 + stats + CH_SPACEDIM * (1 + m_params.track_center)); auto origin = get_origin(); @@ -500,8 +510,10 @@ void ApparentHorizon::write_outputs( int idx = 0; values[idx++] = step; values[idx++] = m_area; +#if GR_SPACEDIM == 3 // GR_SPACEDIM, not CH_SPACEDIM !!! #if CH_SPACEDIM == 3 values[idx++] = m_spin; +#endif values[idx++] = m_mass; #endif for (int i = 0; i < CH_SPACEDIM; ++i) @@ -513,15 +525,16 @@ void ApparentHorizon::write_outputs( // print headers to stats file in the beginning of evolution if (!m_printed_once) { - std::vector headers(1 + (CH_SPACEDIM == 3 ? 3 : 1) + - CH_SPACEDIM * - (1 + m_params.track_center)); + std::vector headers( + 1 + stats + CH_SPACEDIM * (1 + m_params.track_center)); idx = 0; headers[idx++] = "file"; headers[idx++] = "area"; +#if GR_SPACEDIM == 3 // GR_SPACEDIM, not CH_SPACEDIM !!! #if CH_SPACEDIM == 3 headers[idx++] = "spin"; +#endif headers[idx++] = "mass"; #endif headers[idx++] = "origin_x"; @@ -776,10 +789,16 @@ void ApparentHorizon::restart( int cols = stats.size(); std::array origin; +#if GR_SPACEDIM == 3 // GR_SPACEDIM, not CH_SPACEDIM !!! #if CH_SPACEDIM == 3 bool was_center_tracked = (cols > 5 + CH_SPACEDIM); // 5 for time + file + area + spin + mass -#elif CH_SPACEDIM == 2 +#else + // 3D -> 2D Cartoon method + bool was_center_tracked = + (cols > 4 + CH_SPACEDIM); // 5 for time + file + area + mass +#endif +#else bool was_center_tracked = (cols > 3 + CH_SPACEDIM); // 3 for time + file + area #endif @@ -1318,10 +1337,9 @@ ApparentHorizon::calculate_spin_dimensionless( { for (int u = m_umin; u < m_umax; ++u) { - AHDeriv deriv = diff(in, u, v); - if (u_equator == u) { + AHDeriv deriv = diff(in, u, v); const auto geometry_data = m_interp.get_geometry_data(idx); const auto data = m_interp.get_data(idx); const auto coords = m_interp.get_coords(idx); @@ -1485,7 +1503,9 @@ double ApparentHorizon::calculate_area() // assume a (GR_SPACEDIM - CH_SPACEDIM)-sphere leftover #if GR_SPACEDIM != CH_SPACEDIM double n_sphere = (GR_SPACEDIM - CH_SPACEDIM); - element *= pow(sqrt(func.get_metric_hd()) * m_F[idx], n_sphere); + element *= pow(sqrt(func.get_metric_hd()) * + coords_cart[CH_SPACEDIM - 1], + n_sphere); #endif inner_integral += element; diff --git a/Source/GRChomboCore/SimulationParametersBase.hpp b/Source/GRChomboCore/SimulationParametersBase.hpp index 06ddbf129..4e47f3350 100644 --- a/Source/GRChomboCore/SimulationParametersBase.hpp +++ b/Source/GRChomboCore/SimulationParametersBase.hpp @@ -17,11 +17,9 @@ #include "AHFinder.hpp" #endif -#if CH_SPACEDIM == 3 #include "SphericalExtraction.hpp" // add this type alias here for backwards compatibility using extraction_params_t = SphericalExtraction::params_t; -#endif class SimulationParametersBase : public ChomboParameters { @@ -59,72 +57,79 @@ class SimulationParametersBase : public ChomboParameters pp.load("min_chi", min_chi, 1e-4); pp.load("min_lapse", min_lapse, 1e-4); -#if CH_SPACEDIM == 3 // Extraction params pp.load("num_extraction_radii", extraction_params.num_extraction_radii, - 1); + 0); // Check for multiple extraction radii, otherwise load single // radius/level (for backwards compatibility). - if (pp.contains("extraction_levels")) - { - pp.load("extraction_levels", extraction_params.extraction_levels, - extraction_params.num_extraction_radii); - } - else - { - pp.load("extraction_level", extraction_params.extraction_levels, 1, - 0); - } - if (pp.contains("extraction_radii")) + if (extraction_params.num_extraction_radii > 0) { - pp.load("extraction_radii", extraction_params.extraction_radii, - extraction_params.num_extraction_radii); - } - else - { - pp.load("extraction_radius", extraction_params.extraction_radii, 1, - 0.1); - } + if (pp.contains("extraction_levels")) + { + pp.load("extraction_levels", + extraction_params.extraction_levels, + extraction_params.num_extraction_radii); + } + else + { + pp.load("extraction_level", extraction_params.extraction_levels, + 1, 0); + } + if (pp.contains("extraction_radii")) + { + pp.load("extraction_radii", extraction_params.extraction_radii, + extraction_params.num_extraction_radii); + } + else + { + pp.load("extraction_radius", extraction_params.extraction_radii, + 1, 0.1); + } - pp.load("num_points_phi", extraction_params.num_points_phi, 2); - pp.load("num_points_theta", extraction_params.num_points_theta, 5); - if (extraction_params.num_points_theta % 2 == 0) - { - extraction_params.num_points_theta += 1; - pout() << "Parameter: num_points_theta incompatible with Simpson's " - << "rule so increased by 1.\n"; - } - pp.load("extraction_center", extraction_params.center, center); + pp.load("num_points_phi", extraction_params.num_points_phi, 2); + pp.load("num_points_theta", extraction_params.num_points_theta, 5); + if (extraction_params.num_points_theta % 2 == 0) + { + extraction_params.num_points_theta += 1; + pout() << "Parameter: num_points_theta incompatible with " + "Simpson's " + << "rule so increased by 1.\n"; + } + pp.load("extraction_center", extraction_params.center, center); - if (pp.contains("modes")) - { - pp.load("num_modes", extraction_params.num_modes); - std::vector extraction_modes_vect(2 * - extraction_params.num_modes); - pp.load("modes", extraction_modes_vect, - 2 * extraction_params.num_modes); - extraction_params.modes.resize(extraction_params.num_modes); - for (int i = 0; i < extraction_params.num_modes; ++i) + if (pp.contains("modes")) { - extraction_params.modes[i].first = extraction_modes_vect[2 * i]; - extraction_params.modes[i].second = - extraction_modes_vect[2 * i + 1]; + pp.load("num_modes", extraction_params.num_modes); + std::vector extraction_modes_vect( + 2 * extraction_params.num_modes); + pp.load("modes", extraction_modes_vect, + 2 * extraction_params.num_modes); + extraction_params.modes.resize(extraction_params.num_modes); + for (int i = 0; i < extraction_params.num_modes; ++i) + { + extraction_params.modes[i].first = + extraction_modes_vect[2 * i]; + extraction_params.modes[i].second = + extraction_modes_vect[2 * i + 1]; + } } - } - else - { - // by default extraction (l,m) = (2,0), (2,1) and (2,2) - extraction_params.num_modes = 3; - extraction_params.modes.resize(3); - for (int i = 0; i < 3; ++i) + else { - extraction_params.modes[i].first = 2; - extraction_params.modes[i].second = i; + // by default extraction (l,m) = (2,0), (2,1) and (2,2) + extraction_params.num_modes = 3; + extraction_params.modes.resize(3); + for (int i = 0; i < 3; ++i) + { + extraction_params.modes[i].first = 2; + extraction_params.modes[i].second = i; + } } - } - pp.load("write_extraction", extraction_params.write_extraction, false); -#endif + pp.load("write_extraction", extraction_params.write_extraction, + false); + pp.load("extraction_prefix", + extraction_params.extraction_prefix, ""); + } #ifdef USE_AHFINDER // Apparent horizon parameters @@ -212,47 +217,55 @@ class SimulationParametersBase : public ChomboParameters std::numeric_limits::epsilon(), "set to 2.0 for 1+log slicing"); -#if CH_SPACEDIM == 3 // Now extraction parameters - FOR1(idir) + if (extraction_params.num_extraction_radii > 0) { - std::string center_name = - "extraction_center[" + std::to_string(idir) + "]"; - double center_in_dir = extraction_params.center[idir]; - check_parameter(center_name, center_in_dir, - (center_in_dir >= reflective_domain_lo[idir]) && - (center_in_dir <= reflective_domain_hi[idir]), - "must be in the computational domain after " - "applying reflective symmetry"); - for (int iradius = 0; - iradius < extraction_params.num_extraction_radii; ++iradius) + + FOR1(idir) { - std::string radius_name = - "extraction_radii[" + std::to_string(iradius) + "]"; - double radius = extraction_params.extraction_radii[iradius]; - if (idir == 0) - check_parameter(radius_name, radius, radius >= 0.0, - "must be >= 0.0"); + std::string center_name = + "extraction_center[" + std::to_string(idir) + "]"; + double center_in_dir = extraction_params.center[idir]; check_parameter( - radius_name, radius, - (center_in_dir - radius >= reflective_domain_lo[idir]) && - (center_in_dir + radius <= reflective_domain_hi[idir]), - "extraction sphere must lie within the computational " - "domain after applying reflective symmetry"); + center_name, center_in_dir, + (center_in_dir >= reflective_domain_lo[idir]) && + (center_in_dir <= reflective_domain_hi[idir]), + "must be in the computational domain after " + "applying reflective symmetry"); + for (int iradius = 0; + iradius < extraction_params.num_extraction_radii; + ++iradius) + { + std::string radius_name = + "extraction_radii[" + std::to_string(iradius) + "]"; + double radius = extraction_params.extraction_radii[iradius]; + if (idir == 0) + check_parameter(radius_name, radius, radius >= 0.0, + "must be >= 0.0"); + check_parameter( + radius_name, radius, + (center_in_dir - radius >= + reflective_domain_lo[idir]) && + (center_in_dir + radius <= + reflective_domain_hi[idir]), + "extraction sphere must lie within the computational " + "domain after applying reflective symmetry"); + } + } + for (int imode = 0; imode < extraction_params.num_modes; ++imode) + { + auto &mode = extraction_params.modes[imode]; + int l = mode.first; + int m = mode.second; + std::string mode_name = "modes[" + std::to_string(imode) + "]"; + std::string value_str = "(" + std::to_string(mode.first) + + ", " + std::to_string(mode.second) + + ")"; + check_parameter( + mode_name, value_str, (l >= 2) && (abs(m) <= l), + "l must be >= 2 and m must satisfy -l <= m <= l"); } } - for (int imode = 0; imode < extraction_params.num_modes; ++imode) - { - auto &mode = extraction_params.modes[imode]; - int l = mode.first; - int m = mode.second; - std::string mode_name = "modes[" + std::to_string(imode) + "]"; - std::string value_str = "(" + std::to_string(mode.first) + ", " + - std::to_string(mode.second) + ")"; - check_parameter(mode_name, value_str, (l >= 2) && (abs(m) <= l), - "l must be >= 2 and m must satisfy -l <= m <= l"); - } -#endif } public: @@ -266,9 +279,7 @@ class SimulationParametersBase : public ChomboParameters // Collection of parameters necessary for the CCZ4 RHS and extraction CCZ4::params_t ccz4_params; -#if CH_SPACEDIM == 3 SphericalExtraction::params_t extraction_params; -#endif #ifdef USE_AHFINDER bool AH_activate; diff --git a/Source/TaggingCriteria/FixedGridsTaggingCriterion.hpp b/Source/TaggingCriteria/FixedGridsTaggingCriterion.hpp index 90b94f3cf..905c134ff 100644 --- a/Source/TaggingCriteria/FixedGridsTaggingCriterion.hpp +++ b/Source/TaggingCriteria/FixedGridsTaggingCriterion.hpp @@ -23,7 +23,7 @@ class FixedGridsTaggingCriterion FixedGridsTaggingCriterion(const double dx, const int a_level, const double a_L, const std::array a_center) - : m_dx(dx), m_level(a_level), m_L(a_L), m_center(a_center){}; + : m_dx(dx), m_L(a_L), m_level(a_level), m_center(a_center){}; template void compute(Cell current_cell) const { diff --git a/Source/utils/DimensionDefinitions.hpp b/Source/utils/DimensionDefinitions.hpp index fb680a3b8..9894e8e80 100644 --- a/Source/utils/DimensionDefinitions.hpp +++ b/Source/utils/DimensionDefinitions.hpp @@ -11,7 +11,7 @@ #endif #ifndef DEFAULT_TENSOR_DIM -#define DEFAULT_TENSOR_DIM 3 +#define DEFAULT_TENSOR_DIM CH_SPACEDIM #endif #define FOR1(IDX) for (int IDX = 0; IDX < DEFAULT_TENSOR_DIM; ++IDX) diff --git a/Tests/ApparentHorizonFinderTest2D/GNUmakefile b/Tests/ApparentHorizonFinderTest2D/GNUmakefile index 655f421cd..918c6d265 100644 --- a/Tests/ApparentHorizonFinderTest2D/GNUmakefile +++ b/Tests/ApparentHorizonFinderTest2D/GNUmakefile @@ -8,7 +8,7 @@ # DIM = 2 -cxxcppflags := ${cxxcppflags} -DGR_SPACEDIM=2 -DDEFAULT_TENSOR_DIM=2 +cxxcppflags := ${cxxcppflags} -DGR_SPACEDIM=2 -DDEFAULT_TENSOR_DIM=2 GRCHOMBO_SOURCE = ../../Source diff --git a/Tests/ApparentHorizonFinderTest2D/UserVariables.hpp b/Tests/ApparentHorizonFinderTest2D/UserVariables.hpp index 8d3433bdd..1b7091284 100644 --- a/Tests/ApparentHorizonFinderTest2D/UserVariables.hpp +++ b/Tests/ApparentHorizonFinderTest2D/UserVariables.hpp @@ -25,9 +25,8 @@ static const std::array variable_names = {"V"}; #include "UserVariables.inc.hpp" -// already the 2D default -// #include "AHStringGeometry.hpp" -// #define AHSurfaceGeometry AHStringGeometry +#include "AHStringGeometry.hpp" +#define AHSurfaceGeometry AHStringGeometry #include "AHTest2DFunction.hpp" #define AHFunction AHTest2DFunction From e61576e36fa6eb73edb6941368f224462869de9d Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Fri, 19 Feb 2021 09:55:06 +0000 Subject: [PATCH 07/92] Minor changes --- Source/AMRInterpolator/InterpSource.hpp | 3 +-- Source/ApparentHorizonFinder/AHSphericalGeometry.hpp | 4 ++++ Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp | 11 ++++++++++- Tests/ApparentHorizonFinderTest2D/GNUmakefile | 2 +- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/Source/AMRInterpolator/InterpSource.hpp b/Source/AMRInterpolator/InterpSource.hpp index 45e63851f..2a53c5848 100644 --- a/Source/AMRInterpolator/InterpSource.hpp +++ b/Source/AMRInterpolator/InterpSource.hpp @@ -22,8 +22,7 @@ template class InterpSource public: virtual const LevelData &getLevelData( const VariableType var_type = VariableType::evolution) const = 0; - virtual bool - contains(const std::array &point) const = 0; + virtual bool contains(const std::array &point) const = 0; }; #endif /* INTERPSOURCE_H_ */ diff --git a/Source/ApparentHorizonFinder/AHSphericalGeometry.hpp b/Source/ApparentHorizonFinder/AHSphericalGeometry.hpp index 8f905d5bc..41bfcc63c 100644 --- a/Source/ApparentHorizonFinder/AHSphericalGeometry.hpp +++ b/Source/ApparentHorizonFinder/AHSphericalGeometry.hpp @@ -43,6 +43,10 @@ class AHSphericalGeometry : public SphericalGeometry AHGeometryData get_geometry_data(double r, double theta, double phi = 0.) const { + // for 2D Cartoon spherical geometry (3D->2D), this should always be + // called with no 3rd argument (so phi=0), which means the formulas + // below are effectively polar coordinates + CH_TIME("AHSphericalGeometry::get_geometry_data"); double costheta = cos(theta); diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp index 93e69d236..fd345543a 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp @@ -429,7 +429,7 @@ void ApparentHorizon::solve(double a_dt, m_spin = spin_dimensionless * m_mass; #endif - if (m_params.verbose > AHFinder::MIN) + if (m_params.verbose > AHFinder::NONE) { pout() << "mass = " << m_mass << endl; } @@ -471,6 +471,13 @@ template void ApparentHorizon::write_outputs( double a_dt, double a_time, double a_restart_time) { + // print step. Printing the step allows the user to modify params as + // 'solve_interval', 'print_interval' or even 'dt_multiplier' (that change + // the frequency of writing outputs) and the class will still be able to + // restart from the correct file (this only applies for frequency increase, + // for which the file number will suddenly jump, as for frequency decrease + // the AHFinder may override old coord files) + // print stats (area, spin, origin, center) and coordinates // stop printing if it stopped converging // then in 'write_coords_file' nothing is printed if not converged @@ -698,6 +705,8 @@ void ApparentHorizon::restart( else if (idx == 0 && stats[0].size() > 1) old_print_dt = stats[0][idx + 1] - stats[0][idx]; // there's still time steps forward + else // if we can't know, just default to the current one + old_print_dt = current_solve_dt * m_params.print_interval; if (m_params.verbose > AHFinder::SOME) { diff --git a/Tests/ApparentHorizonFinderTest2D/GNUmakefile b/Tests/ApparentHorizonFinderTest2D/GNUmakefile index 918c6d265..45c0adbeb 100644 --- a/Tests/ApparentHorizonFinderTest2D/GNUmakefile +++ b/Tests/ApparentHorizonFinderTest2D/GNUmakefile @@ -8,7 +8,7 @@ # DIM = 2 -cxxcppflags := ${cxxcppflags} -DGR_SPACEDIM=2 -DDEFAULT_TENSOR_DIM=2 +cxxcppflags := ${cxxcppflags} -DGR_SPACEDIM=2 GRCHOMBO_SOURCE = ../../Source From 74abc55aa050d071daa83d89f0b5ad58250e6862 Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Mon, 1 Mar 2021 12:49:37 +0000 Subject: [PATCH 08/92] Minor changes --- Source/ApparentHorizonFinder/AHFinder.cpp | 5 +++++ Source/ApparentHorizonFinder/AHSphericalGeometry.hpp | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/Source/ApparentHorizonFinder/AHFinder.cpp b/Source/ApparentHorizonFinder/AHFinder.cpp index 221ece9b8..cadc737da 100644 --- a/Source/ApparentHorizonFinder/AHFinder.cpp +++ b/Source/ApparentHorizonFinder/AHFinder.cpp @@ -97,6 +97,11 @@ void AHFinder::solve(double a_dt, double a_time, double a_restart_time) SOLVED }; + // in case this was called at specificPostTimeStep at t=0 due to + // MultiLevelTask (solve already happened when AH was created) + if (a_time == 0.) + return; + // solve if it has never converged before OR if has already converged in the // past this means it will stop solving as soon as it stops converging but // keeps trying (with same initial guess) if never found one diff --git a/Source/ApparentHorizonFinder/AHSphericalGeometry.hpp b/Source/ApparentHorizonFinder/AHSphericalGeometry.hpp index 41bfcc63c..a3d0c4ce3 100644 --- a/Source/ApparentHorizonFinder/AHSphericalGeometry.hpp +++ b/Source/ApparentHorizonFinder/AHSphericalGeometry.hpp @@ -6,6 +6,12 @@ #ifndef AHSPHERICALGEOMETRY_HPP_ #define AHSPHERICALGEOMETRY_HPP_ +#include "DimensionDefinitions.hpp" // make sure GR_SPACEDIM exists + +#if GR_SPACEDIM != 3 +#error "This file should only be included for GR_SPACEDIM == 3." +#endif + // Chombo includes #include "CH_Timer.H" From d7fb648f777db26d2e62dd0e2284f86802cff66d Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Mon, 1 Mar 2021 12:49:53 +0000 Subject: [PATCH 09/92] Add spin as a vector calculation --- Examples/BinaryBH/SimulationParameters.hpp | 2 +- Examples/KerrBH/SimulationParameters.hpp | 2 + Examples/KerrBH/params.txt | 2 +- Examples/KerrBH/params_cheap.txt | 25 +- Examples/SingleBH/params.txt | 6 +- .../AHFunctionDefault.hpp | 27 ++ Source/ApparentHorizonFinder/AHFunctions.hpp | 165 +++++---- .../ApparentHorizonFinder/ApparentHorizon.hpp | 25 +- .../ApparentHorizon.impl.hpp | 232 +++++++++++-- .../ApparentHorizon_petsc.impl.hpp | 2 +- .../InitialConditions/BlackHoles/KerrBH.hpp | 4 +- .../BlackHoles/KerrBH.impl.hpp | 52 ++- Source/utils/CoordinateTransformations.hpp | 312 ++++++++++-------- .../ApparentHorizonTest3D.cpp | 9 +- .../ApparentHorizonTest3D.inputs | 15 +- .../ApparentHorizonTest3DLevel.hpp | 6 +- .../SimulationParameters.hpp | 2 + 17 files changed, 601 insertions(+), 287 deletions(-) diff --git a/Examples/BinaryBH/SimulationParameters.hpp b/Examples/BinaryBH/SimulationParameters.hpp index c6d0e2c5a..4112c12b0 100644 --- a/Examples/BinaryBH/SimulationParameters.hpp +++ b/Examples/BinaryBH/SimulationParameters.hpp @@ -22,12 +22,12 @@ class SimulationParameters : public SimulationParametersBase public: SimulationParameters(GRParmParse &pp) : SimulationParametersBase(pp) { - read_shared_params(pp); #ifdef USE_TWOPUNCTURES read_tp_params(pp); #else read_bh_params(pp); #endif + read_shared_params(pp); check_params(); } diff --git a/Examples/KerrBH/SimulationParameters.hpp b/Examples/KerrBH/SimulationParameters.hpp index ed4b6dcc4..f6bbb5088 100644 --- a/Examples/KerrBH/SimulationParameters.hpp +++ b/Examples/KerrBH/SimulationParameters.hpp @@ -29,6 +29,8 @@ class SimulationParameters : public SimulationParametersBase pp.load("kerr_mass", kerr_params.mass); pp.load("kerr_spin", kerr_params.spin); pp.load("kerr_center", kerr_params.center, center); + pp.load("kerr_spin_direction", kerr_params.spin_direction, + {0., 0., 1.}); #ifdef USE_AHFINDER pp.load("AH_initial_guess", AH_initial_guess, 0.5 * kerr_params.mass); diff --git a/Examples/KerrBH/params.txt b/Examples/KerrBH/params.txt index 95ff1ea2d..394973f1f 100644 --- a/Examples/KerrBH/params.txt +++ b/Examples/KerrBH/params.txt @@ -128,7 +128,7 @@ AH_stop_if_max_fails = 1 #AH_max_fails_after_lost = 0 # -1 to never #AH_print_geometry_data = 0 #AH_re_solve_at_restart = 0 -AH_verbose = 3 +AH_verbose = 2 #AH_initial_guess = 0.5 diff --git a/Examples/KerrBH/params_cheap.txt b/Examples/KerrBH/params_cheap.txt index 1b5735588..63dc452c1 100644 --- a/Examples/KerrBH/params_cheap.txt +++ b/Examples/KerrBH/params_cheap.txt @@ -13,7 +13,7 @@ plot_prefix = KerrBHp_ # 'N2' or 'N2_full' and 'N3' or 'N3_full' ( then dx_coarsest = L/N(max) ) # NB - the N values need to be multiples of the block_factor N_full = 64 -L_full = 128 +L_full = 16 # Spatial derivative order (only affects CCZ4 RHS) max_spatial_derivative_order = 4 # can be 4 or 6 @@ -21,6 +21,7 @@ max_spatial_derivative_order = 4 # can be 4 or 6 # Params for Kerr BH kerr_mass = 1.0 kerr_spin = 0.5 +kerr_spin_direction = 0.1 0.1 0. # doesn't need to be normalized # This is now defaulted to center of the grid so it does not need to be updated # when changing L (unless one wants to put the BH somewhere off center) #kerr_center = 64 64 64 @@ -52,7 +53,7 @@ isPeriodic = 0 0 0 # 0 = static, 1 = sommerfeld, 2 = reflective # (see BoundaryConditions.hpp for details) hi_boundary = 1 1 1 -lo_boundary = 1 1 2 +lo_boundary = 1 1 1 # if reflective boundaries selected, must set # parity of all vars (in order given by UserVariables.hpp) @@ -108,7 +109,7 @@ sigma = 0.3 #extraction params activate_extraction = 0 -num_extraction_radii = 3 +num_extraction_radii = 0 # 3 extraction_radii = 30. 40. 50. extraction_levels = 1 1 0 num_points_phi = 50 @@ -116,9 +117,9 @@ num_points_theta = 52 #Apparent Horizon finder AH_activate = 1 -AH_num_ranks = 2 #20 -AH_num_points_u = 11 #30 -AH_num_points_v = 10 #50 +AH_num_ranks = 4 #20 +AH_num_points_u = 15 #30 +AH_num_points_v = 20 #50 AH_solve_interval = 2 AH_print_interval = 2 AH_track_center = false @@ -126,14 +127,16 @@ AH_predict_origin = false AH_level_to_run = 0 #AH_allow_re_attempt = 0 #AH_stop_if_max_fails = 0 -AH_start_time = 0. -AH_verbose = 3 -AH_print_geometry_data = 0 -AH_re_solve_at_restart = 0 +#AH_start_time = 0. +#AH_give_up_time = -1. # -1 to never +#AH_max_fails_after_lost = 0 # -1 to never +#AH_print_geometry_data = 0 +#AH_re_solve_at_restart = 0 +AH_verbose = 2 #AH_initial_guess = 0.5 -look_for_chi_contour = 0.2 +#look_for_chi_contour = 0.2 AH_num_write_vars = 2 AH_write_vars = chi d1_Ham \ No newline at end of file diff --git a/Examples/SingleBH/params.txt b/Examples/SingleBH/params.txt index e60fb6c1d..af5f21ca7 100644 --- a/Examples/SingleBH/params.txt +++ b/Examples/SingleBH/params.txt @@ -14,10 +14,10 @@ max_spatial_derivative_order = 4 # can be 4 or 6 mass = 1.0 offset = 0 0 0 -momentum = 0. -0.5 0.0 +momentum = 0. -0.1 0.0 regrid_threshold = 0.05 -max_level = 2 +max_level = 3 regrid_interval = 1 2 4 8 16 @@ -87,7 +87,7 @@ sigma = 0.3 #Apparent Horizon finder AH_activate = 1 -AH_num_ranks = 2 +AH_num_ranks = 4 AH_num_points_u = 11 AH_num_points_v = 20 AH_solve_interval = 1 diff --git a/Source/ApparentHorizonFinder/AHFunctionDefault.hpp b/Source/ApparentHorizonFinder/AHFunctionDefault.hpp index 53df9f56f..902604f90 100644 --- a/Source/ApparentHorizonFinder/AHFunctionDefault.hpp +++ b/Source/ApparentHorizonFinder/AHFunctionDefault.hpp @@ -6,6 +6,9 @@ #ifndef _AHFUNCTIONDEFAULT_HPP_ #define _AHFUNCTIONDEFAULT_HPP_ +#include "AHData.hpp" +#include "AHDeriv.hpp" +#include "AHGeometryData.hpp" #include "AlwaysInline.hpp" #include "Tensor.hpp" @@ -42,6 +45,10 @@ struct AHFunctionDefault FOR1(i) { g[i][i] = 1.; } return g; } + ALWAYS_INLINE const Tensor<2, double> get_extrinsic_curvature() const + { + return {0.}; + } // case of reduced Cartoon methods that have an extra metric component #if GR_SPACEDIM != CH_SPACEDIM // hd - higher dimensions @@ -52,6 +59,26 @@ struct AHFunctionDefault { }; + // not defined by default + Tensor<1, double> + get_level_function_derivative(const AHGeometryData &geo_data, + const AHDeriv &deriv) const + { + return {0.}; + } + Tensor<1, double> get_spatial_normal_U(const Tensor<1, double> &s_L) const + { + return {0.}; + } + // not defined by default + Tensor<2, double> get_level_function_2nd_covariant_derivative( + const AHGeometryData &geo_data, const AHDeriv &deriv, + const Tensor<1, double> &s_L) const + { + return {0.}; + } + + // WHAT TO ADD TO YOUR OWN FUNCTIONS: // some constructor with these arguments: // AHFunctionDefault(const AHData &a_data, // const Tensor<1, double> &a_coords, diff --git a/Source/ApparentHorizonFinder/AHFunctions.hpp b/Source/ApparentHorizonFinder/AHFunctions.hpp index dd04ef99c..bebec63a4 100644 --- a/Source/ApparentHorizonFinder/AHFunctions.hpp +++ b/Source/ApparentHorizonFinder/AHFunctions.hpp @@ -10,9 +10,6 @@ #include "CH_Timer.H" // Other includes -#include "AHData.hpp" -#include "AHDeriv.hpp" -#include "AHGeometryData.hpp" #include "DimensionDefinitions.hpp" // make sure GR_SPACEDIM exists #include "TensorAlgebra.hpp" #include "UserVariables.hpp" @@ -48,7 +45,7 @@ struct ExpansionFunction : AHFunctionDefault // require the coordinates to calculate the expansion #if GR_SPACEDIM != CH_SPACEDIM // hd - higher dimensions - Tensor<1, double> x; // cartesian coordinates + Tensor<1, double> coords; // cartesian coordinates double g_hd; Tensor<1, double> dg_hd; @@ -69,6 +66,10 @@ struct ExpansionFunction : AHFunctionDefault static ALWAYS_INLINE int d1_vars_max() { return c_K - 1; } ALWAYS_INLINE const Tensor<2, double> get_metric() const { return g; } + ALWAYS_INLINE const Tensor<2, double> get_extrinsic_curvature() const + { + return K; + } static int num_write_vars() { @@ -154,7 +155,6 @@ struct ExpansionFunction : AHFunctionDefault Tensor<2, double, CH_SPACEDIM> h_UU = TensorAlgebra::compute_inverse_sym(h_DD); - FOR2(i, j) { g_UU[i][j] = chi * h_UU[i][j]; } // Reconstructing ADM variables Tensor<1, double, CH_SPACEDIM> dchi; @@ -166,10 +166,15 @@ struct ExpansionFunction : AHFunctionDefault for (int j = i; j < CH_SPACEDIM; ++j) { { - const double gij = h_DD[i][j] / (chi); + const double gij = h_DD[i][j] / chi; g[i][j] = gij; g[j][i] = gij; } + { + const double g_UUij = h_UU[i][j] * chi; + g_UU[i][j] = g_UUij; + g_UU[j][i] = g_UUij; + } { const double Aij = a_data.vars.at(A[i][j]); @@ -199,7 +204,7 @@ struct ExpansionFunction : AHFunctionDefault int comp_Aww = c_Aww; // is (c_Theta-1) more general? Tensor<1, double, CH_SPACEDIM> dhww; - x = a_coords_cartesian; + coords = a_coords_cartesian; double hww = a_data.vars.at(comp_hww); FOR1(a) { dhww[a] = a_data.d1.at(comp_hww)[a]; } @@ -212,53 +217,107 @@ struct ExpansionFunction : AHFunctionDefault #endif } - struct params // no params needed + struct params { double expansion_radius_power = 1.; }; double get(const AHGeometryData &geo_data, const AHDeriv &deriv, const params &a_params) const { - // first calculate D_a L of 6.7.12 of Alcubierre for some level function + Tensor<1, double> s_L = get_level_function_derivative(geo_data, deriv); + Tensor<2, double> Ds = + get_level_function_2nd_covariant_derivative(geo_data, deriv, s_L); + Tensor<1, double> S_U = get_spatial_normal_U(s_L); + + // calculate D_i S^i and S^i S^j K_ij + double DiSi = 0.; + double Kij_dot_Si_Sj = 0.; + FOR2(a, b) + { + DiSi += (g_UU[a][b] - S_U[a] * S_U[b]) * Ds[a][b]; + Kij_dot_Si_Sj += S_U[a] * S_U[b] * K[a][b]; + } + + // Calculation of expansion - as in (6.7.9 / 6.7.13) of Alcubierre + double expansion = DiSi - trK + Kij_dot_Si_Sj; + + // part from extra dimensions in the case of Cartoon methods +#if GR_SPACEDIM != CH_SPACEDIM + expansion += (GR_SPACEDIM - CH_SPACEDIM) * S_U[CH_SPACEDIM - 1] / + coords[CH_SPACEDIM - 1]; + FOR1(a) + { + expansion += + (GR_SPACEDIM - CH_SPACEDIM) * 0.5 * dg_hd[a] / g_hd * S_U[a]; + } +#endif + + // using "r * Expansion" significantly improves the convergence + // (making a Schw. BH converge for any radius >~ 0.5*r_AH instead of + // only up to ~ 3 * r_AH as it happens just with the expansion) + return expansion * pow(f, a_params.expansion_radius_power); + } + + // extra stuff: + Tensor<1, double> + get_level_function_derivative(const AHGeometryData &geo_data, + const AHDeriv &deriv) const + { + // calculate D_a L of 6.7.12 of Alcubierre for some level function // L picking L = f - F(u,v) // D_a L = d_a f - dF/du * du/dx^a - dF/dv * dv/dx^a -#if CH_SPACEDIM == 3 - Tensor<1, double> s = {0.}; + Tensor<1, double> s_L = {0.}; // not normalized, just D_a L FOR1(a) { - s[a] = geo_data.df[a] - (deriv.duF * geo_data.du[a]) - - (deriv.dvF * geo_data.dv[a]); + s_L[a] = geo_data.df[a] - (deriv.duF * geo_data.du[a]) +#if CH_SPACEDIM == 3 + - (deriv.dvF * geo_data.dv[a]) +#endif + ; } - // just the partial derivative of the above + return s_L; + } + Tensor<1, double> get_spatial_normal_U(const Tensor<1, double> &s_L) const + { + // calculate S_U = the real 's' of Alcubierre + + // norm of s_L = | D_a L| (the 'u' in 6.7.12 of Alcubierre) + double norm_s = 0.0; + FOR2(a, b) { norm_s += g_UU[a][b] * s_L[a] * s_L[b]; } + norm_s = sqrt(norm_s); + + // raise s_L + Tensor<1, double> s_U = {0.}; + FOR2(a, b) { s_U[a] += g_UU[a][b] * s_L[b]; } + + Tensor<1, double> S_U = {0.}; + FOR1(a) { S_U[a] = s_U[a] / norm_s; } + + return S_U; + } + + Tensor<2, double> get_level_function_2nd_covariant_derivative( + const AHGeometryData &geo_data, const AHDeriv &deriv, + const Tensor<1, double> &s_L) const + { + // calculates D_a D_b L, required for 6.7.13 of Alcubierre + // for the level function L = f - F(u,v) + Tensor<2, double> ds = {0.}; FOR2(a, b) { ds[a][b] = geo_data.ddf[a][b] - (deriv.duF * geo_data.ddu[a][b]) - - (deriv.dvF * geo_data.ddv[a][b]) - - (deriv.duduF * geo_data.du[a] * geo_data.du[b]) - + (deriv.duduF * geo_data.du[a] * geo_data.du[b]) +#if CH_SPACEDIM == 3 + - (deriv.dvF * geo_data.ddv[a][b]) - (deriv.dvdvF * geo_data.dv[a] * geo_data.dv[b]) - (deriv.dudvF * (geo_data.du[a] * geo_data.dv[b] + - geo_data.du[b] * geo_data.dv[a])); - } -#elif CH_SPACEDIM == 2 - Tensor<1, double> s = {0.}; - FOR1(a) { s[a] = geo_data.df[a] - (deriv.duF * geo_data.du[a]); } - - // just the partial derivative of the above - Tensor<2, double> ds = {0.}; - FOR2(a, b) - { - ds[a][b] = geo_data.ddf[a][b] - (deriv.duF * geo_data.ddu[a][b]) - - (deriv.duduF * geo_data.du[a] * geo_data.du[b]); - } + geo_data.du[b] * geo_data.dv[a])) #endif - - // calculate the norm. This is also the norm of s_U (s^a)) - double norm_s = 0.0; - FOR2(a, b) { norm_s += g_UU[a][b] * s[a] * s[b]; } - norm_s = sqrt(norm_s); + ; + } // calculate Christoffels on this point (u,v,f) Tensor<3, double> chris = {0.}; @@ -268,49 +327,15 @@ struct ExpansionFunction : AHFunctionDefault 0.5 * g_UU[a][d] * (dg[b][d][c] + dg[c][d][b] - dg[b][c][d]); } - // raise s_a - Tensor<1, double> s_U = {0.}; - FOR2(a, b) { s_U[a] += g_UU[a][b] * s[b]; } - - // S = the real 's' of Alcubierre - Tensor<1, double> S_U = {0.}; - FOR1(a) { S_U[a] = s_U[a] / norm_s; } - // covariant derivatrive of s_a to use for DS Tensor<2, double> Ds = {0.}; FOR2(a, b) { Ds[a][b] = ds[a][b]; - FOR1(c) { Ds[a][b] -= chris[c][a][b] * s[c]; } + FOR1(c) { Ds[a][b] -= chris[c][a][b] * s_L[c]; } } - // calculate D_i S^i and S^i S^j K_ij - double DiSi = 0.; - double Kij_dot_Si_Sj = 0.; - FOR2(a, b) - { - DiSi += (g_UU[a][b] - S_U[a] * S_U[b]) * Ds[a][b] / norm_s; - Kij_dot_Si_Sj += S_U[a] * S_U[b] * K[a][b]; - } - - // Calculation of expansion - as in (6.7.9) of Alcubierre - double expansion = DiSi - trK + Kij_dot_Si_Sj; - - // part from extra dimensions in the case of Cartoon methods -#if GR_SPACEDIM != CH_SPACEDIM - expansion += (GR_SPACEDIM - CH_SPACEDIM) * S_U[CH_SPACEDIM - 1] / - x[CH_SPACEDIM - 1]; - FOR1(a) - { - expansion += - (GR_SPACEDIM - CH_SPACEDIM) * 0.5 * dg_hd[a] / g_hd * S_U[a]; - } -#endif - - // using "r * Expansion" significantly improves the convergence - // (making a Schw. BH converge for any radius >~ 0.5*r_AH instead of - // only up to ~ 3 * r_AH as it happens just with the expansion) - return expansion * pow(f, a_params.expansion_radius_power); + return Ds; } }; diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.hpp index 5310672cc..34fcdc107 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.hpp @@ -61,12 +61,14 @@ template class ApparentHorizon const std::array &get_origin() const; const std::array &get_center() const; - void set_origin(const std::array &a_origin); double get_initial_guess() const; void set_initial_guess(double a_initial_guess); - const Interpolation &get_ah_interp() const; + // set origin to whatever you want (e.g. punctures) before solving if you + // want, otherwise we use the last center or the estimate next center + void set_origin(const std::array &a_origin); + double get_max_F() const; double get_min_F() const; double get_ave_F() const; @@ -99,15 +101,19 @@ template class ApparentHorizon double calculate_area(); //!< calculate AH area #if CH_SPACEDIM == 3 + Tensor<1, double> calculate_angular_momentum_J(); //!< calculate spin, ONLY + //!< FOR 3D double calculate_spin_dimensionless( double a_area); //!< calculate spin with 'z' direction, ONLY FOR 3D #endif - // estimate based on area and spin - ALWAYS_INLINE double calculate_mass(double area, double spin_dimensionless) + // estimate based on area and angular momentum J + ALWAYS_INLINE double calculate_mass(double area, double J_norm) + { + return sqrt(area / (16. * M_PI) + J_norm * J_norm * 4. * M_PI / area); + } + ALWAYS_INLINE double calculate_irreducible_mass(double area) { - return sqrt( - area / (8. * M_PI * - (1. + sqrt(1. - spin_dimensionless * spin_dimensionless)))); + return calculate_mass(area, 0.); } void calculate_minmax_F() const; @@ -167,8 +173,9 @@ template class ApparentHorizon // method std::array m_integration_methods; - double m_area, m_spin, - m_mass; // just to save the result temporarily at each iteration + // just to save the result temporarily at each iteration + double m_area, m_spin, m_mass, m_irreducible_mass, m_spin_z_alt; + Tensor<1, double> m_dimensionless_spin_vector; ///////////////////////////////////////////////////////////////////////// /////////////////////////// PETSc stuff below /////////////////////////// diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp index fd345543a..2ee5ad254 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp @@ -414,24 +414,49 @@ void ApparentHorizon::solve(double a_dt, return; // do nothing else } + // update center + // note that after this, the point that corresponds to the (u,v,r) + // coordinates is 'get_origin()', not 'get_center()' + if (m_params.track_center) + calculate_center(); + m_area = calculate_area(); #if GR_SPACEDIM == 3 // GR_SPACEDIM, not CH_SPACEDIM !!! - #if CH_SPACEDIM == 3 - double spin_dimensionless = calculate_spin_dimensionless(m_area); + Tensor<1, double> J = calculate_angular_momentum_J(); + double J_norm = sqrt(J[0] * J[0] + J[1] * J[1] + J[2] * J[2]); #elif CH_SPACEDIM == 2 - double spin_dimensionless = 0.; + double J_norm = 0.; #endif - - m_mass = calculate_mass(m_area, spin_dimensionless); + m_mass = calculate_mass(m_area, J_norm); + m_irreducible_mass = calculate_irreducible_mass(m_area); #if CH_SPACEDIM == 3 - m_spin = spin_dimensionless * m_mass; + m_spin = J_norm / m_mass; + FOR1(a) { m_dimensionless_spin_vector[a] = J[a] / (m_mass * m_mass); } + + m_spin_z_alt = calculate_spin_dimensionless(m_area); #endif if (m_params.verbose > AHFinder::NONE) { pout() << "mass = " << m_mass << endl; +#if CH_SPACEDIM == 3 + pout() << "spin = " << m_spin << endl; +#endif + if (m_params.verbose > AHFinder::MIN) + { + pout() << "irreducible mass = " << m_irreducible_mass << endl; +#if CH_SPACEDIM == 3 + pout() << "dimensionless spin vector = (" + << m_dimensionless_spin_vector[0] << ", " + << m_dimensionless_spin_vector[1] << ", " + << m_dimensionless_spin_vector[2] << ")" << endl; + pout() << "dimensionless spin in z (from equator-length " + "integral) = " + << m_spin_z_alt << endl; +#endif + } } #endif @@ -442,12 +467,6 @@ void ApparentHorizon::solve(double a_dt, m_std_F = 0.; } - // update center - // note that after this, the point that corresponds to the (u,v,r) - // coordinates is 'get_origin()' - if (m_params.track_center) - calculate_center(); - write_outputs(a_dt, a_time, a_restart_time); // break if 'AH_stop_if_max_fails == true' @@ -504,9 +523,9 @@ void ApparentHorizon::write_outputs( CH_assert(CH_SPACEDIM == 3 || CH_SPACEDIM == 2); // first '1' corresponds to 'step' - // area+spin+mass OR area+mass in 2D Cartoon OR only area for other - // cases (e.g. 4D -> 2D cartoon) - int stats = (GR_SPACEDIM == 3 ? (CH_SPACEDIM == 3 ? 3 : 2) : 1); + // area+mass+irred.mass+spin+spin_vec+spin_alt OR area+mass+irred.mass + // in 2D Cartoon OR only area for other cases (e.g. 4D -> 2D cartoon) + int stats = (GR_SPACEDIM == 3 ? (CH_SPACEDIM == 3 ? 8 : 3) : 1); std::vector values(1 + stats + CH_SPACEDIM * (1 + m_params.track_center)); @@ -518,10 +537,13 @@ void ApparentHorizon::write_outputs( values[idx++] = step; values[idx++] = m_area; #if GR_SPACEDIM == 3 // GR_SPACEDIM, not CH_SPACEDIM !!! + values[idx++] = m_mass; + values[idx++] = m_irreducible_mass; #if CH_SPACEDIM == 3 values[idx++] = m_spin; + FOR1(a) { values[idx++] = m_dimensionless_spin_vector[a]; } + values[idx++] = m_spin_z_alt; #endif - values[idx++] = m_mass; #endif for (int i = 0; i < CH_SPACEDIM; ++i) values[idx++] = origin[i]; @@ -539,10 +561,15 @@ void ApparentHorizon::write_outputs( headers[idx++] = "file"; headers[idx++] = "area"; #if GR_SPACEDIM == 3 // GR_SPACEDIM, not CH_SPACEDIM !!! + headers[idx++] = "mass"; + headers[idx++] = "irreducible mass"; #if CH_SPACEDIM == 3 headers[idx++] = "spin"; + headers[idx++] = "dimless spin-x"; + headers[idx++] = "dimless spin-y"; + headers[idx++] = "dimless spin-z"; + headers[idx++] = "dimless spin-z-alt"; #endif - headers[idx++] = "mass"; #endif headers[idx++] = "origin_x"; headers[idx++] = "origin_y"; @@ -801,11 +828,15 @@ void ApparentHorizon::restart( #if GR_SPACEDIM == 3 // GR_SPACEDIM, not CH_SPACEDIM !!! #if CH_SPACEDIM == 3 bool was_center_tracked = - (cols > 5 + CH_SPACEDIM); // 5 for time + file + area + spin + mass + (cols > + 10 + + CH_SPACEDIM); // 10 for time + file + area + mass + irreducible + // + spin + spin_vector + spin_z_alt #else // 3D -> 2D Cartoon method bool was_center_tracked = - (cols > 4 + CH_SPACEDIM); // 5 for time + file + area + mass + (cols > 5 + CH_SPACEDIM); // 5 for time + file + area + mass + + // irreducible mass #endif #else bool was_center_tracked = @@ -1307,6 +1338,9 @@ void ApparentHorizon::check_integration_methods() #if CH_SPACEDIM == 3 // ONLY FOR 3D +// OLD method to calculate the spin using the 1D equator length +// Adopted area integral 'calculate_angular_momentum_J' that allows to get the +// direction of the spin as well template double ApparentHorizon::calculate_spin_dimensionless( @@ -1407,13 +1441,167 @@ ApparentHorizon::calculate_spin_dimensionless( if (m_params.verbose > AHFinder::NONE) { - pout() << "dimensionless spin = " << spin_dimensionless << endl; + pout() << "dimensionless spin = " << spin_dimensionless << std::endl; } return spin_dimensionless; } #endif +#if CH_SPACEDIM == 3 +// ONLY FOR 3D +template +Tensor<1, double> +ApparentHorizon::calculate_angular_momentum_J() +{ + CH_assert(CH_SPACEDIM == 3); + CH_TIME("ApparentHorizon::calculate_angular_momentum_J"); + + if (!get_converged()) + return {NAN}; + + Tensor<1, double> J; + Tensor<1, double> integrals = {0.}; // temporary, but defined outside to be + // in scope for non-PETSc processes + + // PETSc processes go inside 'if', others "wait" until 'if' gets to + // 'm_interp.break_interpolation_loop()' + if (m_interp.keep_interpolating_if_inactive()) + { + + int idx = 0; + + Vec localF; + + DMGetLocalVector(m_dmda, &localF); + DMGlobalToLocalBegin(m_dmda, m_snes_soln, INSERT_VALUES, localF); + DMGlobalToLocalEnd(m_dmda, m_snes_soln, INSERT_VALUES, localF); + + dmda_arr_t in; + DMDAVecGetArray(m_dmda, localF, &in); + + // m_F is already set from solve + + const std::array ¢er = get_center(); + + for (int v = m_vmin; v < m_vmax; ++v) + { + Tensor<1, double> inner_integral = {0.}; + for (int u = m_umin; u < m_umax; ++u) + { + AHDeriv deriv = diff(in, u, v); + const auto geometric_data = m_interp.get_geometry_data(idx); + const auto data = m_interp.get_data(idx); + const auto coords = m_interp.get_coords(idx); + const auto coords_cart = m_interp.get_cartesian_coords(idx); + AHFunction func(data, coords, coords_cart); + Tensor<2, double> g = func.get_metric(); + Tensor<2, double> K = func.get_extrinsic_curvature(); + Tensor<1, double> s_L = + func.get_level_function_derivative(geometric_data, deriv); + Tensor<1, double> S_U = func.get_spatial_normal_U(s_L); + + Tensor<1, double> coords_cart_centered; + FOR1(i) + { + coords_cart_centered[i] = coords_cart[i] - center[i]; + } + + Tensor<1, double> spin_integrand = {0.}; + double directions[3][3] = { + {0, -coords_cart_centered[2], coords_cart_centered[1]}, + {coords_cart_centered[2], 0., -coords_cart_centered[0]}, + {-coords_cart_centered[1], coords_cart_centered[0], 0.}}; + FOR3(a, b, c) + { + // as in arXiv:gr-qc/0206008 eq. 25 + // but computing with 'killing vector' in directions 'x,y,z' + // ('x,y,z' killing vector are directions[c][a] = + // epsilon[c][j][a] * x[j], just like for ADM Momentum) + spin_integrand[c] += directions[c][a] * S_U[b] * K[a][b]; + } + + // now calculate sqrt(-g) area element + + // Calculate Jacobian matrix for transformation from Cartesian + // to (f,u,v) coords + Tensor<2, double> Jac; + FOR1(k) + { + Jac[0][k] = geometric_data.dxdf[k]; + Jac[1][k] = geometric_data.dxdu[k]; + Jac[2][k] = geometric_data.dxdv[k]; + } + + // Now do the coordinate transformation + Tensor<2, double> g_spherical = {0.}; + FOR4(i, j, k, l) + { + g_spherical[i][j] += Jac[i][k] * Jac[j][l] * g[k][l]; + } + + // Construct the 2-metric on the horizon in (u,v) coords + // i.e. substitute df = (df/du)du + (df/dv)dv + // into the spherical metric + Tensor<2, double, CH_SPACEDIM - 1> g_horizon = {0.}; + g_horizon[0][0] = g_spherical[1][1] + + g_spherical[0][0] * deriv.duF * deriv.duF + + 2.0 * g_spherical[0][1] * deriv.duF; + g_horizon[1][1] = g_spherical[2][2] + + g_spherical[0][0] * deriv.dvF * deriv.dvF + + 2.0 * g_spherical[0][2] * deriv.dvF; + g_horizon[0][1] = g_horizon[1][0] = + g_spherical[1][2] + + g_spherical[0][0] * deriv.duF * deriv.dvF + + g_spherical[0][1] * deriv.dvF + + g_spherical[0][2] * deriv.duF; + + double det = TensorAlgebra::compute_determinant(g_horizon); + + double weight = m_integration_methods[0].weight( + u, m_params.num_points_u, m_interp.is_u_periodic()); + + FOR1(a) + { + double element = spin_integrand[a] / (8. * M_PI) * + sqrt(det) * weight * m_du; + + // hack for the poles (theta=0,\pi) + // where nans appear in 'spin_integrand', but + // 'det' should be 0 + if (std::isnan(element)) + element = 0.; + + inner_integral[a] += element; + } + idx++; + } + double weight = m_integration_methods[1].weight( + v, m_params.num_points_v, m_interp.is_v_periodic()); + FOR1(a) { integrals[a] += weight * m_dv * inner_integral[a]; } + } + + DMDAVecRestoreArray(m_dmda, localF, &in); + DMRestoreLocalVector(m_dmda, &localF); + + m_interp.break_interpolation_loop(); + } + + // reduction across all Chombo processes (note that 'integral' is 0 for + // non-PETSc processes) because SmallDataIO uses rank 0 to write this + // ensures rank 0 will have 'integral' (even though for now the PETSc + // processes always include rank 0) +#ifdef CH_MPI + MPI_Allreduce(&integrals, &J, GR_SPACEDIM, MPI_DOUBLE, MPI_SUM, + Chombo_MPI::comm); +#else // serial + FOR1(a) { J[a] = integrals[a]; } +#endif + + return J; +} +#endif + template double ApparentHorizon::calculate_area() { @@ -1456,7 +1644,7 @@ double ApparentHorizon::calculate_area() const auto coords = m_interp.get_coords(idx); const auto coords_cart = m_interp.get_cartesian_coords(idx); AHFunction func(data, coords, coords_cart); - auto &g = func.get_metric(); + Tensor<2, double> g = func.get_metric(); // Calculate Jacobian matrix for transformation from Cartesian // to (f,u,v) coords @@ -1469,7 +1657,7 @@ double ApparentHorizon::calculate_area() Jac[2][k] = geometric_data.dxdv[k]; #endif } - // auto det = TensorAlgebra::compute_determinant(Jac); + AHDeriv deriv = diff(in, u #if CH_SPACEDIM == 3 , diff --git a/Source/ApparentHorizonFinder/ApparentHorizon_petsc.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon_petsc.impl.hpp index 74fb8faab..91585d733 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon_petsc.impl.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon_petsc.impl.hpp @@ -164,7 +164,7 @@ void ApparentHorizon::initialise_PETSc() &ksp_maxits); pout() << "-------------------------------------\n"; - pout() << "AHFinder Options:\n"; + pout() << "ApparentHorizon Options:\n"; pout() << "-------------------------------------\n"; pout() << "PETSc SNES Options:\n"; pout() << "Type: " << snes_type << "\n"; diff --git a/Source/InitialConditions/BlackHoles/KerrBH.hpp b/Source/InitialConditions/BlackHoles/KerrBH.hpp index bbc5e0c97..acd00dc18 100644 --- a/Source/InitialConditions/BlackHoles/KerrBH.hpp +++ b/Source/InitialConditions/BlackHoles/KerrBH.hpp @@ -30,6 +30,8 @@ class KerrBH double mass; //!<< The mass of the Kerr BH std::array center; //!< The center of the Kerr BH double spin; //!< The spin param a = J/M, so 0 <= |a| <= M + std::array spin_direction = { + 0., 0., 1.}; // default to 'z' axis; doesn't need to be normalized }; protected: @@ -60,7 +62,7 @@ class KerrBH Tensor<1, data_t> &spherical_shift, //!<< The spherical components of the shift data_t &kerr_lapse, //!<< The lapse for the kerr solution - const Coordinates coords //!<< Coords of current cell + const Tensor<1, data_t> &coords //!<< Coords of current cell ) const; }; diff --git a/Source/InitialConditions/BlackHoles/KerrBH.impl.hpp b/Source/InitialConditions/BlackHoles/KerrBH.impl.hpp index 769d5d6c1..a7b544308 100644 --- a/Source/InitialConditions/BlackHoles/KerrBH.impl.hpp +++ b/Source/InitialConditions/BlackHoles/KerrBH.impl.hpp @@ -16,6 +16,15 @@ // 2010, arxiv gr-qc/1001.4077 template void KerrBH::compute(Cell current_cell) const { + using namespace CoordinateTransformations; + using namespace TensorAlgebra; + + static const Tensor<1, double> z_dir = {0., 0., 1.}; + const Tensor<1, double> spin_dir = {m_params.spin_direction[0], + m_params.spin_direction[1], + m_params.spin_direction[2]}; + Tensor<2, double> R = rotation_matrix(z_dir, spin_dir); + // set up vars for the metric and extrinsic curvature, shift and lapse in // spherical coords Tensor<2, data_t> spherical_g; @@ -27,22 +36,32 @@ template void KerrBH::compute(Cell current_cell) const Vars vars; Coordinates coords(current_cell, m_dx, m_params.center); + // rotate point + Tensor<1, data_t> xyz = {coords.x, coords.y, coords.z}; + xyz = transform_vector(xyz, R); + // Compute the components in spherical coords as per 1401.1548 - compute_kerr(spherical_g, spherical_K, spherical_shift, kerr_lapse, coords); + compute_kerr(spherical_g, spherical_K, spherical_shift, kerr_lapse, xyz); // work out where we are on the grid - data_t x = coords.x; - double y = coords.y; - double z = coords.z; + data_t x = xyz[0]; + data_t y = xyz[1]; + data_t z = xyz[2]; - using namespace CoordinateTransformations; // Convert spherical components to cartesian components using coordinate - // transforms - vars.h = spherical_to_cartesian_LL(spherical_g, x, y, z); - vars.A = spherical_to_cartesian_LL(spherical_K, x, y, z); - vars.shift = spherical_to_cartesian_U(spherical_shift, x, y, z); + // transform_tensor_UU + Tensor<2, data_t> cartesian_h = + spherical_to_cartesian_LL(spherical_g, x, y, z); + Tensor<2, data_t> cartesian_K = + spherical_to_cartesian_LL(spherical_K, x, y, z); + Tensor<1, data_t> cartesian_shift = + spherical_to_cartesian_U(spherical_shift, x, y, z); + + // now rotate tensors back to original coordinates + vars.h = transform_tensor_LL(cartesian_h, R); + vars.A = transform_tensor_LL(cartesian_K, R); + vars.shift = transform_vector(cartesian_shift, R); - using namespace TensorAlgebra; // Convert to BSSN vars data_t deth = compute_determinant(vars.h); auto h_UU = compute_inverse_sym(vars.h); @@ -76,19 +95,22 @@ void KerrBH::compute_kerr(Tensor<2, data_t> &spherical_g, Tensor<2, data_t> &spherical_K, Tensor<1, data_t> &spherical_shift, data_t &kerr_lapse, - const Coordinates coords) const + const Tensor<1, data_t> &coords) const { // Kerr black hole params - mass M and spin a double M = m_params.mass; double a = m_params.spin; // work out where we are on the grid - data_t x = coords.x; - double y = coords.y; - double z = coords.z; + data_t x = coords[0]; + data_t y = coords[1]; + data_t z = coords[2]; // the radius, subject to a floor - data_t r = coords.get_radius(); + data_t r = sqrt(D_TERM(x * x, +y * y, +z * z)); + static const double minimum_r = 1e-6; + r = simd_max(r, minimum_r); + data_t r2 = r * r; // the radius in xy plane, subject to a floor diff --git a/Source/utils/CoordinateTransformations.hpp b/Source/utils/CoordinateTransformations.hpp index 1a346abf4..96574a88b 100644 --- a/Source/utils/CoordinateTransformations.hpp +++ b/Source/utils/CoordinateTransformations.hpp @@ -15,9 +15,9 @@ namespace CoordinateTransformations { // Jacobian transformation matrix -template -static Tensor<2, data_t> spherical_jacobian(const data_t x, const double y, - const double z) +template +Tensor<2, data_t, 3> spherical_jacobian(const data_t x, const data2_t y, + const data2_t z) { // calculate useful position quantities data_t rho2 = simd_max(x * x + y * y, 1e-12); @@ -30,7 +30,7 @@ static Tensor<2, data_t> spherical_jacobian(const data_t x, const double y, data_t sin_phi = y / rho; // derivatives for jacobian matrix - drdx etc - Tensor<2, data_t> jac; + Tensor<2, data_t, 3> jac; jac[0][0] = x / r; jac[1][0] = cos_phi * z / r2; jac[2][0] = -y / rho2; @@ -44,9 +44,9 @@ static Tensor<2, data_t> spherical_jacobian(const data_t x, const double y, } // Incerse Jacobian -template -static Tensor<2, data_t> -inverse_spherical_jacobian(const data_t x, const double y, const double z) +template +Tensor<2, data_t, 3> inverse_spherical_jacobian(const data_t x, const data2_t y, + const data2_t z) { // calculate useful position quantities data_t rho2 = simd_max(x * x + y * y, 1e-12); @@ -60,7 +60,7 @@ inverse_spherical_jacobian(const data_t x, const double y, const double z) data_t sin_phi = y / rho; // derivatives for inverse jacobian matrix - drdx etc - Tensor<2, data_t> inv_jac; + Tensor<2, data_t, 3> inv_jac; inv_jac[0][0] = x / r; inv_jac[1][0] = y / r; inv_jac[2][0] = z / r; @@ -73,196 +73,226 @@ inverse_spherical_jacobian(const data_t x, const double y, const double z) return inv_jac; } -// Convert a Tensor (with two lower indices) in spherical coords to cartesian -// coords -template -static Tensor<2, data_t> -spherical_to_cartesian_LL(const Tensor<2, data_t> &spherical_g, const data_t x, - const double y, const double z) +// transform a vector according to a given jacobian +// jacobian can be "inverse jacobian" in opposite transformation +template +Tensor<1, data_t, 3> transform_vector(const Tensor<1, data_t, 3> &vec_U, + const Tensor<2, data2_t, 3> &jacobian) { - Tensor<2, data_t> cartesian_g; + Tensor<1, data_t, 3> transformed_U; + FOR1(i) + { + transformed_U[i] = 0.0; + FOR1(j) { transformed_U[i] += jacobian[i][j] * vec_U[j]; } + } + return transformed_U; +} - // derivatives for jacobian matrix - drdx etc - Tensor<2, data_t> jac = spherical_jacobian(x, y, z); +// transform a covector according to a given jacobian +// jacobian can be "inverse jacobian" in opposite transformation +template +Tensor<1, data_t, 3> transform_covector(const Tensor<1, data_t, 3> &vec_L, + const Tensor<2, data2_t, 3> &jacobian) +{ + Tensor<1, data_t, 3> transformed_L; + FOR1(i) + { + transformed_L[i] = 0.0; + FOR1(j) { transformed_L[i] += vec_L[j] * jacobian[j][i]; } + } + return transformed_L; +} - // Convert the Tensor to cartesian coords +// transform a tensor with two raised indices according to a given jacobian +// jacobian can be "inverse jacobian" in opposite transformation +template +Tensor<2, data_t, 3> transform_tensor_UU(const Tensor<2, data_t, 3> &tensor_UU, + const Tensor<2, data2_t, 3> &jacobian) +{ + Tensor<2, data_t, 3> transformed_UU; FOR2(i, j) { - cartesian_g[i][j] = 0.; + transformed_UU[i][j] = 0.; FOR2(k, l) { - cartesian_g[i][j] += spherical_g[k][l] * jac[k][i] * jac[l][j]; + transformed_UU[i][j] += + jacobian[i][k] * jacobian[j][l] * tensor_UU[k][l]; } } - return cartesian_g; + return transformed_UU; } -// Convert a Tensor (with two upper indices) in spherical coords to cartesian -// coords -template -static Tensor<2, data_t> -spherical_to_cartesian_UU(const Tensor<2, data_t> &spherical_g_UU, - const data_t x, const double y, const double z) +// transform a tensor with two lowered indices according to a given jacobian +// jacobian can be "inverse jacobian" in opposite transformation +template +Tensor<2, data_t, 3> transform_tensor_LL(const Tensor<2, data_t, 3> &tensor_LL, + const Tensor<2, data2_t, 3> &jacobian) { - Tensor<2, data_t> cartesian_g_UU; - - // derivatives for jacobian matrix - drdx etc - Tensor<2, data_t> inv_jac = inverse_spherical_jacobian(x, y, z); - - // Convert the Tensor to cartesian coords + Tensor<2, data_t, 3> transformed_LL; FOR2(i, j) { - cartesian_g_UU[i][j] = 0.; + transformed_LL[i][j] = 0.; FOR2(k, l) { - cartesian_g_UU[i][j] += - spherical_g_UU[k][l] * inv_jac[i][k] * inv_jac[j][l]; + transformed_LL[i][j] += + tensor_LL[k][l] * jacobian[k][i] * jacobian[l][j]; } } - return cartesian_g_UU; + return transformed_LL; } -// Convert a Tensor (with two lower indices) in cartesian coords to spherical +// Convert a Tensor (with two lower indices) in spherical coords to cartesian // coords -template -static Tensor<2, data_t> -cartesian_to_spherical_LL(const Tensor<2, data_t> &cartesian_g, const data_t x, - const double y, const double z) +template +Tensor<2, data_t, 3> +spherical_to_cartesian_LL(const Tensor<2, data_t, 3> &spherical_g, + const data_t x, const data2_t y, const data2_t z) { - Tensor<2, data_t> spherical_g; + return transform_tensor_LL(spherical_g, spherical_jacobian(x, y, z)); +} - // derivatives for inverse jacobian matrix - drdx etc - Tensor<2, data_t> inv_jac = inverse_spherical_jacobian(x, y, z); +// Convert a Tensor (with two upper indices) in spherical coords to cartesian +// coords +template +Tensor<2, data_t, 3> +spherical_to_cartesian_UU(const Tensor<2, data_t, 3> &spherical_g_UU, + const data_t x, const data2_t y, const data2_t z) +{ + return transform_tensor_UU(spherical_g_UU, + inverse_spherical_jacobian(x, y, z)); +} - // Convert the Tensor to spherical coords - FOR2(i, j) - { - spherical_g[i][j] = 0.; - FOR2(k, l) - { - spherical_g[i][j] += - cartesian_g[k][l] * inv_jac[k][i] * inv_jac[l][j]; - } - } - return spherical_g; +// Convert a Tensor (with two lower indices) in cartesian coords to spherical +// coords +template +Tensor<2, data_t, 3> +cartesian_to_spherical_LL(const Tensor<2, data_t, 3> &cartesian_g, + const data_t x, const data2_t y, const data2_t z) +{ + return transform_tensor_LL(cartesian_g, + inverse_spherical_jacobian(x, y, z)); } // Convert a Tensor (with two upper indices) in cartesian coords to spherical // coords -template -static Tensor<2, data_t> -cartesian_to_spherical_UU(const Tensor<2, data_t> &cartesian_g_UU, data_t x, - double y, double z) +template +Tensor<2, data_t, 3> +cartesian_to_spherical_UU(const Tensor<2, data_t, 3> &cartesian_g_UU, data_t x, + data2_t y, data2_t z) { - Tensor<2, data_t> spherical_g_UU; - - // derivatives for jacobian matrix - drdx etc - Tensor<2, data_t> jac = spherical_jacobian(x, y, z); - - // Convert the Tensor to spherical coords - FOR2(i, j) - { - spherical_g_UU[i][j] = 0.; - FOR2(k, l) - { - spherical_g_UU[i][j] += - cartesian_g_UU[k][l] * jac[i][k] * jac[j][l]; - } - } - return spherical_g_UU; + return transform_tensor_UU(cartesian_g_UU, spherical_jacobian(x, y, z)); } // Convert a vector (with one upper index) in spherical coords to cartesian // coords -template -Tensor<1, data_t> -spherical_to_cartesian_U(const Tensor<1, data_t> &spherical_v_U, data_t x, - double y, double z) +template +Tensor<1, data_t, 3> +spherical_to_cartesian_U(const Tensor<1, data_t, 3> &spherical_v_U, data_t x, + data2_t y, data2_t z) { - Tensor<1, data_t> cartesian_v_U; - - // derivatives for inverse jacobian matrix - drdx etc - Tensor<2, data_t> inv_jac = inverse_spherical_jacobian(x, y, z); - - // transform the vector to cartesian coords - FOR1(i) - { - cartesian_v_U[i] = 0.0; - FOR1(j) { cartesian_v_U[i] += inv_jac[i][j] * spherical_v_U[j]; } - } - return cartesian_v_U; + return transform_vector(spherical_v_U, inverse_spherical_jacobian(x, y, z)); } // Convert a vector (with one lower index) in spherical coords to cartesian // coords -template -Tensor<1, data_t> -spherical_to_cartesian_L(const Tensor<1, data_t> &spherical_v_L, data_t x, - double y, double z) +template +Tensor<1, data_t, 3> +spherical_to_cartesian_L(const Tensor<1, data_t, 3> &spherical_v_L, data_t x, + data2_t y, data2_t z) { - Tensor<1, data_t> cartesian_v_L; - - // derivatives for jacobian matrix - drdx etc - Tensor<2, data_t> jac = spherical_jacobian(x, y, z); - - // transform the vector to cartesian coords - FOR1(i) - { - cartesian_v_L[i] = 0.0; - FOR1(j) { cartesian_v_L[i] += spherical_v_L[j] * jac[j][i]; } - } - return cartesian_v_L; + return transform_covector(spherical_v_L, spherical_jacobian(x, y, z)); } // Convert a vector (with one upper index) in cartesian coords to spherical // coords -template -Tensor<1, data_t> -cartesian_to_spherical_U(const Tensor<1, data_t> &cartesian_v_U, data_t x, - double y, double z) +template +Tensor<1, data_t, 3> +cartesian_to_spherical_U(const Tensor<1, data_t, 3> &cartesian_v_U, data_t x, + data2_t y, data2_t z) { - Tensor<1, data_t> spherical_v_U; - - // derivatives for jacobian matrix - drdx etc - Tensor<2, data_t> jac = spherical_jacobian(x, y, z); - - // transform the vector to cartesian coords - FOR1(i) - { - spherical_v_U[i] = 0.0; - FOR1(j) { spherical_v_U[i] += jac[i][j] * cartesian_v_U[j]; } - } - return spherical_v_U; + return transform_vector(cartesian_v_U, spherical_jacobian(x, y, z)); } // Convert a vector (with one lower index) in cartesian coords to spherical // coords -template -Tensor<1, data_t> -cartesian_to_spherical_L(const Tensor<1, data_t> &cartesian_v_L, data_t x, - double y, double z) +template +Tensor<1, data_t, 3> +cartesian_to_spherical_L(const Tensor<1, data_t, 3> &cartesian_v_L, data_t x, + data2_t y, data2_t z) { - Tensor<1, data_t> spherical_v_L; - - // derivatives for inverse jacobian matrix - drdx etc - Tensor<2, data_t> inv_jac = inverse_spherical_jacobian(x, y, z); - - // transform the vector to cartesian coords - FOR1(i) - { - spherical_v_L[i] = 0.0; - FOR1(j) { spherical_v_L[i] += cartesian_v_L[j] * inv_jac[j][i]; } - } - return spherical_v_L; + return transform_covector(cartesian_v_L, + inverse_spherical_jacobian(x, y, z)); } // The area element of a sphere template -data_t area_element_sphere(const Tensor<2, data_t> &spherical_g) +data_t area_element_sphere(const Tensor<2, data_t, 3> &spherical_g) { return sqrt(spherical_g[1][1] * spherical_g[2][2] - spherical_g[1][2] * spherical_g[2][1]); } +// cartesian coordinates rotation matrix +// axis must be normalized to 1 +template +Tensor<2, data_t, 3> rotation_matrix(const Tensor<1, data_t, 3> &axis, + data_t cos_angle) +{ + data_t sine = sqrt(1. - cos_angle * cos_angle); + data_t one_minus_cos = 1. - cos_angle; + + // as in + // https://en.wikipedia.org/wiki/Rotation_matrix#Rotation_matrix_from_axis_and_angle + Tensor<2, data_t, 3> R; + R[0][0] = axis[0] * axis[0] * one_minus_cos + cos_angle; + R[0][1] = axis[1] * axis[0] * one_minus_cos + axis[2] * sine; + R[0][2] = axis[2] * axis[0] * one_minus_cos - axis[1] * sine; + R[1][0] = axis[0] * axis[1] * one_minus_cos - axis[2] * sine; + R[1][1] = axis[1] * axis[1] * one_minus_cos + cos_angle; + R[1][2] = axis[2] * axis[1] * one_minus_cos + axis[0] * sine; + R[2][0] = axis[0] * axis[2] * one_minus_cos + axis[1] * sine; + R[2][1] = axis[1] * axis[2] * one_minus_cos - axis[0] * sine; + R[2][2] = axis[2] * axis[2] * one_minus_cos + cos_angle; + + return R; +} + +// cartesian coordinates rotation matrix +template +Tensor<2, data_t, 3> rotation_matrix(const Tensor<1, data_t, 3> &origin, + const Tensor<1, data_t, 3> &destination) +{ + // Calculate axis of rotation: + Tensor<1, data_t, 3> axis = { + origin[1] * destination[2] - origin[2] * destination[1], + origin[2] * destination[0] - origin[0] * destination[2], + origin[0] * destination[1] - origin[1] * destination[0]}; + + static const double eps = 1.e-13; + if (std::abs(axis[0]) < eps && std::abs(axis[1]) < eps && + std::abs(axis[2]) < eps) + return rotation_matrix({0., 0., 0.}, 1.); + + data_t norm = + sqrt(axis[0] * axis[0] + axis[1] * axis[1] + axis[2] * axis[2]); + + axis[0] /= norm; + axis[1] /= norm; + axis[2] /= norm; + + // Calculate angle of rotation + data_t norm_origin = sqrt(origin[0] * origin[0] + origin[1] * origin[1] + + origin[2] * origin[2]); + data_t norm_dest = + sqrt(destination[0] * destination[0] + destination[1] * destination[1] + + destination[2] * destination[2]); + + data_t cos_angle = origin[0] * destination[0] + origin[1] * destination[1] + + origin[2] * destination[2]; + cos_angle /= (norm_origin * norm_dest); + + return rotation_matrix(axis, cos_angle); +} + } // namespace CoordinateTransformations #endif /* COORDINATETRANSFORMATIONS_HPP_ */ diff --git a/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.cpp b/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.cpp index 0f1bf27f1..43df00de3 100755 --- a/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.cpp +++ b/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.cpp @@ -84,14 +84,19 @@ int runApparentHorizonTest3D(int argc, char *argv[]) status = 1; else { - double spin = stats[3][0]; - double mass = stats[4][0]; + double mass = stats[3][0]; + double spin = stats[5][0]; + double spin_x = stats[6][0] * mass; double error_perc = fabs(1. - mass / sim_params.kerr_params.mass) * 100; if (sim_params.kerr_params.spin != 0.) + { error_perc += fabs(1. - spin / sim_params.kerr_params.spin) * 100; + error_perc += + fabs(1. - spin_x / sim_params.kerr_params.spin) * 100; + } pout() << "error = " << error_perc << "%" << std::endl; status |= (error_perc > 0.1) || +std::isnan( diff --git a/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.inputs b/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.inputs index a374c32cf..422bd65d5 100755 --- a/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.inputs +++ b/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.inputs @@ -4,14 +4,12 @@ verbosity = 0 N_full = 32 L_full = 16 -num_ghosts = 3 -max_level = 1 -regrid_interval = 1 1 1 1 0 0 0 0 0 -ref_ratio = 2 2 2 2 2 2 2 2 2 +max_level = 0 +regrid_interval = 0 isPeriodic = 0 0 0 hi_boundary = 1 1 1 -lo_boundary = 1 1 2 +lo_boundary = 2 1 1 vars_parity = 0 0 4 6 0 5 0 #chi and hij 0 0 4 6 0 5 0 #K and Aij @@ -21,16 +19,15 @@ vars_parity = 0 0 4 6 0 5 0 #chi and hij vars_asymptotic_values = 1. 1. 0. 0. 1. 0. 1. #chi and hij 0. 0. 0. 0. 0. 0. 0. #K and Aij - 0. 0. 0. 0. #Theta and Gamma + 0. 0. 0. 0. #Theta and Gamma 1. 0. 0. 0. 0. 0. 0. #lapse shift and B 0. 0. 0. 0. # Ham and Mom #Max and min box sizes max_grid_size = 32 -block_factor = 4 -tag_buffer_size = 3 -checkpoint_interval = 1 +block_factor = 16 kerr_mass = 10. kerr_spin = 5. +kerr_spin_direction = 1. 0. 0. #initial_guess = 5 diff --git a/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3DLevel.hpp b/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3DLevel.hpp index 30fb581de..f86247e3b 100755 --- a/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3DLevel.hpp +++ b/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3DLevel.hpp @@ -80,6 +80,7 @@ class ApparentHorizonTest3DLevel : public GRAMRLevel make_compute_pack(SetValue(0.), KerrBH(m_p.kerr_params, m_dx)), m_state_new, m_state_new, INCLUDE_GHOST_CELLS); + // Gamma's are not needed // fillAllGhosts(); // BoxLoops::loop(GammaCalculator(m_dx), m_state_new, m_state_new, // EXCLUDE_GHOST_CELLS); @@ -91,7 +92,10 @@ class ApparentHorizonTest3DLevel : public GRAMRLevel } virtual void computeTaggingCriterion(FArrayBox &tagging_criterion, - const FArrayBox ¤t_state){}; + const FArrayBox ¤t_state) + { + tagging_criterion.setVal(0.); + }; }; #endif /* APPARENT_HORIZON_TEST3DLEVEL_HPP_ */ diff --git a/Tests/ApparentHorizonFinderTest3D/SimulationParameters.hpp b/Tests/ApparentHorizonFinderTest3D/SimulationParameters.hpp index 34e6a3373..9d75d4708 100755 --- a/Tests/ApparentHorizonFinderTest3D/SimulationParameters.hpp +++ b/Tests/ApparentHorizonFinderTest3D/SimulationParameters.hpp @@ -26,6 +26,8 @@ class SimulationParameters : public ChomboParameters pp.load("kerr_mass", kerr_params.mass); pp.load("kerr_spin", kerr_params.spin); pp.load("kerr_center", kerr_params.center, center); + pp.load("kerr_spin_direction", kerr_params.spin_direction, + {0., 0., 1.}); #ifdef USE_AHFINDER pp.load("initial_guess", initial_guess, kerr_params.mass * 0.5); From 84d3aa501769486ea53e09f0b3ccbb59ce499272 Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Mon, 1 Mar 2021 16:28:07 +0000 Subject: [PATCH 10/92] Allow setting punctures as AHs origins --- Examples/BinaryBH/BinaryBHLevel.cpp | 7 + Examples/BinaryBH/SimulationParameters.hpp | 3 + Examples/BinaryBH/params.txt | 41 ++--- Examples/BinaryBH/params_two_punctures.txt | 56 +++++-- .../BinaryBH/params_two_punctures_cheap.txt | 148 ++++++++++++++++++ Examples/BinaryBH/params_very_cheap.txt | 9 +- Source/ApparentHorizonFinder/AHFinder.cpp | 47 +++++- Source/ApparentHorizonFinder/AHFinder.hpp | 56 +++++-- Source/ApparentHorizonFinder/AHFunctions.hpp | 1 + .../ApparentHorizonFinder/ApparentHorizon.hpp | 4 + .../ApparentHorizon.impl.hpp | 43 +++-- 11 files changed, 355 insertions(+), 60 deletions(-) create mode 100644 Examples/BinaryBH/params_two_punctures_cheap.txt diff --git a/Examples/BinaryBH/BinaryBHLevel.cpp b/Examples/BinaryBH/BinaryBHLevel.cpp index e6c25a1ed..d8fb5243f 100644 --- a/Examples/BinaryBH/BinaryBHLevel.cpp +++ b/Examples/BinaryBH/BinaryBHLevel.cpp @@ -204,7 +204,14 @@ void BinaryBHLevel::specificPostTimeStep() #ifdef USE_AHFINDER if (m_p.AH_activate && m_level == m_p.AH_params.level_to_run) + { + if (m_p.AH_set_origins_to_punctures && m_p.track_punctures) + { + m_bh_amr.m_ah_finder.set_origins( + m_bh_amr.m_puncture_tracker.get_puncture_coords()); + } m_bh_amr.m_ah_finder.solve(m_dt, m_time, m_restart_time); + } #endif } diff --git a/Examples/BinaryBH/SimulationParameters.hpp b/Examples/BinaryBH/SimulationParameters.hpp index 4112c12b0..9a414ddb1 100644 --- a/Examples/BinaryBH/SimulationParameters.hpp +++ b/Examples/BinaryBH/SimulationParameters.hpp @@ -47,6 +47,8 @@ class SimulationParameters : public SimulationParametersBase 0.5 * bh1_params.mass); pp.load("AH_2_initial_guess", AH_2_initial_guess, 0.5 * bh2_params.mass); + pp.load("AH_set_origins_to_punctures", AH_set_origins_to_punctures, + false); #endif } @@ -320,6 +322,7 @@ class SimulationParameters : public SimulationParametersBase #ifdef USE_AHFINDER double AH_1_initial_guess; double AH_2_initial_guess; + bool AH_set_origins_to_punctures; #endif }; diff --git a/Examples/BinaryBH/params.txt b/Examples/BinaryBH/params.txt index cffe815e9..afb055a1a 100644 --- a/Examples/BinaryBH/params.txt +++ b/Examples/BinaryBH/params.txt @@ -16,16 +16,16 @@ plot_prefix = BinaryBHPlot_ # NB - if you have a non-cubic grid, you can specify 'N1' or 'N1_full', # 'N2' or 'N2_full' and 'N3' or 'N3_full' ( then dx_coarsest = L/N(max) ) # NB - the N values need to be multiples of the block_factor -N_full = 64 +N_full = 128 L_full = 512 -# Spatial derivative order (only affects CCZ4 RHS) -max_spatial_derivative_order = 4 # can be 4 or 6 - # Defaults to center of grid L/2 # uncomment to change #center = 256.0 256.0 256.0 +# Spatial derivative order (only affects CCZ4 RHS) +max_spatial_derivative_order = 6 # can be 4 or 6 + # BH params, use puncture tracking to # ensure horizons resolved track_punctures = 1 @@ -39,18 +39,18 @@ momentumB = 0.0841746 0.000510846 0.0 # regridding control, specify threshold (same on each level) regrid_threshold = 0.05 -max_level = 9 +max_level = 8 # need max_level entries for regrid interval -regrid_interval = 0 0 0 64 64 64 64 64 64 +regrid_interval = 0 1 0 0 0 0 0 0 0 # Max and min box sizes -max_grid_size = 16 +max_grid_size = 32 block_factor = 16 # Set up time steps # dt will be dx*dt_multiplier on each grid level # HDF5files are written every dt = L/N*dt_multiplier*checkpoint_interval -checkpoint_interval = 100 +checkpoint_interval = 50 # set to zero to turn off plot files, comps defined in BinaryBHLevel.cpp plot_interval = 10 num_plot_vars = 3 @@ -97,6 +97,7 @@ shift_Gamma_coeff = 0.75 eta = 1.0 # CCZ4 parameters +formulation = 0 # 1 for BSSN, 0 for CCZ4 kappa1 = 0.1 kappa2 = 0 kappa3 = 1.0 @@ -109,11 +110,11 @@ sigma = 1.0 # default of extraction_center is center, uncomment to change #extraction_center = 256 256 256 activate_extraction = 1 -num_extraction_radii = 2 -extraction_radii = 50.0 100.0 -extraction_levels = 2 1 -num_points_phi = 24 -num_points_theta = 36 +num_extraction_radii = 3 +extraction_radii = 50.0 75.0 100.0 +extraction_levels = 2 1 1 +num_points_phi = 32 +num_points_theta = 48 num_modes = 8 modes = 2 0 # l m for spherical harmonics 2 1 @@ -127,15 +128,13 @@ modes = 2 0 # l m for spherical harmonics #Apparent Horizon finder AH_activate = 1 -AH_num_ranks = 20 -AH_num_points_u = 30 -AH_num_points_v = 50 +AH_num_ranks = 65 +AH_num_points_u = 65 +AH_num_points_v = 48 #AH_solve_interval = 1 #AH_print_interval = 1 #AH_track_center = true #AH_predict_origin = true -#AH_merger_search_factor = 1. -#AH_merger_pre_factor = 1. #AH_level_to_run = 0 #AH_start_time = 0. #AH_give_up_time = -1. @@ -151,5 +150,7 @@ AH_num_points_v = 50 #AH_1_initial_guess = 0.3 #AH_2_initial_guess = 0.3 -AH_num_extra_vars = 2 -AH_extra_vars = chi d1_Ham d2_A11 +#AH_num_extra_vars = 2 +#AH_extra_vars = chi d1_Ham d2_A11 + +AH_set_origins_to_punctures = 1 \ No newline at end of file diff --git a/Examples/BinaryBH/params_two_punctures.txt b/Examples/BinaryBH/params_two_punctures.txt index 1f5f239bb..7a2db8103 100644 --- a/Examples/BinaryBH/params_two_punctures.txt +++ b/Examples/BinaryBH/params_two_punctures.txt @@ -14,13 +14,16 @@ plot_prefix = BinaryBHPlot_ # NB - if you have a non-cubic grid, you can specify 'N1' or 'N1_full', # 'N2' or 'N2_full' and 'N3' or 'N3_full' ( then dx_coarsest = L/N(max) ) # NB - the N values need to be multiples of the block_factor -N_full = 64 +N_full = 128 L_full = 512 # Defaults to center of grid L/2 # uncomment to change #center = 256.0 256.0 256.0 +# Spatial derivative order (only affects CCZ4 RHS) +max_spatial_derivative_order = 6 # can be 4 or 6 + # BH params, use puncture tracking to # ensure horizons resolved track_punctures = 1 @@ -28,18 +31,18 @@ puncture_tracking_level = 5 # regridding control, specify threshold (same on each level) regrid_threshold = 0.05 -max_level = 9 +max_level = 8 # need max_level entries for regrid interval -regrid_interval = 0 0 0 64 64 64 64 64 64 +regrid_interval = 0 1 0 0 0 0 0 0 0 # Max and min box sizes -max_grid_size = 16 +max_grid_size = 32 block_factor = 16 # Set up time steps # dt will be dx*dt_multiplier on each grid level # HDF5files are written every dt = L/N*dt_multiplier*checkpoint_interval -checkpoint_interval = 100 +checkpoint_interval = 50 # set to zero to turn off plot files, comps defined in BinaryBHLevel.cpp plot_interval = 10 num_plot_vars = 3 @@ -86,10 +89,11 @@ shift_Gamma_coeff = 0.75 eta = 1.0 # CCZ4 parameters +formulation = 0 # 1 for BSSN, 0 for CCZ4 kappa1 = 0.1 kappa2 = 0 kappa3 = 1.0 -covariantZ4 = 1 # 0: default. 1: dampk1 -> dampk1/lapse +covariantZ4 = 1 # 0: keep kappa1; 1 [default]: replace kappa1 -> kappa1/lapse # coefficient for KO numerical dissipation sigma = 1.0 @@ -98,11 +102,11 @@ sigma = 1.0 # default of extraction_center is center, uncomment to change #extraction_center = 256 256 256 activate_extraction = 1 -num_extraction_radii = 2 -extraction_radii = 50.0 100.0 -extraction_levels = 2 1 -num_points_phi = 24 -num_points_theta = 36 +num_extraction_radii = 3 +extraction_radii = 50.0 75.0 100.0 +extraction_levels = 2 1 1 +num_points_phi = 32 +num_points_theta = 48 num_modes = 8 modes = 2 0 # l m for spherical harmonics 2 1 @@ -114,6 +118,36 @@ modes = 2 0 # l m for spherical harmonics 4 4 +#Apparent Horizon finder +AH_activate = 1 +AH_num_ranks = 65 +AH_num_points_u = 65 +AH_num_points_v = 48 +#AH_solve_interval = 1 +#AH_print_interval = 1 +#AH_track_center = true +#AH_predict_origin = true +#AH_level_to_run = 0 +#AH_start_time = 0. +#AH_give_up_time = -1. +#AH_merger_search_factor = 1. +#AH_merger_pre_factor = 1. +#AH_allow_re_attempt = 0 +#AH_max_fails_after_lost = -1 +#AH_verbose = 1 +#AH_print_geometry_data = 0 +#AH_re_solve_at_restart = 0 +#AH_stop_if_max_fails = 0 + +#AH_1_initial_guess = 0.3 +#AH_2_initial_guess = 0.3 + +#AH_num_extra_vars = 2 +#AH_extra_vars = chi d1_Ham d2_A11 + +AH_set_origins_to_punctures = 1 + + ### Two Punctures params # Main BH params diff --git a/Examples/BinaryBH/params_two_punctures_cheap.txt b/Examples/BinaryBH/params_two_punctures_cheap.txt new file mode 100644 index 000000000..e9e5af4cd --- /dev/null +++ b/Examples/BinaryBH/params_two_punctures_cheap.txt @@ -0,0 +1,148 @@ +verbosity = 0 +chk_prefix = BinaryBH_ +plot_prefix = BinaryBHPlot_ +#restart_file = BinaryBH_000065.3d.hdf5 + +# Set up grid spacings and regrid params +# NB - the N values need to be multiples of block_factor + +# Spatial derivative order (only affects CCZ4 RHS) +max_spatial_derivative_order = 4 # can be 4 or 6 + +N_full = 32 +L_full = 16 + +regrid_threshold = 0.1 +max_level = 3 +regrid_interval = 1 1 1 1 0 0 0 0 0 + +#Max and min box sizes +max_grid_size = 32 +block_factor = 8 +tag_buffer_size = 3 + +# Set up time steps +# dt will be dx*dt_multiplier on each grid level +# HDF5files are written every dt = L/N*dt_multiplier*checkpoint_interval + +checkpoint_interval = 1 +# set to zero to turn off plot files +plot_interval = 2 +num_plot_vars = 4 +plot_vars = chi Ham Weyl4_Re Weyl4_Im +dt_multiplier = 0.25 +stop_time = 15.0 + +nan_check = 1 + +#Lapse evolution +lapse_advec_coeff = 0.0 +lapse_coeff = 2.0 +lapse_power = 1.0 + +# Shift evolution +shift_advec_coeff = 0.0 +shift_Gamma_coeff = 0.75 +eta = 1.82 + +# CCZ4 parameters +kappa1 = 0.1 +kappa2 = 0 +kappa3 = 1. +covariantZ4 = 1 # 0: keep kappa1; 1 [default]: replace kappa1 -> kappa1/lapse + +#coefficient for KO numerical dissipation +sigma = 0.3 + +# boundaries and periodicity of grid +# Periodic directions - 0 = false, 1 = true +isPeriodic = 0 0 0 +# if not periodic, then specify the boundary type +# 0 = static, 1 = sommerfeld, 2 = reflective +# (see BoundaryConditions.hpp for details) +hi_boundary = 1 1 1 +lo_boundary = 1 1 2 + +# if reflective boundaries selected, must set +# parity of all vars (in order given by UserVariables.hpp) +# 0 = even +# 1,2,3 = odd x, y, z +# 4,5,6 = odd xy, yz, xz +# 7 = odd xyz +vars_parity = 0 0 4 6 0 5 0 #chi and hij + 0 0 4 6 0 5 0 #K and Aij + 0 1 2 3 #Theta and Gamma + 0 1 2 3 1 2 3 #lapse shift and B +vars_parity_diagnostic = 0 1 2 3 #Ham and Mom + 0 7 #Weyl + +# if sommerfeld boundaries selected, must select +# non zero asymptotic values +num_nonzero_asymptotic_vars = 5 +nonzero_asymptotic_vars = chi h11 h22 h33 lapse +nonzero_asymptotic_values = 1.0 1.0 1.0 1.0 1.0 + +track_punctures = 1 +puncture_tracking_level = 1 + +# extraction params +# default of extraction_center is center, uncomment to change +activate_extraction = 1 +num_extraction_radii = 1 +extraction_radii = 6.0 +extraction_levels = 0 +num_points_phi = 24 +num_points_theta = 36 +num_modes = 3 +modes = 2 0 # l m for spherical harmonics + 2 1 + 2 2 + +#Apparent Horizon finder +AH_activate = 1 +AH_num_ranks = 4 +AH_num_points_u = 11 +AH_num_points_v = 10 +AH_solve_interval = 2 +AH_print_interval = 2 +#AH_merger_search_factor = 1. +#AH_merger_pre_factor = 1. + +AH_set_origins_to_punctures = 1 + + +### Two Punctures params + +# Main BH params +# Either calculate target masses or set bare masses explicitly below +TP_calculate_target_masses = true +TP_target_mass_plus = 0.5 +TP_target_mass_minus = 0.5 +#TP_adm_tol = 1e-10 +# offset in x direction (or z if TP_swap_xz set true) +TP_offset_plus = 2. +TP_offset_minus = -2. +#TP_swap_xz = false +TP_momentum_plus = -0.2 0.0 0.0 +TP_momentum_minus = 0.2 0.0 0.0 +TP_spin_plus = 0.0 0.0 0.0 +TP_spin_minus = 0.0 0.0 0.0 + +# Solver params +#TP_npoints_A = 30 +#TP_npoints_B = 30 +#TP_npoints_phi = 16; +#TP_Newton_tol = 1e-10 +#TP_Newton_maxit = 5 +TP_epsilon = 1e-6 +#TP_Tiny = 0.0 +#TP_Extend_Radius = 0.0 + +# Initial data params +TP_use_spectral_interpolation = true +TP_initial_lapse = psi^n +TP_initial_lapse_psi_exponent = -2.0 + +# Debug output +#TP_do_residuum_debug_output = false +#TP_do_initial_debug_output = false diff --git a/Examples/BinaryBH/params_very_cheap.txt b/Examples/BinaryBH/params_very_cheap.txt index 6199aca9e..a208b7a6f 100644 --- a/Examples/BinaryBH/params_very_cheap.txt +++ b/Examples/BinaryBH/params_very_cheap.txt @@ -21,8 +21,8 @@ offsetB = 2 0 0 momentumA = 0. -0.14025 0.0 momentumB = 0. 0.14025 0.0 -regrid_threshold = 0.3 -max_level = 1 +regrid_threshold = 0.1 +max_level = 3 regrid_interval = 1 1 1 1 0 0 0 0 0 #Max and min box sizes @@ -40,7 +40,7 @@ plot_interval = 2 num_plot_vars = 4 plot_vars = chi Ham Weyl4_Re Weyl4_Im dt_multiplier = 0.25 -stop_time = 2.0 +stop_time = 10.0 nan_check = 1 @@ -114,5 +114,6 @@ AH_num_points_u = 11 AH_num_points_v = 10 AH_solve_interval = 2 AH_print_interval = 2 -#AH_merger_search_factor = 1 +#AH_merger_search_factor = 1. +#AH_merger_pre_factor = 1. diff --git a/Source/ApparentHorizonFinder/AHFinder.cpp b/Source/ApparentHorizonFinder/AHFinder.cpp index cadc737da..46a76cb61 100644 --- a/Source/ApparentHorizonFinder/AHFinder.cpp +++ b/Source/ApparentHorizonFinder/AHFinder.cpp @@ -254,7 +254,7 @@ bool AHFinder::solve_merger(int ah1, int ah2, double &initial_guess_merger, // ensure we don't catch the inner AH double merger_pre_factor = std::max(AH1->m_params.merger_pre_factor, AH2->m_params.merger_pre_factor); - initial_guess_merger = merger_pre_factor * initial_guess_sum; + initial_guess_merger = merger_pre_factor * 4. * initial_guess_sum; // update center of merged, otherwise it does it by // itself in solve @@ -279,7 +279,7 @@ bool AHFinder::solve_merger(int ah1, int ah2, double &initial_guess_merger, double merger_search_factor = std::max(AH1->m_params.merger_search_factor, AH2->m_params.merger_search_factor); - double min_distance = merger_search_factor * 4.0 * initial_guess_sum; + double min_distance = merger_search_factor * 4. * initial_guess_sum; bool do_solve = false; @@ -288,8 +288,9 @@ bool AHFinder::solve_merger(int ah1, int ah2, double &initial_guess_merger, do_solve = merger_search_factor <= 0. || distance <= min_distance; if (do_solve) - // radius of guess is bigger than AH distance - CH_assert(min_distance < initial_guess_merger); + // make sure twice the radius of the guess is bigger than AH + // distance + CH_assert(min_distance <= 2. * initial_guess_merger); if (AH1->m_params.verbose > NONE) { @@ -307,6 +308,44 @@ bool AHFinder::solve_merger(int ah1, int ah2, double &initial_guess_merger, return do_solve; } +void AHFinder::set_origins( + const std::vector> &origins, + bool includes_mergers) +{ + const int num_ah = m_apparent_horizons.size(); + + int non_merger_counter = 0; // for 'includes_mergers == false' + for (int i = 0; i < num_ah; ++i) + { + if (includes_mergers) + { + m_apparent_horizons[i]->set_origin(origins[i]); + } + else + { + // if it is a merger that has not yet been found + if (m_merger_pairs[i].first >= 0) + { + if (!m_apparent_horizons[i]->has_been_found()) + { + auto origin1 = m_apparent_horizons[m_merger_pairs[i].first] + ->get_origin(); + auto origin2 = m_apparent_horizons[m_merger_pairs[i].second] + ->get_origin(); + std::array origin; + FOR1(i) { origin[i] = (origin1[i] + origin2[i]) / 2.; } + m_apparent_horizons[i]->set_origin(origin); + } + } + else + { + m_apparent_horizons[i]->set_origin(origins[non_merger_counter]); + ++non_merger_counter; + } + } + } +} + void AHFinder::params::read_params(GRParmParse &pp, const ChomboParameters &a_p) { pp.load("AH_num_ranks", num_ranks, 0); // 0 means "all" diff --git a/Source/ApparentHorizonFinder/AHFinder.hpp b/Source/ApparentHorizonFinder/AHFinder.hpp index 6c202a41a..bff080ad8 100644 --- a/Source/ApparentHorizonFinder/AHFinder.hpp +++ b/Source/ApparentHorizonFinder/AHFinder.hpp @@ -48,6 +48,38 @@ template class AHInterpolation; template class ApparentHorizon; +/* +TF: +A general note on the radius and the value of chi on the AH surface. +For a Kerr BH initial data with dimensionless spin 's' and mass 'M': + r_AH = M / 4 (1 + sqrt{1 - s^2}) + chi_AH = [(1-s^2)^(1/6) / 16 * ((1 + sqrt{1-s^2})/2)^{2/3} for theta=0, + (1-s^2)^{1/6} / 16 * ((1 + sqrt{1-s^2})/2)^{1/3} for theta=pi/2] + +After puncture gauge settled: + r_AH ~ M * (0.275 + 0.76 * sqrt{1-s^2}) + chi_AH ~ 0.26 * sqrt{1-s^2} + +Take it with a grain of salt, but these results are from a numerical fit I did +for numerical Kerr BH runs, approximately settled at around t~50M, from spin=0 +to spin=0.99. You can use it generically for other things, like realizing that +for spin=0 you can plot in VisIt the contour chi=0.26 to get the AH, or that for +spin=0 the AH goes from r=M/2 to r~M + +Binaries: +After some inspiral cases I've been through, I realized that merger happened +when the distance between the punctures was about (roughtly!) ~[0.5,2]*(M1+M2) +and with a radius of ~[0.7,1.1]*(M1+M2) (wobbly of course, and bigger values for +smaller final spin). It makes sense to start looking for the merger before this, +but remember that for PETSc diverging is significantly slower than converging, +so the closer the better. Hence, as default, start looking for a merger when the +separation is about ~<2*(M1+M2) and with a initial guess of ~2*(M1+M2) (twice as +big as what we'll probably get to make sure it converges to the outer AH; even +then it might converge to the inner one, but it is much easer for the AH to +converge if the initial guess is bigger than the actual radius, but the opposite +is more senstitive). +*/ + //! Class to manage AHs and its mergers + control PETSc MPI sub-communicator class AHFinder { @@ -81,19 +113,19 @@ class AHFinder //! mergers will be searched when distance between 'parent' BHs is //! distance < merger_search_factor * 4. * (AH_initial_guess_1 + - //! AH_initial_guess_2) should be roughly '2M' for initial guess at M/2 - //! (set to non-positive to 'always search') - double merger_search_factor; // typically around ~0.8*[2(M1+M2)] - // (default 1) - //! initial guess for merger is 'merger_pre_factor * (AH_initial_guess_1 - //! + AH_initial_guess_2)' - double merger_pre_factor; // typically ~0.6*[M1+M2] (default 1. to - // avoid finding the inner AH) - bool allow_re_attempt; //!< re-attempt with initial guess if + //! AH_initial_guess_2) should be roughly '2M=2(m1+m2)' for initial + //! guess at m/2 (set to non-positive to 'always search') + double merger_search_factor; // see note above (default is 1) + //! initial guess for merger is 'merger_pre_factor * 4. * + //! (AH_initial_guess_1 + AH_initial_guess_2)' + //! set to somethig bigger to avoid finding the inner AH + double merger_pre_factor; // see note above (default to 1.) + + bool allow_re_attempt; //!< re-attempt with initial guess if //!< previous convergence failed (default false) int max_fails_after_lost; //!< number of time steps to try again after //!< (-1 to never) the AH was lost - //!< (default is < 0) + //!< (default is 0) int verbose; //!< print information during execution (default is 1) @@ -193,6 +225,10 @@ class AHFinder m_interpolator = a_interpolator; } + void + set_origins(const std::vector> &origins, + bool includes_mergers = false); + private: //! returns false if 'parent' AHs are too far //! sets the initial guess and the origin for the merger diff --git a/Source/ApparentHorizonFinder/AHFunctions.hpp b/Source/ApparentHorizonFinder/AHFunctions.hpp index bebec63a4..8ad48d9f1 100644 --- a/Source/ApparentHorizonFinder/AHFunctions.hpp +++ b/Source/ApparentHorizonFinder/AHFunctions.hpp @@ -255,6 +255,7 @@ struct ExpansionFunction : AHFunctionDefault // using "r * Expansion" significantly improves the convergence // (making a Schw. BH converge for any radius >~ 0.5*r_AH instead of // only up to ~ 3 * r_AH as it happens just with the expansion) + // (see arXiv:gr-qc/0512169, around Fig 3.4) return expansion * pow(f, a_params.expansion_radius_power); } diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.hpp index 34fcdc107..233653ef6 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.hpp @@ -177,6 +177,10 @@ template class ApparentHorizon double m_area, m_spin, m_mass, m_irreducible_mass, m_spin_z_alt; Tensor<1, double> m_dimensionless_spin_vector; + // prevents resetting the origin when the user externally did 'set_origin' + // before 'solve' + bool origin_already_updated; + ///////////////////////////////////////////////////////////////////////// /////////////////////////// PETSc stuff below /////////////////////////// ///////////////////////////////////////////////////////////////////////// diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp index 2ee5ad254..9c5172106 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp @@ -68,7 +68,10 @@ ApparentHorizon::ApparentHorizon( IntegrationMethod::simpson }), - m_area(NAN), m_spin(NAN), m_mass(NAN), + m_area(NAN), m_spin(NAN), m_mass(NAN), m_irreducible_mass(NAN), + m_spin_z_alt(NAN), m_dimensionless_spin_vector({NAN}), + + origin_already_updated(false), m_interp(a_interp), m_interp_plus(a_interp), m_interp_minus(a_interp), @@ -149,6 +152,16 @@ void ApparentHorizon::set_origin( m_interp.set_origin(a_origin); m_interp_plus.set_origin(a_origin); m_interp_minus.set_origin(a_origin); + origin_already_updated = true; + + if (m_params.verbose > AHFinder::SOME) + { + pout() << "Setting origin to (" << a_origin[0] << "," << a_origin[1] +#if CH_SPACEDIM == 3 + << "," << a_origin[2] +#endif + << ")" << std::endl; + } } template @@ -350,7 +363,7 @@ void ApparentHorizon::solve(double a_dt, m_params.extra_vars); // (ALL CHOMBO ranks do it!!) // estimate the next position where origin will be - if (get_converged()) + if (!origin_already_updated && get_converged()) { if (m_params.predict_origin) predict_next_origin(); @@ -420,6 +433,8 @@ void ApparentHorizon::solve(double a_dt, if (m_params.track_center) calculate_center(); + origin_already_updated = false; + m_area = calculate_area(); #if GR_SPACEDIM == 3 // GR_SPACEDIM, not CH_SPACEDIM !!! #if CH_SPACEDIM == 3 @@ -509,14 +524,13 @@ void ApparentHorizon::write_outputs( } // write stats - - // fake_dt is relevant for the 'remove_duplicate_time_data' part double fake_dt = a_dt * m_params.solve_interval * m_params.print_interval; SmallDataIO file(m_stats, fake_dt, a_time, a_restart_time, SmallDataIO::APPEND, !m_printed_once); - file.remove_duplicate_time_data(); + // not needed -> already done in restart: + // file.remove_duplicate_time_data(); // std::string coords_filename = file.get_new_file_number(fake_dt, // a_time); @@ -922,7 +936,9 @@ void ApparentHorizon::restart( << std::endl; else pout() - << "AH time step changed. Recovering " + << "Old AH time step is different than current one " + "(this might happen if 'AH_print_interval != " + "1').\n Recovering " << old_centers_time_index.size() << " old centers from interpolation, for accurate " "prediction of next origin." @@ -1020,6 +1036,16 @@ void ApparentHorizon::restart( #endif } + // force 'remove_duplicate_time_data' here because merger AH might only run + // a long time after 'a_restart_time' and don't remove duplicate time data + // because of it (if they happen not to be solved right from the restart) + // fake_dt is relevant for the 'remove_duplicate_time_data' part + double fake_dt = current_solve_dt * m_params.print_interval; + SmallDataIO file_cleanup(m_stats, fake_dt, current_time, current_time, + SmallDataIO::APPEND, !m_printed_once); + + file_cleanup.remove_duplicate_time_data(); + // if t=0, solve only if it's a restart and if solve_first_step = true (it // may be false for example for a merger of a binary, for which we don't // want to solve!) @@ -1439,11 +1465,6 @@ ApparentHorizon::calculate_spin_dimensionless( : sqrt(1. - factor * factor)); // factor>1 means numerical // error with spin as 0 - if (m_params.verbose > AHFinder::NONE) - { - pout() << "dimensionless spin = " << spin_dimensionless << std::endl; - } - return spin_dimensionless; } #endif From 9b2cf5075f2e34a6503a0b06f0ca034b3d06a58d Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Fri, 5 Mar 2021 11:56:15 +0000 Subject: [PATCH 11/92] Fix for triple mergers --- Examples/BinaryBH/SimulationParameters.hpp | 2 +- Source/ApparentHorizonFinder/AHFinder.cpp | 58 +++++++++++++++++++--- 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/Examples/BinaryBH/SimulationParameters.hpp b/Examples/BinaryBH/SimulationParameters.hpp index 9a414ddb1..c70df177f 100644 --- a/Examples/BinaryBH/SimulationParameters.hpp +++ b/Examples/BinaryBH/SimulationParameters.hpp @@ -282,7 +282,7 @@ class SimulationParameters : public SimulationParametersBase warn_array_parameter( "momentumB", bh2_params.momentum, std::sqrt(ArrayTools::norm2(bh2_params.momentum)) < - 0.3 * bh1_params.mass, + 0.3 * bh2_params.mass, "approximation used for boosted BH only valid for small boosts"); FOR1(idir) { diff --git a/Source/ApparentHorizonFinder/AHFinder.cpp b/Source/ApparentHorizonFinder/AHFinder.cpp index 46a76cb61..137d05ecb 100644 --- a/Source/ApparentHorizonFinder/AHFinder.cpp +++ b/Source/ApparentHorizonFinder/AHFinder.cpp @@ -150,16 +150,18 @@ void AHFinder::solve(double a_dt, double a_time, double a_restart_time) { auto pair = m_merger_pairs[i]; - if (!((ah_solved[pair.first] == SKIPPED || - ah_solved[pair.first] == SOLVED) && - (ah_solved[pair.second] == SKIPPED || - ah_solved[pair.second] == SOLVED))) + if (ah_solved[pair.first] == NOT_SOLVED || + ah_solved[pair.second] == NOT_SOLVED) continue; // continue if one of the 'parents' was not - // solved/skipped yet + // dealt with yet if (!(m_apparent_horizons[pair.first]->has_been_found() && m_apparent_horizons[pair.second]->has_been_found())) { + if (m_apparent_horizons[i]->m_params.verbose > NONE) + { + pout() << "Skipping BH #" << i << std::endl; + } ++ahs_solved; ah_solved[i] = SKIPPED; continue; // skip if one of the parents was never found yet @@ -175,6 +177,7 @@ void AHFinder::solve(double a_dt, double a_time, double a_restart_time) } ++ahs_solved; ah_solved[i] = TOO_FAR; + continue; } double initial_guess_merger; @@ -186,6 +189,7 @@ void AHFinder::solve(double a_dt, double a_time, double a_restart_time) // if distance is too large, ignore this one and move on if (!do_solve) { + // already printed 'Skipping' when calling 'solve_merger' ++ahs_solved; ah_solved[i] = TOO_FAR; continue; @@ -220,6 +224,11 @@ void AHFinder::solve(double a_dt, double a_time, double a_restart_time) } else { + if (m_apparent_horizons[i]->m_params.verbose > NONE) + { + pout() << "Skipping BH #" << i << ". Not good to go." + << std::endl; + } ++ahs_solved; ah_solved[i] = SKIPPED; } @@ -247,8 +256,33 @@ bool AHFinder::solve_merger(int ah1, int ah2, double &initial_guess_merger, auto center1 = AH1->get_center(); auto center2 = AH2->get_center(); - double initial_guess_sum = - (AH1->get_initial_guess() + AH2->get_initial_guess()); + auto initial_guess1 = AH1->get_initial_guess(); + auto initial_guess2 = AH2->get_initial_guess(); + + if (m_merger_pairs[ah1].first >= 0) + { + double merger_pre_factor = + std::max(m_apparent_horizons[m_merger_pairs[ah1].first] + ->m_params.merger_pre_factor, + m_apparent_horizons[m_merger_pairs[ah1].second] + ->m_params.merger_pre_factor); + if (merger_pre_factor > + 0.) // undo the factor is this a merger from a merger + initial_guess1 /= (merger_pre_factor * 4.); + } + if (m_merger_pairs[ah2].first >= 0) + { + double merger_pre_factor = + std::max(m_apparent_horizons[m_merger_pairs[ah2].first] + ->m_params.merger_pre_factor, + m_apparent_horizons[m_merger_pairs[ah2].second] + ->m_params.merger_pre_factor); + if (merger_pre_factor > + 0.) // undo the factor is this a merger from a merger + initial_guess2 /= (merger_pre_factor * 4.); + } + + double initial_guess_sum = (initial_guess1 + initial_guess2); // some tests revealed it was about 1.2 * initial_guess_sum, but 1.5 to // ensure we don't catch the inner AH @@ -304,6 +338,14 @@ bool AHFinder::solve_merger(int ah1, int ah2, double &initial_guess_merger, pout() << std::endl; } } + else + { + if (AH1->m_params.verbose > NONE) + { + pout() << "Original BH not yet found. Skipping solve for merger." + << std::endl; + } + } return do_solve; } @@ -333,7 +375,7 @@ void AHFinder::set_origins( auto origin2 = m_apparent_horizons[m_merger_pairs[i].second] ->get_origin(); std::array origin; - FOR1(i) { origin[i] = (origin1[i] + origin2[i]) / 2.; } + FOR1(a) { origin[a] = (origin1[a] + origin2[a]) / 2.; } m_apparent_horizons[i]->set_origin(origin); } } From 5fadef95bfdad13918220b190b833b923d021619 Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Fri, 5 Mar 2021 11:56:33 +0000 Subject: [PATCH 12/92] Fix restart bug More fixes for tripler mergers --- Source/ApparentHorizonFinder/AHFinder.cpp | 36 +++++++++++-------- .../ApparentHorizon.impl.hpp | 7 ++-- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/Source/ApparentHorizonFinder/AHFinder.cpp b/Source/ApparentHorizonFinder/AHFinder.cpp index 137d05ecb..4b396afa8 100644 --- a/Source/ApparentHorizonFinder/AHFinder.cpp +++ b/Source/ApparentHorizonFinder/AHFinder.cpp @@ -319,23 +319,31 @@ bool AHFinder::solve_merger(int ah1, int ah2, double &initial_guess_merger, if (AH1->has_been_found() && AH2->has_been_found()) { - do_solve = merger_search_factor <= 0. || distance <= min_distance; + if (AH1->get_converged() && AH2->get_converged()) + { + do_solve = merger_search_factor <= 0. || distance <= min_distance; - if (do_solve) - // make sure twice the radius of the guess is bigger than AH - // distance - CH_assert(min_distance <= 2. * initial_guess_merger); + if (do_solve) + // make sure twice the radius of the guess is bigger than AH + // distance + CH_assert(min_distance <= 2. * initial_guess_merger); - if (AH1->m_params.verbose > NONE) + if (AH1->m_params.verbose > NONE) + { + pout() << "BHs #" << ah1 << " and #" << ah2 + << " at distance = " << distance; + // if distance is too large, ignore this one and move on + if (!do_solve) + pout() << " > minimum distance = " << min_distance + << ". Skipping solve for merger..."; + + pout() << std::endl; + } + } + else { - pout() << "BHs #" << ah1 << " and #" << ah2 - << " at distance = " << distance; - // if distance is too large, ignore this one and move on - if (!do_solve) - pout() << " > minimum distance = " << min_distance - << ". Skipping solve for merger..."; - - pout() << std::endl; + // if AH1 or AH2 was lost, probably merger is already around + do_solve = true; } } else diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp index 9c5172106..8be064d8d 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp @@ -938,7 +938,7 @@ void ApparentHorizon::restart( pout() << "Old AH time step is different than current one " "(this might happen if 'AH_print_interval != " - "1').\n Recovering " + "1').\nRecovering " << old_centers_time_index.size() << " old centers from interpolation, for accurate " "prediction of next origin." @@ -1041,8 +1041,9 @@ void ApparentHorizon::restart( // because of it (if they happen not to be solved right from the restart) // fake_dt is relevant for the 'remove_duplicate_time_data' part double fake_dt = current_solve_dt * m_params.print_interval; - SmallDataIO file_cleanup(m_stats, fake_dt, current_time, current_time, - SmallDataIO::APPEND, !m_printed_once); + SmallDataIO file_cleanup(m_stats, fake_dt, current_time + fake_dt, + current_time, SmallDataIO::APPEND, + !m_printed_once); file_cleanup.remove_duplicate_time_data(); From ecaf9eeea8ef3f111249adcfb61897eb6d8e0270 Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Fri, 5 Mar 2021 11:56:52 +0000 Subject: [PATCH 13/92] Show how to change AHFunction parameters (expansion radius power) --- Examples/BinaryBH/Main_BinaryBH.cpp | 6 ++++-- Examples/BinaryBH/SimulationParameters.hpp | 3 +++ Examples/BinaryBH/params.txt | 1 + Examples/BinaryBH/params_two_punctures.txt | 1 + 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Examples/BinaryBH/Main_BinaryBH.cpp b/Examples/BinaryBH/Main_BinaryBH.cpp index 43e878ec2..f54075a1a 100644 --- a/Examples/BinaryBH/Main_BinaryBH.cpp +++ b/Examples/BinaryBH/Main_BinaryBH.cpp @@ -83,9 +83,11 @@ int runGRChombo(int argc, char *argv[]) AHSphericalGeometry sph2(sim_params.bh2_params.center); bh_amr.m_ah_finder.add_ah(sph1, sim_params.AH_1_initial_guess, - sim_params.AH_params); + sim_params.AH_params, + sim_params.expansion_params); bh_amr.m_ah_finder.add_ah(sph2, sim_params.AH_2_initial_guess, - sim_params.AH_params); + sim_params.AH_params, + sim_params.expansion_params); bh_amr.m_ah_finder.add_ah_merger(0, 1, sim_params.AH_params); } #endif diff --git a/Examples/BinaryBH/SimulationParameters.hpp b/Examples/BinaryBH/SimulationParameters.hpp index c70df177f..1423954ff 100644 --- a/Examples/BinaryBH/SimulationParameters.hpp +++ b/Examples/BinaryBH/SimulationParameters.hpp @@ -49,6 +49,8 @@ class SimulationParameters : public SimulationParametersBase 0.5 * bh2_params.mass); pp.load("AH_set_origins_to_punctures", AH_set_origins_to_punctures, false); + pp.load("AH_expansion_radius_power", + expansion_params.expansion_radius_power, 1.); #endif } @@ -323,6 +325,7 @@ class SimulationParameters : public SimulationParametersBase double AH_1_initial_guess; double AH_2_initial_guess; bool AH_set_origins_to_punctures; + ExpansionFunction::params expansion_params; #endif }; diff --git a/Examples/BinaryBH/params.txt b/Examples/BinaryBH/params.txt index afb055a1a..ee5f32cec 100644 --- a/Examples/BinaryBH/params.txt +++ b/Examples/BinaryBH/params.txt @@ -146,6 +146,7 @@ AH_num_points_v = 48 #AH_print_geometry_data = 0 #AH_re_solve_at_restart = 0 #AH_stop_if_max_fails = 0 +#AH_expansion_radius_power = 1. #AH_1_initial_guess = 0.3 #AH_2_initial_guess = 0.3 diff --git a/Examples/BinaryBH/params_two_punctures.txt b/Examples/BinaryBH/params_two_punctures.txt index 7a2db8103..84185cf65 100644 --- a/Examples/BinaryBH/params_two_punctures.txt +++ b/Examples/BinaryBH/params_two_punctures.txt @@ -138,6 +138,7 @@ AH_num_points_v = 48 #AH_print_geometry_data = 0 #AH_re_solve_at_restart = 0 #AH_stop_if_max_fails = 0 +#AH_expansion_radius_power = 1. #AH_1_initial_guess = 0.3 #AH_2_initial_guess = 0.3 From 37ed079ab88aff84cd0d4f55c20fa267181cf5e7 Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Fri, 5 Mar 2021 11:57:05 +0000 Subject: [PATCH 14/92] Fix bug - ghosts not being filled correctly for diagnostics that the user wants to print at the AH --- Examples/KerrBH/KerrBHLevel.cpp | 2 ++ Source/ApparentHorizonFinder/AHInterpolation.impl.hpp | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Examples/KerrBH/KerrBHLevel.cpp b/Examples/KerrBH/KerrBHLevel.cpp index ff08a91be..829280a2a 100644 --- a/Examples/KerrBH/KerrBHLevel.cpp +++ b/Examples/KerrBH/KerrBHLevel.cpp @@ -50,9 +50,11 @@ void KerrBHLevel::initialData() BoxLoops::loop(GammaCalculator(m_dx), m_state_new, m_state_new, EXCLUDE_GHOST_CELLS); +#ifdef USE_AHFINDER // Diagnostics needed for AHFinder BoxLoops::loop(Constraints(m_dx, c_Ham, Interval(c_Mom1, c_Mom3)), m_state_new, m_state_diagnostics, EXCLUDE_GHOST_CELLS); +#endif } #ifdef CH_USE_HDF5 diff --git a/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp b/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp index 6f3d094ed..fa5b2b558 100644 --- a/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp +++ b/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp @@ -422,7 +422,7 @@ void AHInterpolation::refresh_interpolator( if (printing_step && min_diagnostic_var != -1) { m_interpolator->fill_multilevel_ghosts( - VariableType::evolution, + VariableType::diagnostic, Interval(min_diagnostic_var, max_diagnostic_var)); } } From fad356ecfc2a6265ee7b786e5eb564d588f42297 Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Wed, 10 Mar 2021 17:29:04 +0000 Subject: [PATCH 15/92] Run dos2unix --- Tests/ApparentHorizonFinderTest2D/GNUmakefile | 56 +++++++++---------- Tests/ApparentHorizonFinderTest3D/GNUmakefile | 54 +++++++++--------- Tests/SphericalExtractionTest/GNUmakefile | 44 +++++++-------- 3 files changed, 77 insertions(+), 77 deletions(-) diff --git a/Tests/ApparentHorizonFinderTest2D/GNUmakefile b/Tests/ApparentHorizonFinderTest2D/GNUmakefile index 45c0adbeb..d91ab3fca 100644 --- a/Tests/ApparentHorizonFinderTest2D/GNUmakefile +++ b/Tests/ApparentHorizonFinderTest2D/GNUmakefile @@ -1,28 +1,28 @@ -# -*- Mode: Makefile -*- - -### This makefile produces an executable for each name in the `ebase' -### variable using the libraries named in the `LibNames' variable. - -# Included makefiles need an absolute path to the Chombo installation -# CHOMBO_HOME := Please set the CHOMBO_HOME locally (e.g. export CHOMBO_HOME=... in bash) -# - -DIM = 2 -cxxcppflags := ${cxxcppflags} -DGR_SPACEDIM=2 - -GRCHOMBO_SOURCE = ../../Source - -ebase := ApparentHorizonTest2D - -LibNames := AMRTimeDependent AMRTools BoxTools - -src_dirs := $(GRCHOMBO_SOURCE)/utils \ - $(GRCHOMBO_SOURCE)/simd \ - $(GRCHOMBO_SOURCE)/BoxUtils \ - $(GRCHOMBO_SOURCE)/GRChomboCore \ - $(GRCHOMBO_SOURCE)/AMRInterpolator \ - $(GRCHOMBO_SOURCE)/ApparentHorizonFinder \ - $(GRCHOMBO_SOURCE)/InitialConditions/BlackHoles \ - $(GRCHOMBO_SOURCE)/BlackHoles - -include $(CHOMBO_HOME)/mk/Make.test +# -*- Mode: Makefile -*- + +### This makefile produces an executable for each name in the `ebase' +### variable using the libraries named in the `LibNames' variable. + +# Included makefiles need an absolute path to the Chombo installation +# CHOMBO_HOME := Please set the CHOMBO_HOME locally (e.g. export CHOMBO_HOME=... in bash) +# + +DIM = 2 +cxxcppflags := ${cxxcppflags} -DGR_SPACEDIM=2 + +GRCHOMBO_SOURCE = ../../Source + +ebase := ApparentHorizonTest2D + +LibNames := AMRTimeDependent AMRTools BoxTools + +src_dirs := $(GRCHOMBO_SOURCE)/utils \ + $(GRCHOMBO_SOURCE)/simd \ + $(GRCHOMBO_SOURCE)/BoxUtils \ + $(GRCHOMBO_SOURCE)/GRChomboCore \ + $(GRCHOMBO_SOURCE)/AMRInterpolator \ + $(GRCHOMBO_SOURCE)/ApparentHorizonFinder \ + $(GRCHOMBO_SOURCE)/InitialConditions/BlackHoles \ + $(GRCHOMBO_SOURCE)/BlackHoles + +include $(CHOMBO_HOME)/mk/Make.test diff --git a/Tests/ApparentHorizonFinderTest3D/GNUmakefile b/Tests/ApparentHorizonFinderTest3D/GNUmakefile index c01c5b935..e804f540c 100755 --- a/Tests/ApparentHorizonFinderTest3D/GNUmakefile +++ b/Tests/ApparentHorizonFinderTest3D/GNUmakefile @@ -1,27 +1,27 @@ -# -*- Mode: Makefile -*- - -### This makefile produces an executable for each name in the `ebase' -### variable using the libraries named in the `LibNames' variable. - -# Included makefiles need an absolute path to the Chombo installation -# CHOMBO_HOME := Please set the CHOMBO_HOME locally (e.g. export CHOMBO_HOME=... in bash) -# - -GRCHOMBO_SOURCE = $(shell pwd)/../../Source - -ebase := ApparentHorizonTest3D - -LibNames := AMRTimeDependent AMRTools BoxTools - -src_dirs := $(GRCHOMBO_SOURCE)/utils \ - $(GRCHOMBO_SOURCE)/simd \ - $(GRCHOMBO_SOURCE)/BoxUtils \ - $(GRCHOMBO_SOURCE)/GRChomboCore \ - $(GRCHOMBO_SOURCE)/AMRInterpolator \ - $(GRCHOMBO_SOURCE)/ApparentHorizonFinder \ - $(GRCHOMBO_SOURCE)/InitialConditions/BlackHoles \ - $(GRCHOMBO_SOURCE)/BlackHoles \ - $(GRCHOMBO_SOURCE)/CCZ4 - -include $(CHOMBO_HOME)/mk/Make.test - +# -*- Mode: Makefile -*- + +### This makefile produces an executable for each name in the `ebase' +### variable using the libraries named in the `LibNames' variable. + +# Included makefiles need an absolute path to the Chombo installation +# CHOMBO_HOME := Please set the CHOMBO_HOME locally (e.g. export CHOMBO_HOME=... in bash) +# + +GRCHOMBO_SOURCE = $(shell pwd)/../../Source + +ebase := ApparentHorizonTest3D + +LibNames := AMRTimeDependent AMRTools BoxTools + +src_dirs := $(GRCHOMBO_SOURCE)/utils \ + $(GRCHOMBO_SOURCE)/simd \ + $(GRCHOMBO_SOURCE)/BoxUtils \ + $(GRCHOMBO_SOURCE)/GRChomboCore \ + $(GRCHOMBO_SOURCE)/AMRInterpolator \ + $(GRCHOMBO_SOURCE)/ApparentHorizonFinder \ + $(GRCHOMBO_SOURCE)/InitialConditions/BlackHoles \ + $(GRCHOMBO_SOURCE)/BlackHoles \ + $(GRCHOMBO_SOURCE)/CCZ4 + +include $(CHOMBO_HOME)/mk/Make.test + diff --git a/Tests/SphericalExtractionTest/GNUmakefile b/Tests/SphericalExtractionTest/GNUmakefile index 8b72a8165..f224a99d5 100644 --- a/Tests/SphericalExtractionTest/GNUmakefile +++ b/Tests/SphericalExtractionTest/GNUmakefile @@ -1,22 +1,22 @@ -# -*- Mode: Makefile -*- - -### This makefile produces an executable for each name in the `ebase' -### variable using the libraries named in the `LibNames' variable. - -# Included makefiles need an absolute path to the Chombo installation -# CHOMBO_HOME := Please set the CHOMBO_HOME locally (e.g. export CHOMBO_HOME=... in bash) - -GRCHOMBO_SOURCE = $(shell pwd)/../../Source - -ebase := SphericalExtractionTest - -LibNames := AMRTimeDependent AMRTools BoxTools - -src_dirs := $(GRCHOMBO_SOURCE)/utils \ - $(GRCHOMBO_SOURCE)/simd \ - $(GRCHOMBO_SOURCE)/BoxUtils \ - $(GRCHOMBO_SOURCE)/CCZ4 \ - $(GRCHOMBO_SOURCE)/GRChomboCore \ - $(GRCHOMBO_SOURCE)/AMRInterpolator - -include $(CHOMBO_HOME)/mk/Make.test +# -*- Mode: Makefile -*- + +### This makefile produces an executable for each name in the `ebase' +### variable using the libraries named in the `LibNames' variable. + +# Included makefiles need an absolute path to the Chombo installation +# CHOMBO_HOME := Please set the CHOMBO_HOME locally (e.g. export CHOMBO_HOME=... in bash) + +GRCHOMBO_SOURCE = $(shell pwd)/../../Source + +ebase := SphericalExtractionTest + +LibNames := AMRTimeDependent AMRTools BoxTools + +src_dirs := $(GRCHOMBO_SOURCE)/utils \ + $(GRCHOMBO_SOURCE)/simd \ + $(GRCHOMBO_SOURCE)/BoxUtils \ + $(GRCHOMBO_SOURCE)/CCZ4 \ + $(GRCHOMBO_SOURCE)/GRChomboCore \ + $(GRCHOMBO_SOURCE)/AMRInterpolator + +include $(CHOMBO_HOME)/mk/Make.test From dfa02389bace83cc042addd0c7d7f593baa47843 Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Wed, 10 Mar 2021 17:29:20 +0000 Subject: [PATCH 16/92] [Important!] Fix Expansion calculation bug!!! --- Source/ApparentHorizonFinder/AHFunctions.hpp | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/Source/ApparentHorizonFinder/AHFunctions.hpp b/Source/ApparentHorizonFinder/AHFunctions.hpp index 8ad48d9f1..30339f5b8 100644 --- a/Source/ApparentHorizonFinder/AHFunctions.hpp +++ b/Source/ApparentHorizonFinder/AHFunctions.hpp @@ -227,14 +227,16 @@ struct ExpansionFunction : AHFunctionDefault Tensor<1, double> s_L = get_level_function_derivative(geo_data, deriv); Tensor<2, double> Ds = get_level_function_2nd_covariant_derivative(geo_data, deriv, s_L); - Tensor<1, double> S_U = get_spatial_normal_U(s_L); + double norm_s; + Tensor<1, double> S_U; + get_spatial_normal_U_and_norm(S_U, norm_s, s_L); // calculate D_i S^i and S^i S^j K_ij double DiSi = 0.; double Kij_dot_Si_Sj = 0.; FOR2(a, b) { - DiSi += (g_UU[a][b] - S_U[a] * S_U[b]) * Ds[a][b]; + DiSi += (g_UU[a][b] - S_U[a] * S_U[b]) * Ds[a][b] / norm_s; Kij_dot_Si_Sj += S_U[a] * S_U[b] * K[a][b]; } @@ -281,11 +283,19 @@ struct ExpansionFunction : AHFunctionDefault return s_L; } Tensor<1, double> get_spatial_normal_U(const Tensor<1, double> &s_L) const + { + Tensor<1, double> S_U; + double norm_s; + get_spatial_normal_U_and_norm(S_U, norm_s, s_L); + return S_U; + } + void get_spatial_normal_U_and_norm(Tensor<1, double> &S_U, double &norm_s, + const Tensor<1, double> &s_L) const { // calculate S_U = the real 's' of Alcubierre // norm of s_L = | D_a L| (the 'u' in 6.7.12 of Alcubierre) - double norm_s = 0.0; + norm_s = 0.0; FOR2(a, b) { norm_s += g_UU[a][b] * s_L[a] * s_L[b]; } norm_s = sqrt(norm_s); @@ -293,10 +303,7 @@ struct ExpansionFunction : AHFunctionDefault Tensor<1, double> s_U = {0.}; FOR2(a, b) { s_U[a] += g_UU[a][b] * s_L[b]; } - Tensor<1, double> S_U = {0.}; FOR1(a) { S_U[a] = s_U[a] / norm_s; } - - return S_U; } Tensor<2, double> get_level_function_2nd_covariant_derivative( From 068a30a1adbfcf86ad723988e2d73344968f559e Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Fri, 12 Mar 2021 16:32:20 +0000 Subject: [PATCH 17/92] Default AH fix (with opt=HIGH, flat metric was not initialized to 0 in off-diagonal terms) Fix the fix. Recent fix to restart was uneeded. --- Source/ApparentHorizonFinder/AHFinder.hpp | 4 ++-- Source/ApparentHorizonFinder/AHFunctionDefault.hpp | 2 +- .../ApparentHorizonFinder/ApparentHorizon.impl.hpp | 14 +------------- 3 files changed, 4 insertions(+), 16 deletions(-) diff --git a/Source/ApparentHorizonFinder/AHFinder.hpp b/Source/ApparentHorizonFinder/AHFinder.hpp index bff080ad8..6c48d4305 100644 --- a/Source/ApparentHorizonFinder/AHFinder.hpp +++ b/Source/ApparentHorizonFinder/AHFinder.hpp @@ -205,8 +205,8 @@ class AHFinder int add_ah(const AHSurfaceGeometry &a_coord_system, double a_initial_guess, const params &a_params, const typename AHFunction::params &a_func_params, - const std::string &a_stats = "stats", - const std::string &a_coords = "coords", + const std::string &a_stats = "stats_AH", + const std::string &a_coords = "coords_AH", bool solve_first_step = true); // returns the index of the AH in m_apparent_horizons diff --git a/Source/ApparentHorizonFinder/AHFunctionDefault.hpp b/Source/ApparentHorizonFinder/AHFunctionDefault.hpp index 902604f90..cd7b599f2 100644 --- a/Source/ApparentHorizonFinder/AHFunctionDefault.hpp +++ b/Source/ApparentHorizonFinder/AHFunctionDefault.hpp @@ -41,7 +41,7 @@ struct AHFunctionDefault ALWAYS_INLINE const Tensor<2, double> get_metric() const { // cartesian flat metric - Tensor<2, double> g; + Tensor<2, double> g = {0.}; FOR1(i) { g[i][i] = 1.; } return g; } diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp index 8be064d8d..4830ee4d6 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp @@ -529,8 +529,7 @@ void ApparentHorizon::write_outputs( SmallDataIO file(m_stats, fake_dt, a_time, a_restart_time, SmallDataIO::APPEND, !m_printed_once); - // not needed -> already done in restart: - // file.remove_duplicate_time_data(); + file.remove_duplicate_time_data(); // std::string coords_filename = file.get_new_file_number(fake_dt, // a_time); @@ -1036,17 +1035,6 @@ void ApparentHorizon::restart( #endif } - // force 'remove_duplicate_time_data' here because merger AH might only run - // a long time after 'a_restart_time' and don't remove duplicate time data - // because of it (if they happen not to be solved right from the restart) - // fake_dt is relevant for the 'remove_duplicate_time_data' part - double fake_dt = current_solve_dt * m_params.print_interval; - SmallDataIO file_cleanup(m_stats, fake_dt, current_time + fake_dt, - current_time, SmallDataIO::APPEND, - !m_printed_once); - - file_cleanup.remove_duplicate_time_data(); - // if t=0, solve only if it's a restart and if solve_first_step = true (it // may be false for example for a merger of a binary, for which we don't // want to solve!) From b6de6e155e9a7f52fb23d4ddb463e5ce62bf7cdc Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Fri, 23 Apr 2021 16:55:09 +0100 Subject: [PATCH 18/92] Minor change to make extraction more generic --- Source/AMRInterpolator/SphericalGeometry.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/AMRInterpolator/SphericalGeometry.hpp b/Source/AMRInterpolator/SphericalGeometry.hpp index aa35d4ab6..53967caf6 100644 --- a/Source/AMRInterpolator/SphericalGeometry.hpp +++ b/Source/AMRInterpolator/SphericalGeometry.hpp @@ -66,13 +66,13 @@ class SphericalGeometry //! returns the theta coordinate associated to the theta/u index ALWAYS_INLINE double u(int a_itheta, int a_num_points_theta) const { - return a_itheta * du(a_num_points_theta); + return a_itheta * du(a_num_points_theta) + get_domain_u_min(); } //! returns the phi coordinate associated to the phi/v index ALWAYS_INLINE double v(int a_iphi, int a_num_points_phi) const { - return a_iphi * dv(a_num_points_phi); + return a_iphi * dv(a_num_points_phi) + get_domain_v_min(); } ALWAYS_INLINE bool is_u_periodic() const { return false; } From 7b935939e174bd251033fd016cbbc625a225d0ee Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Fri, 23 Apr 2021 16:56:04 +0100 Subject: [PATCH 19/92] Add Open Integration Methods compatibility and UniformSphericalExtraction. This is to be added to the public code as a separate PR first. --- Examples/BinaryBH/Main_BinaryBH.cpp | 4 +- Examples/KerrBH/Main_KerrBH.cpp | 2 +- Examples/KerrBH/params_cheap.txt | 6 +- Examples/ScalarField/Main_ScalarField.cpp | 2 +- Examples/SingleBH/Main_SingleBH.cpp | 2 +- Source/AMRInterpolator/IntegrationMethod.hpp | 9 +- .../IntegrationMethodSetup.hpp | 16 +- .../AMRInterpolator/SphericalExtraction.hpp | 114 ++++---- Source/AMRInterpolator/SphericalGeometry.hpp | 58 ++-- .../SphericalGeometryUniform.hpp | 145 ++++++++++ Source/AMRInterpolator/SurfaceExtraction.hpp | 63 ++--- .../SurfaceExtraction.impl.hpp | 8 +- .../ApparentHorizonFinder/AHInterpolation.hpp | 20 -- .../AHInterpolation.impl.hpp | 61 ----- .../AHSphericalGeometryUniform.hpp | 172 ++++++++++++ .../AHStringGeometry.hpp | 40 ++- .../ApparentHorizon.impl.hpp | 59 +++-- .../ApparentHorizon_petsc.impl.hpp | 54 ++-- .../GRChomboCore/SimulationParametersBase.hpp | 4 +- .../ChiExtractionTaggingCriterion.hpp | 6 +- .../ChiPunctureExtractionTaggingCriterion.hpp | 4 +- Source/utils/WeylExtraction.hpp | 4 +- .../ApparentHorizonTest3D.cpp | 13 +- .../ApparentHorizonTest3D.inputs | 5 +- .../SimulationParameters.hpp | 5 +- .../SimulationParameters.hpp | 2 +- .../SphericalExtractionTest.cpp | 56 +++- .../SphericalExtractionTest.inputs | 9 +- .../GNUmakefile | 22 ++ .../SetHarmonic.hpp | 42 +++ .../SetHarmonic.impl.hpp | 33 +++ .../SimulationParameters.hpp | 58 ++++ .../SphericalExtractionUniformTest.cpp | 248 ++++++++++++++++++ .../SphericalExtractionUniformTest.inputs | 29 ++ .../SphericalExtractionUniformTestLevel.hpp | 38 +++ .../UserVariables.hpp | 30 +++ 36 files changed, 1160 insertions(+), 283 deletions(-) create mode 100644 Source/AMRInterpolator/SphericalGeometryUniform.hpp create mode 100644 Source/ApparentHorizonFinder/AHSphericalGeometryUniform.hpp create mode 100644 Tests/SphericalExtractionUniformTest/GNUmakefile create mode 100644 Tests/SphericalExtractionUniformTest/SetHarmonic.hpp create mode 100644 Tests/SphericalExtractionUniformTest/SetHarmonic.impl.hpp create mode 100644 Tests/SphericalExtractionUniformTest/SimulationParameters.hpp create mode 100644 Tests/SphericalExtractionUniformTest/SphericalExtractionUniformTest.cpp create mode 100644 Tests/SphericalExtractionUniformTest/SphericalExtractionUniformTest.inputs create mode 100644 Tests/SphericalExtractionUniformTest/SphericalExtractionUniformTestLevel.hpp create mode 100644 Tests/SphericalExtractionUniformTest/UserVariables.hpp diff --git a/Examples/BinaryBH/Main_BinaryBH.cpp b/Examples/BinaryBH/Main_BinaryBH.cpp index f54075a1a..b5604b9ad 100644 --- a/Examples/BinaryBH/Main_BinaryBH.cpp +++ b/Examples/BinaryBH/Main_BinaryBH.cpp @@ -79,8 +79,8 @@ int runGRChombo(int argc, char *argv[]) #ifdef USE_AHFINDER if (sim_params.AH_activate) { - AHSphericalGeometry sph1(sim_params.bh1_params.center); - AHSphericalGeometry sph2(sim_params.bh2_params.center); + AHSurfaceGeometry sph1(sim_params.bh1_params.center); + AHSurfaceGeometry sph2(sim_params.bh2_params.center); bh_amr.m_ah_finder.add_ah(sph1, sim_params.AH_1_initial_guess, sim_params.AH_params, diff --git a/Examples/KerrBH/Main_KerrBH.cpp b/Examples/KerrBH/Main_KerrBH.cpp index f2e82f89c..e6bc0fa09 100644 --- a/Examples/KerrBH/Main_KerrBH.cpp +++ b/Examples/KerrBH/Main_KerrBH.cpp @@ -53,7 +53,7 @@ int runGRChombo(int argc, char *argv[]) #ifdef USE_AHFINDER if (sim_params.AH_activate) { - AHSphericalGeometry sph(sim_params.kerr_params.center); + AHSurfaceGeometry sph(sim_params.kerr_params.center); #ifdef USE_CHI_CONTOURS // uncomment in UserVariables std::string str_chi = std::to_string(sim_params.look_for_chi_contour); diff --git a/Examples/KerrBH/params_cheap.txt b/Examples/KerrBH/params_cheap.txt index 63dc452c1..c77648d5c 100644 --- a/Examples/KerrBH/params_cheap.txt +++ b/Examples/KerrBH/params_cheap.txt @@ -119,9 +119,9 @@ num_points_theta = 52 AH_activate = 1 AH_num_ranks = 4 #20 AH_num_points_u = 15 #30 -AH_num_points_v = 20 #50 -AH_solve_interval = 2 -AH_print_interval = 2 +AH_num_points_v = 18 #50 +AH_solve_interval = 1 +AH_print_interval = 1 AH_track_center = false AH_predict_origin = false AH_level_to_run = 0 diff --git a/Examples/ScalarField/Main_ScalarField.cpp b/Examples/ScalarField/Main_ScalarField.cpp index ca2b7c491..fd6f711c4 100644 --- a/Examples/ScalarField/Main_ScalarField.cpp +++ b/Examples/ScalarField/Main_ScalarField.cpp @@ -51,7 +51,7 @@ int runGRChombo(int argc, char *argv[]) #ifdef USE_AHFINDER if (sim_params.AH_activate) { - AHSphericalGeometry sph(sim_params.center); + AHSurfaceGeometry sph(sim_params.center); bh_amr.m_ah_finder.add_ah(sph, sim_params.AH_initial_guess, sim_params.AH_params); } diff --git a/Examples/SingleBH/Main_SingleBH.cpp b/Examples/SingleBH/Main_SingleBH.cpp index 45fcb60a8..f36dcd2cd 100644 --- a/Examples/SingleBH/Main_SingleBH.cpp +++ b/Examples/SingleBH/Main_SingleBH.cpp @@ -42,7 +42,7 @@ int runGRChombo(int argc, char *argv[]) #ifdef USE_AHFINDER if (sim_params.AH_activate) { - AHSphericalGeometry sph(sim_params.bh_params.center); + AHSurfaceGeometry sph(sim_params.bh_params.center); bh_amr.m_ah_finder.add_ah(sph, sim_params.AH_initial_guess, sim_params.AH_params); } diff --git a/Source/AMRInterpolator/IntegrationMethod.hpp b/Source/AMRInterpolator/IntegrationMethod.hpp index f95d65efc..86c68b1f2 100644 --- a/Source/AMRInterpolator/IntegrationMethod.hpp +++ b/Source/AMRInterpolator/IntegrationMethod.hpp @@ -70,10 +70,17 @@ class IntegrationMethod return m_weights[weight_index]; } + // Closed default methods static const IntegrationMethod trapezium; - static const IntegrationMethod midpoint; static const IntegrationMethod simpson; + static const IntegrationMethod simpson38; static const IntegrationMethod boole; + + // Open default methods + static const IntegrationMethod midpoint; + static const IntegrationMethod milne_regularized; + static const IntegrationMethod open_3rd_order; + static const IntegrationMethod open_4th_order; }; #endif /* INTEGRATIONMETHOD_HPP_ */ diff --git a/Source/AMRInterpolator/IntegrationMethodSetup.hpp b/Source/AMRInterpolator/IntegrationMethodSetup.hpp index 81d10bb05..49ffdd8ee 100644 --- a/Source/AMRInterpolator/IntegrationMethodSetup.hpp +++ b/Source/AMRInterpolator/IntegrationMethodSetup.hpp @@ -9,12 +9,26 @@ #include "IntegrationMethod.hpp" // define the static IntegrationMethods here +// Closed methods: const IntegrationMethod IntegrationMethod::trapezium({0.5}); -const IntegrationMethod IntegrationMethod::midpoint({1.0}, false); const IntegrationMethod IntegrationMethod::simpson({0.3333333333333333, 1.3333333333333333}); +const IntegrationMethod IntegrationMethod::simpson38({0.375, 1.125, 1.125}); const IntegrationMethod IntegrationMethod::boole({0.3111111111111111, 1.4222222222222222, 0.53333333333333, 1.4222222222222222}); +// Open Methods: +const IntegrationMethod IntegrationMethod::midpoint({1.0}, false); +const IntegrationMethod + IntegrationMethod::milne_regularized({1.125, 0.75, 1.125}, false); +const IntegrationMethod + IntegrationMethod::open_3rd_order({1.0833333333333333, 0.9166666666666666, + 0.9166666666666666, 1.0833333333333333}, + false); +const IntegrationMethod IntegrationMethod::open_4th_order( + {1.1935763888888888, 0.4340277777777778, 1.7447916666666667, + 0.4340277777777778, 1.1935763888888888}, + false); + #endif /* INTEGRATIONMETHODSETUP_HPP_ */ diff --git a/Source/AMRInterpolator/SphericalExtraction.hpp b/Source/AMRInterpolator/SphericalExtraction.hpp index 348317cc6..d9659eccf 100644 --- a/Source/AMRInterpolator/SphericalExtraction.hpp +++ b/Source/AMRInterpolator/SphericalExtraction.hpp @@ -7,67 +7,72 @@ #define SPHERICALEXTRACTION_HPP_ #include "SphericalGeometry.hpp" +#include "SphericalGeometryUniform.hpp" #include "SphericalHarmonics.hpp" #include "SurfaceExtraction.hpp" +struct spherical_extraction_params_t : surface_extraction_params_t +{ + int &num_extraction_radii = num_surfaces; + std::vector &extraction_radii = surface_param_values; + int &num_points_theta = num_points_u; + int &num_points_phi = num_points_v; + std::array center; //!< the center of the spherical + //!< shells + std::array &extraction_center = center; + int num_modes; //!< the number of modes to extract + std::vector> modes; //!< the modes to extract + //!< l = first, m = second + // constructor + spherical_extraction_params_t() = default; + + // copy constructor defined due to references pointing to the wrong + // things with the default copy constructor + spherical_extraction_params_t(const spherical_extraction_params_t ¶ms) + : surface_extraction_params_t(params), center(params.center), + num_modes(params.num_modes), modes(params.modes) + { + } +}; + //! A child class of SurfaceExtraction for extraction on spherical shells -class SphericalExtraction : public SurfaceExtraction +template +class ModeExtraction : public SurfaceExtraction { public: - struct params_t : SurfaceExtraction::params_t - { - int &num_extraction_radii = num_surfaces; - std::vector &extraction_radii = surface_param_values; - int &num_points_theta = num_points_u; - int &num_points_phi = num_points_v; - std::array center; //!< the center of the spherical - //!< shells - std::array &extraction_center = center; - int num_modes; //!< the number of modes to extract - std::vector> modes; //!< the modes to extract - //!< l = first, m = second - // constructor - params_t() = default; - - // copy constructor defined due to references pointing to the wrong - // things with the default copy constructor - params_t(const params_t ¶ms) - : SurfaceExtraction::params_t(params), center(params.center), - num_modes(params.num_modes), modes(params.modes) - { - } - }; const std::array m_center; const int m_num_modes; const std::vector> m_modes; - SphericalExtraction(const params_t &a_params, double a_dt, double a_time, - bool a_first_step, double a_restart_time = 0.0) - : SurfaceExtraction(a_params.center, a_params, a_dt, a_time, - a_first_step, a_restart_time), + using vars_t = SurfaceExtraction; + + ModeExtraction(const spherical_extraction_params_t &a_params, double a_dt, + double a_time, bool a_first_step, + double a_restart_time = 0.0) + : SurfaceExtraction(a_params.center, a_params, a_dt, + a_time, a_first_step, + a_restart_time), m_center(a_params.center), m_num_modes(a_params.num_modes), m_modes(a_params.modes) { } - SphericalExtraction(const params_t &a_params, - const std::vector &a_vars, double a_dt, - double a_time, bool a_first_step, - double a_restart_time = 0.0) - : SphericalExtraction(a_params, a_dt, a_time, a_first_step, - a_restart_time) + ModeExtraction(const spherical_extraction_params_t &a_params, + const std::vector &a_vars, double a_dt, + double a_time, bool a_first_step, + double a_restart_time = 0.0) + : ModeExtraction(a_params, a_dt, a_time, a_first_step, a_restart_time) { - add_vars(a_vars); + this->add_vars(a_vars); } - SphericalExtraction(const params_t &a_params, - const std::vector &a_evolution_vars, double a_dt, - double a_time, bool a_first_step, - double a_restart_time = 0.0) - : SphericalExtraction(a_params, a_dt, a_time, a_first_step, - a_restart_time) + ModeExtraction(const spherical_extraction_params_t &a_params, + const std::vector &a_evolution_vars, double a_dt, + double a_time, bool a_first_step, + double a_restart_time = 0.0) + : ModeExtraction(a_params, a_dt, a_time, a_first_step, a_restart_time) { - add_evolution_vars(a_evolution_vars); + this->add_evolution_vars(a_evolution_vars); } // alias this long type used for complex functions defined on the surface @@ -86,13 +91,13 @@ class SphericalExtraction : public SurfaceExtraction const bool a_broadcast_integral = false) { // {x, y, z} - SphericalGeometry::UP_DIR up_dir = m_geom.get_up_dir(); + SphericalGeometry::UP_DIR up_dir = this->m_geom.get_up_dir(); std::array dirs = {(up_dir + 1) % 3, (up_dir + 2) % 3, up_dir}; std::array center; for (int i = 0; i < 3; ++i) center[i] = (dirs[i] < CH_SPACEDIM ? m_center[dirs[i]] : 0.); - auto integrand_re = [dirs, center, &geom = m_geom, es, el, em, + auto integrand_re = [dirs, center, &geom = this->m_geom, es, el, em, &a_function](std::vector &a_data_here, double r, double theta, double phi) { // note that spin_Y_lm requires the coordinates with the center @@ -107,10 +112,10 @@ class SphericalExtraction : public SurfaceExtraction function_here.second * Y_lm.Im) / (r * r); }; - add_integrand(integrand_re, out_integrals.first, a_method_theta, - a_method_phi, a_broadcast_integral); + this->add_integrand(integrand_re, out_integrals.first, a_method_theta, + a_method_phi, a_broadcast_integral); - auto integrand_im = [dirs, center, &geom = m_geom, es, el, em, + auto integrand_im = [dirs, center, &geom = this->m_geom, es, el, em, &a_function](std::vector &a_data_here, double r, double theta, double phi) { // note that spin_Y_lm requires the coordinates with the center @@ -125,8 +130,8 @@ class SphericalExtraction : public SurfaceExtraction function_here.first * Y_lm.Im) / (r * r); }; - add_integrand(integrand_im, out_integrals.second, a_method_theta, - a_method_phi, a_broadcast_integral); + this->add_integrand(integrand_im, out_integrals.second, a_method_theta, + a_method_phi, a_broadcast_integral); } //! If you only want to extract one mode, you can use this function which @@ -136,16 +141,19 @@ class SphericalExtraction : public SurfaceExtraction const IntegrationMethod &a_method_theta = IntegrationMethod::simpson, const IntegrationMethod &a_method_phi = IntegrationMethod::trapezium) { - m_integrands.clear(); - m_integration_methods.clear(); - m_integrals.clear(); + this->m_integrands.clear(); + this->m_integration_methods.clear(); + this->m_integrals.clear(); std::pair, std::vector> integrals; add_mode_integrand(es, el, em, a_function, integrals, a_method_theta, a_method_phi); - integrate(); + this->integrate(); return integrals; } }; +using SphericalExtraction = ModeExtraction; +using SphericalExtractionUniform = ModeExtraction; + #endif /* SPHERICALEXTRACTION_HPP_ */ diff --git a/Source/AMRInterpolator/SphericalGeometry.hpp b/Source/AMRInterpolator/SphericalGeometry.hpp index 53967caf6..c380c8649 100644 --- a/Source/AMRInterpolator/SphericalGeometry.hpp +++ b/Source/AMRInterpolator/SphericalGeometry.hpp @@ -12,6 +12,7 @@ // Other includes #include "AlwaysInline.hpp" +#include "IntegrationMethod.hpp" #include "MayDay.H" #include #include @@ -44,50 +45,51 @@ class SphericalGeometry } ALWAYS_INLINE UP_DIR get_up_dir() const { return m_up_dir; } - ALWAYS_INLINE double get_domain_u_min() const { return 0.; } - ALWAYS_INLINE double get_domain_u_max() const { return M_PI; } - ALWAYS_INLINE double get_domain_v_min() const { return 0.; } - ALWAYS_INLINE double get_domain_v_max() const { return 2. * M_PI; } + + static ALWAYS_INLINE double get_domain_u_min() { return 0.; } + static ALWAYS_INLINE double get_domain_u_max() { return M_PI; } + static ALWAYS_INLINE double get_domain_v_min() { return 0.; } + static ALWAYS_INLINE double get_domain_v_max() { return 2. * M_PI; } //! returns the grid spacing in theta - ALWAYS_INLINE double du(int a_num_points_theta) const + static ALWAYS_INLINE double du(int a_num_points_theta) { return (get_domain_u_max() - get_domain_u_min()) / (double)(a_num_points_theta - 1); } //! returns the grid spacing in phi - ALWAYS_INLINE double dv(int a_num_points_phi) const + static ALWAYS_INLINE double dv(int a_num_points_phi) { return (get_domain_v_max() - get_domain_v_min()) / ((double)a_num_points_phi); } //! returns the theta coordinate associated to the theta/u index - ALWAYS_INLINE double u(int a_itheta, int a_num_points_theta) const + static ALWAYS_INLINE double u(int a_itheta, int a_num_points_theta) { return a_itheta * du(a_num_points_theta) + get_domain_u_min(); } //! returns the phi coordinate associated to the phi/v index - ALWAYS_INLINE double v(int a_iphi, int a_num_points_phi) const + static ALWAYS_INLINE double v(int a_iphi, int a_num_points_phi) { return a_iphi * dv(a_num_points_phi) + get_domain_v_min(); } - ALWAYS_INLINE bool is_u_periodic() const { return false; } - ALWAYS_INLINE bool is_v_periodic() const { return true; } + static ALWAYS_INLINE bool is_u_periodic() { return false; } + static ALWAYS_INLINE bool is_v_periodic() { return true; } - ALWAYS_INLINE std::string param_name() const { return "r"; } + static ALWAYS_INLINE std::string param_name() { return "r"; } - ALWAYS_INLINE std::string u_name() const { return "theta"; } + static ALWAYS_INLINE std::string u_name() { return "theta"; } - ALWAYS_INLINE std::string v_name() const { return "phi"; } + static ALWAYS_INLINE std::string v_name() { return "phi"; } //! returns the area element on a sphere with radius a_radius at the point //! (a_theta, a_phi) - ALWAYS_INLINE double area_element(double a_radius, double a_theta, - double a_phi) const + static ALWAYS_INLINE double area_element(double a_radius, double a_theta, + double a_phi) { return a_radius * a_radius * sin(a_theta); } @@ -111,6 +113,32 @@ class SphericalGeometry } } + static const IntegrationMethod & + get_recommended_integration_method_u(int a_num_points_theta) + { + static const IntegrationMethod &simpson = IntegrationMethod::simpson; + static const IntegrationMethod &trapezium = + IntegrationMethod::trapezium; + if (simpson.is_valid(a_num_points_theta, is_u_periodic())) + return simpson; + MayDay::Warning("Use an odd number of 'u' points to use simpson rule. " + "Defaulting to trapezium."); + return trapezium; + } + + static const IntegrationMethod & + get_recommended_integration_method_v(int a_num_points_phi) + { + static const IntegrationMethod &simpson = IntegrationMethod::simpson; + static const IntegrationMethod &trapezium = + IntegrationMethod::trapezium; + if (simpson.is_valid(a_num_points_phi, is_v_periodic())) + return simpson; + MayDay::Warning("Use an even number of 'v' points to use simpson rule. " + "Defaulting to trapezium."); + return trapezium; + } + protected: std::array m_center; UP_DIR m_up_dir; diff --git a/Source/AMRInterpolator/SphericalGeometryUniform.hpp b/Source/AMRInterpolator/SphericalGeometryUniform.hpp new file mode 100644 index 000000000..f7c4fbb65 --- /dev/null +++ b/Source/AMRInterpolator/SphericalGeometryUniform.hpp @@ -0,0 +1,145 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef SPHERICALGEOMETRYUNIFORM_HPP_ +#define SPHERICALGEOMETRYUNIFORM_HPP_ + +// Chombo includes +#include "AlwaysInline.hpp" +#include "MayDay.H" + +// Other includes +#include "AlwaysInline.hpp" +#include "IntegrationMethod.hpp" +#include "MayDay.H" +#include "SphericalGeometry.hpp" +#include +#include +#include + +// Chombo namespace +#include "UsingNamespace.H" + +//! This SurfaceGeometry template class provides spherical shell geometry +//! implementation for the SurfaceExtraction class +//! u = u, v = phi +class SphericalGeometryUniform +{ + public: + SphericalGeometryUniform(const std::array &a_center) + : m_center(a_center) + { +#if CH_SPACEDIM == 3 // for now force 'Z', could be an argument later on + m_up_dir = SphericalGeometry::Z; +#elif CH_SPACEDIM == 2 // in Cartoon method, assume 'X' as 'z' axis + m_up_dir = SphericalGeometry::X; +#endif + } + + ALWAYS_INLINE SphericalGeometry::UP_DIR get_up_dir() const + { + return m_up_dir; + } + + static ALWAYS_INLINE double get_domain_u_min() { return -1.; } + static ALWAYS_INLINE double get_domain_u_max() { return 1.; } + static ALWAYS_INLINE double get_domain_v_min() { return 0.; } + static ALWAYS_INLINE double get_domain_v_max() { return 2. * M_PI; } + + //! returns the grid spacing in u + static ALWAYS_INLINE double du(int a_num_points_u) + { + return (get_domain_u_max() - get_domain_u_min()) / + (double)(a_num_points_u); + } + + //! returns the grid spacing in phi + static ALWAYS_INLINE double dv(int a_num_points_phi) + { + return (get_domain_v_max() - get_domain_v_min()) / + ((double)a_num_points_phi); + } + + //! returns the u coordinate associated to the u/u index + static ALWAYS_INLINE double u(int a_iu, int a_num_points_u) + { + return (a_iu + 0.5) * du(a_num_points_u) + get_domain_u_min(); + } + + //! returns the phi coordinate associated to the phi/v index + static ALWAYS_INLINE double v(int a_iphi, int a_num_points_phi) + { + return a_iphi * dv(a_num_points_phi) + get_domain_v_min(); + } + + static ALWAYS_INLINE bool is_u_periodic() { return false; } + static ALWAYS_INLINE bool is_v_periodic() { return true; } + + static ALWAYS_INLINE std::string param_name() { return "r"; } + + static ALWAYS_INLINE std::string u_name() { return "z/r"; } + + static ALWAYS_INLINE std::string v_name() { return "phi"; } + + //! returns the area element on a sphere with radius a_radius at the point + //! (a_u, a_phi) + static ALWAYS_INLINE double area_element(double a_radius, double a_u, + double a_phi) + { + return a_radius * a_radius; + } + + //! returns the Cartesian coordinate in direction a_dir with specified + //! radius, u and phi. + ALWAYS_INLINE double get_grid_coord(int a_dir, double a_radius, double a_u, + double a_phi = 0.) const + { + double center = (a_dir < CH_SPACEDIM ? m_center[a_dir] : 0.); + switch ((a_dir + 2 - m_up_dir) % 3) + { + case (0): + return center + a_radius * sqrt(1. - a_u * a_u) * cos(a_phi); + case (1): + return center + a_radius * sqrt(1. - a_u * a_u) * sin(a_phi); + case (2): + return center + a_radius * a_u; + default: + MayDay::Error("SphericalGeometryUniform: Direction not supported"); + } + } + + static const IntegrationMethod & + get_recommended_integration_method_u(int a_num_points_u) + { + static const IntegrationMethod &milne_regularized = + IntegrationMethod::milne_regularized; + static const IntegrationMethod &midpoint = IntegrationMethod::midpoint; + if (milne_regularized.is_valid(a_num_points_u, is_u_periodic())) + return milne_regularized; + MayDay::Warning( + "Use a multipel of 3 in 'u' points to use milne's rule. " + "Defaulting to midpoint."); + return midpoint; + } + + static const IntegrationMethod & + get_recommended_integration_method_v(int a_num_points_phi) + { + static const IntegrationMethod &simpson = IntegrationMethod::simpson; + static const IntegrationMethod &trapezium = + IntegrationMethod::trapezium; + if (simpson.is_valid(a_num_points_phi, is_v_periodic())) + return simpson; + MayDay::Warning("Use an even number of 'v' points to use simpson rule. " + "Defaulting to trapezium."); + return trapezium; + } + + protected: + std::array m_center; + SphericalGeometry::UP_DIR m_up_dir; +}; + +#endif /* SPHERICALGEOMETRYUNIFORM_HPP_ */ diff --git a/Source/AMRInterpolator/SurfaceExtraction.hpp b/Source/AMRInterpolator/SurfaceExtraction.hpp index eb444f130..267a06107 100644 --- a/Source/AMRInterpolator/SurfaceExtraction.hpp +++ b/Source/AMRInterpolator/SurfaceExtraction.hpp @@ -28,43 +28,43 @@ // Chombo namespace #include "UsingNamespace.H" +struct surface_extraction_params_t +{ + int num_surfaces; //!< number of surfaces over which to extraction + std::vector + surface_param_values; //!< the values of the + //!< parameter that gives the required + //!< surfaces with SurfaceGeom geometry (e.g. + //!< radii for spherical shells) + int num_points_u; //!< the number of points for the first parameter + //!< that parameterises each surface + int num_points_v; //!< the number of points for the second parameter + //!< that parameterises each surfaces + std::vector extraction_levels; //!< the level on which to do the + //!< extraction for each surface + bool write_extraction; //!< whether or not to write the extracted data + + std::string extraction_prefix; + + int min_extraction_level() + { + return *(std::min_element(extraction_levels.begin(), + extraction_levels.end())); + } +}; + //! This class extracts grid variables on 2 dimensional surfaces each //! parameterised by u and v with different surfaces given by level sets of //! another parameter template class SurfaceExtraction { public: - struct params_t - { - int num_surfaces; //!< number of surfaces over which to extraction - std::vector - surface_param_values; //!< the values of the - //!< parameter that gives the required - //!< surfaces with SurfaceGeom geometry (e.g. - //!< radii for spherical shells) - int num_points_u; //!< the number of points for the first parameter - //!< that parameterises each surface - int num_points_v; //!< the number of points for the second parameter - //!< that parameterises each surfaces - std::vector extraction_levels; //!< the level on which to do the - //!< extraction for each surface - bool write_extraction; //!< whether or not to write the extracted data - - std::string extraction_prefix; - - int min_extraction_level() - { - return *(std::min_element(extraction_levels.begin(), - extraction_levels.end())); - } - }; - using vars_t = std::tuple; protected: const SurfaceGeometry m_geom; //!< the geometry class which knows about //!< the particular surface - const params_t m_params; + const surface_extraction_params_t m_params; std::vector> m_vars; //!< the vector of pairs of //!< variables and derivatives to extract @@ -110,8 +110,9 @@ template class SurfaceExtraction public: //! Normal constructor which requires vars to be added after construction //! using add_var or add_vars - SurfaceExtraction(const SurfaceGeometry &a_geom, const params_t &a_params, - double a_dt, double a_time, bool a_first_step, + SurfaceExtraction(const SurfaceGeometry &a_geom, + const surface_extraction_params_t &a_params, double a_dt, + double a_time, bool a_first_step, double a_restart_time = 0.0); //! add a single variable or derivative of variable @@ -131,14 +132,16 @@ template class SurfaceExtraction //! Alternative constructor with a predefined vector of variables and //! derivatives SurfaceExtraction( - const SurfaceGeometry &a_geom, const params_t &a_params, + const SurfaceGeometry &a_geom, + const surface_extraction_params_t &a_params, const std::vector> &a_vars, double a_dt, double a_time, bool a_first_step, double a_restart_time = 0.0); //! Another alternative constructor with a predefined vector of variables //! no derivatives - SurfaceExtraction(const SurfaceGeometry &a_geom, const params_t &a_params, + SurfaceExtraction(const SurfaceGeometry &a_geom, + const surface_extraction_params_t &a_params, const std::vector &a_vars, double a_dt, double a_time, bool a_first_step, double a_restart_time = 0.0); diff --git a/Source/AMRInterpolator/SurfaceExtraction.impl.hpp b/Source/AMRInterpolator/SurfaceExtraction.impl.hpp index ccfe05ffb..d2c2dcee7 100644 --- a/Source/AMRInterpolator/SurfaceExtraction.impl.hpp +++ b/Source/AMRInterpolator/SurfaceExtraction.impl.hpp @@ -14,8 +14,8 @@ //! using add_var or add_vars template SurfaceExtraction::SurfaceExtraction( - const SurfaceGeometry &a_geom, const params_t &a_params, double a_dt, - double a_time, bool a_first_step, double a_restart_time) + const SurfaceGeometry &a_geom, const surface_extraction_params_t &a_params, + double a_dt, double a_time, bool a_first_step, double a_restart_time) : m_geom(a_geom), m_params(a_params), m_dt(a_dt), m_time(a_time), m_first_step(a_first_step), m_restart_time(a_restart_time), m_num_interp_points((procID() == 0) @@ -112,7 +112,7 @@ void SurfaceExtraction::add_diagnostic_vars( //! derivatives template SurfaceExtraction::SurfaceExtraction( - const SurfaceGeometry &a_geom, const params_t &a_params, + const SurfaceGeometry &a_geom, const surface_extraction_params_t &a_params, const std::vector> &a_vars, double a_dt, double a_time, bool a_first_step, double a_restart_time) : SurfaceExtraction(a_geom, a_params, a_dt, a_time, @@ -125,7 +125,7 @@ SurfaceExtraction::SurfaceExtraction( //! no derivatives template SurfaceExtraction::SurfaceExtraction( - const SurfaceGeometry &a_geom, const params_t &a_params, + const SurfaceGeometry &a_geom, const surface_extraction_params_t &a_params, const std::vector &a_vars, double a_dt, double a_time, bool a_first_step, double a_restart_time) : SurfaceExtraction(a_geom, a_params, a_dt, a_time, diff --git a/Source/ApparentHorizonFinder/AHInterpolation.hpp b/Source/ApparentHorizonFinder/AHInterpolation.hpp index a6ae0576c..4c238b97c 100644 --- a/Source/ApparentHorizonFinder/AHInterpolation.hpp +++ b/Source/ApparentHorizonFinder/AHInterpolation.hpp @@ -61,36 +61,16 @@ template class AHInterpolation const AMRInterpolator> *get_interpolator() const; const SurfaceGeometry &get_coord_system() const; - // several of the methods below just call the correspondent method of the - // CoordSystem, as these are needed in the 'ApparentHorizon' class - bool is_u_periodic() const; - double get_domain_u_min() const; //!< lower bound of (u,v) coordinates - double get_domain_u_max() const; //!< upper bound of (u,v) coordinates -#if CH_SPACEDIM == 3 - bool is_v_periodic() const; - double get_domain_v_min() const; //!< lower bound of (u,v) coordinates - double get_domain_v_max() const; //!< upper bound of (u,v) coordinates -#endif - std::vector get_labels() const; //!< get all names (u, v, f) void set_origin( const std::array &); //!< set origin of CoordSystem - const std::array & - get_origin() const; //!< get origin of CoordSystem void refresh_interpolator( bool printing_step, const std::map> &extra_vars); //!< refresh AMRInterpolator 'm_interpolator' - double get_grid_coord(int a_dir, double f, double u -#if CH_SPACEDIM == 3 - , - double v -#endif - ) const; //!< transform from spherical to cartesian - //! returns whether any pointis outside of grid (==> diverging) bool set_coordinates(const std::vector &f, const std::vector &u, diff --git a/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp b/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp index fa5b2b558..940cb6cf2 100644 --- a/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp +++ b/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp @@ -90,40 +90,6 @@ AHInterpolation::get_coord_system() const return m_coord_system; } -template -bool AHInterpolation::is_u_periodic() const -{ - return m_coord_system.is_u_periodic(); -} -template -double AHInterpolation::get_domain_u_min() const -{ - return m_coord_system.get_domain_u_min(); -} -template -double AHInterpolation::get_domain_u_max() const -{ - return m_coord_system.get_domain_u_max(); -} - -#if CH_SPACEDIM == 3 -template -bool AHInterpolation::is_v_periodic() const -{ - return m_coord_system.is_v_periodic(); -} -template -double AHInterpolation::get_domain_v_min() const -{ - return m_coord_system.get_domain_v_min(); -} -template -double AHInterpolation::get_domain_v_max() const -{ - return m_coord_system.get_domain_v_max(); -} -#endif - template std::vector AHInterpolation::get_labels() const @@ -145,13 +111,6 @@ void AHInterpolation::set_origin( m_coord_system.set_origin(origin); } -template -const std::array & -AHInterpolation::get_origin() const -{ - return m_coord_system.get_origin(); -} - template bool AHInterpolation::is_in_grid( const std::array &a_origin, double a_initial_guess) @@ -256,26 +215,6 @@ bool AHInterpolation::fit_in_grid(double &x, return out_of_grid; } -template -double AHInterpolation::get_grid_coord(int a_dir, - double f, - double u -#if CH_SPACEDIM == 3 - , - double v -#endif -) const -{ - CH_TIME("AHInterpolation::get_grid_coord"); - - return m_coord_system.get_grid_coord(a_dir, f, u -#if CH_SPACEDIM == 3 - , - v -#endif - ); -} - //! triplet of functions to be used together in blocks of code that require //! PETSc AND AMRInterpolator to interpolate 'keep_interpolating_if_inactive' //! returns 'true' immediately for PETSc ranks for non-PETSc ranks, it loops in diff --git a/Source/ApparentHorizonFinder/AHSphericalGeometryUniform.hpp b/Source/ApparentHorizonFinder/AHSphericalGeometryUniform.hpp new file mode 100644 index 000000000..00396aafa --- /dev/null +++ b/Source/ApparentHorizonFinder/AHSphericalGeometryUniform.hpp @@ -0,0 +1,172 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef AHSPHERICALGEOMETRYUNIFORM_HPP_ +#define AHSPHERICALGEOMETRYUNIFORM_HPP_ + +#include "DimensionDefinitions.hpp" // make sure GR_SPACEDIM exists + +#if GR_SPACEDIM != 3 +#error "This file should only be included for GR_SPACEDIM == 3." +#endif + +// Chombo includes +#include "CH_Timer.H" + +// Other includes +#include "AHGeometryData.hpp" +#include "AlwaysInline.hpp" +#include "SphericalGeometryUniform.hpp" +#include +#include + +// Chombo namespace +#include "UsingNamespace.H" + +//! This SurfaceGeometry template class provides spherical shell geometry +//! implementation for the SurfaceExtraction class +//! u = theta, v = phi +class AHSphericalGeometryUniform : public SphericalGeometryUniform +{ + public: + AHSphericalGeometryUniform(const std::array &a_origin) + : SphericalGeometryUniform(a_origin) + { + } + + ALWAYS_INLINE void + set_origin(const std::array &a_origin) + { + m_center = a_origin; + } + ALWAYS_INLINE const std::array &get_origin() const + { + return m_center; + } + + AHGeometryData get_geometry_data(double r, double u, double phi = 0.) const + { + // for 2D Cartoon spherical geometry (3D->2D), this should always be + // called with no 3rd argument (so phi=0), which means the formulas + // below are effectively polar coordinates + + CH_TIME("AHSphericalGeometryUniform::get_geometry_data"); + + double sqrt1mu2 = sqrt(1. - u * u); + + double cosphi = cos(phi); + double sinphi = sin(phi); + + double cos2phi = cos(2. * phi); + double sin2phi = sin(2. * phi); + + // in 3D, (x,y,z) are normal + // in 2D Cartoon method, the spherical 'z' axis is the grid 'x' axis, so + // the 2D coordinates (x,y) are (z,x) spherical coordinates + // (that means commenting out everything related to 'y' and to 'v' + // [=phi]) + int iz = m_up_dir; + int ix = (m_up_dir + 1) % 3; +#if CH_SPACEDIM == 3 + int iy = (m_up_dir + 2) % 3; +#endif + + AHGeometryData out; + + out.du[ix] = -u * sqrt1mu2 * cosphi / r; +#if CH_SPACEDIM == 3 + out.du[iy] = -u * sqrt1mu2 * sinphi / r; +#endif + out.du[iz] = (1. - u * u) / r; + +#if CH_SPACEDIM == 3 + out.dv[ix] = (-sinphi) / (r * sqrt1mu2); + out.dv[iy] = (cosphi) / (r * sqrt1mu2); + out.dv[iz] = 0; +#endif + + double dfdx = sqrt1mu2 * cosphi; +#if CH_SPACEDIM == 3 + double dfdy = sqrt1mu2 * sinphi; +#endif + double dfdz = u; + + out.ddu[ix][ix] = -u * + (-1. + 3. * u * u + 3. * (-1. + u * u) * cos2phi) / + (r * r * 2.); + out.ddu[ix][iz] = sqrt1mu2 * (-1. + 3. * u * u) * cosphi / (r * r); + out.ddu[iz][ix] = out.ddu[ix][iz]; + out.ddu[iz][iz] = 3. * u * (-1. + u * u) / (r * r); +#if CH_SPACEDIM == 3 + out.ddu[ix][iy] = -3. * u * (-1. + u * u) * cosphi * sinphi / (r * r); + out.ddu[iy][iy] = + u * (1. - 3. * u * u + 3. * (-1. + u * u) * cos2phi) / (2. * r * r); + out.ddu[iy][iz] = sqrt1mu2 * (-1. + 3. * u * u) * sinphi / (r * r); + out.ddu[iy][ix] = out.ddu[ix][iy]; + out.ddu[iz][iy] = out.ddu[iy][iz]; +#endif + +#if CH_SPACEDIM == 3 + out.ddv[ix][ix] = -sin2phi / (r * r * (-1. + u * u)); + out.ddv[ix][iz] = 0.; + out.ddv[iz][ix] = out.ddv[ix][iz]; + out.ddv[iz][iz] = 0.; + out.ddv[ix][iy] = cos2phi / (r * r * (-1. + u * u)); + out.ddv[iy][iy] = sin2phi / (r * r * (-1. + u * u)); + out.ddv[iy][iz] = 0.; + out.ddv[iy][ix] = out.ddv[ix][iy]; + out.ddv[iz][iy] = out.ddv[iy][iz]; +#endif + + double ddfdxdx = (u * u - (-1. + u * u) * sinphi * sinphi) / r; + double ddfdxdz = -(u * sqrt1mu2 * cosphi) / r; + double ddfdzdz = (1. - u * u) / r; +#if CH_SPACEDIM == 3 + double ddfdxdy = (-1. + u * u) * cosphi * sinphi / r; + double ddfdydy = (u * u - (-1. + u * u) * cosphi * cosphi) / r; + double ddfdydz = -(u * sqrt1mu2 * sinphi) / r; +#endif + + out.df[ix] = dfdx; +#if CH_SPACEDIM == 3 + out.df[iy] = dfdy; +#endif + out.df[iz] = dfdz; + + out.ddf[ix][ix] = ddfdxdx; + out.ddf[ix][iz] = ddfdxdz; + out.ddf[iz][ix] = out.ddf[ix][iz]; + out.ddf[iz][iz] = ddfdzdz; +#if CH_SPACEDIM == 3 + out.ddf[ix][iy] = ddfdxdy; + out.ddf[iy][iy] = ddfdydy; + out.ddf[iy][iz] = ddfdydz; + out.ddf[iy][ix] = out.ddf[ix][iy]; + out.ddf[iz][iy] = out.ddf[iy][iz]; +#endif + + out.dxdu[ix] = -r * u / sqrt1mu2 * cosphi; +#if CH_SPACEDIM == 3 + out.dxdu[iy] = -r * u / sqrt1mu2 * sinphi; +#endif + out.dxdu[iz] = r; + +#if CH_SPACEDIM == 3 + out.dxdv[ix] = -r * sqrt1mu2 * sinphi; + out.dxdv[iy] = r * sqrt1mu2 * cosphi; + out.dxdv[iz] = 0.; +#endif + + out.dxdf[ix] = sqrt1mu2 * cosphi; +#if CH_SPACEDIM == 3 + out.dxdf[iy] = sqrt1mu2 * sinphi; +#endif + out.dxdf[iz] = u; + + return out; + } +}; + +#endif /* AHSPHERICALGEOMETRYUNIFORM_HPP_ */ diff --git a/Source/ApparentHorizonFinder/AHStringGeometry.hpp b/Source/ApparentHorizonFinder/AHStringGeometry.hpp index 4cb410733..09bc8208a 100644 --- a/Source/ApparentHorizonFinder/AHStringGeometry.hpp +++ b/Source/ApparentHorizonFinder/AHStringGeometry.hpp @@ -17,6 +17,7 @@ // Other includes #include "AHGeometryData.hpp" #include "AlwaysInline.hpp" +#include "IntegrationMethod.hpp" #include // Chombo namespace @@ -45,16 +46,30 @@ class AHStringGeometry return fake_origin; } - ALWAYS_INLINE double get_domain_u_min() const { return 0.; } + static ALWAYS_INLINE double get_domain_u_min() { return 0.; } ALWAYS_INLINE double get_domain_u_max() const { return m_string_length; } - ALWAYS_INLINE bool is_u_periodic() const { return true; } + //! returns the grid spacing in u + ALWAYS_INLINE double du(int a_num_points_u) const + { + return (get_domain_u_max() - get_domain_u_min()) / + (double)(a_num_points_u); + } + + //! returns the u coordinate associated to the u/u index + ALWAYS_INLINE double u(int a_iu, int a_num_points_u) const + { + return a_iu * du(a_num_points_u) + get_domain_u_min(); + } - ALWAYS_INLINE std::string param_name() const { return "y"; } + static ALWAYS_INLINE bool is_u_periodic() { return true; } - ALWAYS_INLINE std::string u_name() const { return "x"; } + static ALWAYS_INLINE std::string param_name() { return "y"; } - ALWAYS_INLINE double get_grid_coord(int a_dir, double a_y, double a_x) const + static ALWAYS_INLINE std::string u_name() { return "x"; } + + static ALWAYS_INLINE double get_grid_coord(int a_dir, double a_y, + double a_x) { switch (a_dir) { @@ -67,7 +82,20 @@ class AHStringGeometry } } - AHGeometryData get_geometry_data(double y, double x) const + static const IntegrationMethod & + get_recommended_integration_method_u(int a_num_points_u) + { + static const IntegrationMethod &simpson = IntegrationMethod::simpson; + static const IntegrationMethod &trapezium = + IntegrationMethod::trapezium; + if (simpson.is_valid(a_num_points_u, is_u_periodic())) + return simpson; + MayDay::Warning("Use an odd number of 'u' points to use simpson rule. " + "Defaulting to trapezium."); + return trapezium; + } + + static AHGeometryData get_geometry_data(double y, double x) { CH_TIME("AHStringGeometry::get_geometry_data"); diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp index 4830ee4d6..72f022ac3 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp @@ -62,10 +62,13 @@ ApparentHorizon::ApparentHorizon( m_max_F(0.), m_min_F(0.), m_ave_F(0.), m_std_F(0.), m_integration_methods({ + a_interp.get_coord_system().get_recommended_integration_method_u( + a_params.num_points_u) #if CH_SPACEDIM == 3 - IntegrationMethod::simpson, + , + a_interp.get_coord_system().get_recommended_integration_method_v( + a_params.num_points_v) #endif - IntegrationMethod::simpson }), m_area(NAN), m_spin(NAN), m_mass(NAN), m_irreducible_mass(NAN), @@ -75,12 +78,12 @@ ApparentHorizon::ApparentHorizon( m_interp(a_interp), m_interp_plus(a_interp), m_interp_minus(a_interp), - m_periodic_u(a_interp.is_u_periodic()), + m_periodic_u(a_interp.get_coord_system().is_u_periodic()), m_num_global_u(a_params.num_points_u) #if CH_SPACEDIM == 3 , - m_periodic_v(a_interp.is_v_periodic()), + m_periodic_v(a_interp.get_coord_system().is_v_periodic()), m_num_global_v(a_params.num_points_v) #endif { @@ -135,7 +138,7 @@ template const std::array & ApparentHorizon::get_origin() const { - return m_interp.get_origin(); + return m_interp.get_coord_system().get_origin(); } template @@ -236,7 +239,7 @@ void ApparentHorizon::reset_initial_guess() if (!AHFinder::is_rank_active()) return; - auto origin = m_interp.get_origin(); + auto origin = get_origin(); // verify origin +- initial guess is inside the grid bool out_of_grid = m_interp.is_in_grid(origin, m_initial_guess); @@ -1320,10 +1323,12 @@ void ApparentHorizon::check_integration_methods() { // check if integration methods are valid given periodicity and number of // points - bool valid_u = m_integration_methods[0].is_valid(m_params.num_points_u, - m_interp.is_u_periodic()); + bool valid_u = m_integration_methods[0].is_valid( + m_params.num_points_u, m_interp.get_coord_system().is_u_periodic()); - IntegrationMethod method_default = IntegrationMethod::trapezium; + const IntegrationMethod &method_default_u = + m_interp.get_coord_system().get_recommended_integration_method_u( + m_num_global_u); if (!valid_u) { std::string warn = @@ -1332,12 +1337,15 @@ void ApparentHorizon::check_integration_methods() "Reverting to trapezium rule."; MayDay::Warning(warn.c_str()); pout() << warn << std::endl; - m_integration_methods[0] = method_default; + m_integration_methods[0] = method_default_u; } #if CH_SPACEDIM == 3 - bool valid_v = m_integration_methods[1].is_valid(m_params.num_points_v, - m_interp.is_v_periodic()); + bool valid_v = m_integration_methods[1].is_valid( + m_params.num_points_v, m_interp.get_coord_system().is_v_periodic()); + const IntegrationMethod &method_default_v = + m_interp.get_coord_system().get_recommended_integration_method_v( + m_num_global_v); if (!valid_v) { std::string warn = @@ -1346,7 +1354,7 @@ void ApparentHorizon::check_integration_methods() "Reverting to trapezium rule."; MayDay::Warning(warn.c_str()); pout() << warn << std::endl; - m_integration_methods[1] = method_default; + m_integration_methods[1] = method_default_v; } #endif } @@ -1418,7 +1426,8 @@ ApparentHorizon::calculate_spin_dimensionless( CH_assert(norm2 >= 0.); double weight = m_integration_methods[1].weight( - v, m_params.num_points_v, m_interp.is_v_periodic()); + v, m_params.num_points_v, + m_interp.get_coord_system().is_v_periodic()); integral += sqrt(norm2) * weight * m_dv; } @@ -1569,7 +1578,8 @@ ApparentHorizon::calculate_angular_momentum_J() double det = TensorAlgebra::compute_determinant(g_horizon); double weight = m_integration_methods[0].weight( - u, m_params.num_points_u, m_interp.is_u_periodic()); + u, m_params.num_points_u, + m_interp.get_coord_system().is_u_periodic()); FOR1(a) { @@ -1587,7 +1597,8 @@ ApparentHorizon::calculate_angular_momentum_J() idx++; } double weight = m_integration_methods[1].weight( - v, m_params.num_points_v, m_interp.is_v_periodic()); + v, m_params.num_points_v, + m_interp.get_coord_system().is_v_periodic()); FOR1(a) { integrals[a] += weight * m_dv * inner_integral[a]; } } @@ -1703,7 +1714,8 @@ double ApparentHorizon::calculate_area() double det = TensorAlgebra::compute_determinant(g_horizon); double weight = m_integration_methods[0].weight( - u, m_params.num_points_u, m_interp.is_u_periodic()); + u, m_params.num_points_u, + m_interp.get_coord_system().is_u_periodic()); double element = sqrt(det) * weight * m_du; @@ -1720,7 +1732,8 @@ double ApparentHorizon::calculate_area() } #if CH_SPACEDIM == 3 double weight = m_integration_methods[1].weight( - v, m_params.num_points_v, m_interp.is_v_periodic()); + v, m_params.num_points_v, + m_interp.get_coord_system().is_v_periodic()); integral += weight * m_dv * inner_integral; #elif CH_SPACEDIM == 2 integral += inner_integral; @@ -1802,13 +1815,13 @@ ApparentHorizon::calculate_center() { for (unsigned i = 0; i < CH_SPACEDIM; ++i) { - double coord_i = - m_interp.get_grid_coord(i, m_F[idx], m_u[idx] + double coord_i = m_interp.get_coord_system().get_grid_coord( + i, m_F[idx], m_u[idx] #if CH_SPACEDIM == 3 - , - m_v[idx] + , + m_v[idx] #endif - ); + ); // temp[i] += point[i]; // old method if (coord_i > max_temp[i]) diff --git a/Source/ApparentHorizonFinder/ApparentHorizon_petsc.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon_petsc.impl.hpp index 91585d733..c5a7bee6d 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon_petsc.impl.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon_petsc.impl.hpp @@ -33,13 +33,6 @@ void ApparentHorizon::initialise_PETSc() if (!AHFinder::is_rank_active()) return; - const double lo_u = m_interp.get_domain_u_min(); - const double hi_u = m_interp.get_domain_u_max(); -#if CH_SPACEDIM == 3 // lo_v, hi_v not used otherwise - const double lo_v = m_interp.get_domain_v_min(); - const double hi_v = m_interp.get_domain_v_max(); -#endif - #if PETSC_VERSION_LT(3, 5, 0) #define DM_BOUNDARY_PERIODIC DMDA_BOUNDARY_PERIODIC #define DM_BOUNDARY_GHOSTED DMDA_BOUNDARY_PERIODIC @@ -68,8 +61,7 @@ void ApparentHorizon::initialise_PETSc() DMSetUp(m_dmda); - // reload -> is it the same value? -#if CH_SPACEDIM == 3 // lo_v, hi_v not used otherwise +#if CH_SPACEDIM == 3 DMDAGetInfo(m_dmda, NULL, &m_num_global_u, &m_num_global_v, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); #elif CH_SPACEDIM == 2 @@ -77,12 +69,10 @@ void ApparentHorizon::initialise_PETSc() NULL, NULL, NULL, NULL, NULL, NULL); #endif - m_du = - (hi_u - lo_u) / (m_periodic_u ? m_num_global_u : (m_num_global_u - 1)); + m_du = m_interp.get_coord_system().du(m_num_global_u); #if CH_SPACEDIM == 3 - m_dv = - (hi_v - lo_v) / (m_periodic_v ? m_num_global_v : (m_num_global_v - 1)); + m_dv = m_interp.get_coord_system().dv(m_num_global_v); #endif #if CH_SPACEDIM == 3 @@ -112,10 +102,12 @@ void ApparentHorizon::initialise_PETSc() for (int u = m_umin; u < m_umax; ++u) { #if CH_SPACEDIM == 3 - m_u[(v - m_vmin) * m_nu + (u - m_umin)] = lo_u + u * m_du; - m_v[(v - m_vmin) * m_nu + (u - m_umin)] = lo_v + v * m_dv; + m_u[(v - m_vmin) * m_nu + (u - m_umin)] = + m_interp.get_coord_system().u(u, m_num_global_u); + m_v[(v - m_vmin) * m_nu + (u - m_umin)] = + m_interp.get_coord_system().v(v, m_num_global_v); #elif CH_SPACEDIM == 2 - m_u[u - m_umin] = lo_u + u * m_du; + m_u[u - m_umin] = m_interp.get_coord_system().u(u, m_num_global_u); #endif } } @@ -202,7 +194,6 @@ bool ApparentHorizon::interpolate_ah( // check if number of points changed bool points_changed = false; const double du = old_coords[0][1] - old_coords[0][0]; - const double lo_u = m_interp.get_domain_u_min(); #if CH_SPACEDIM == 3 // assumes ordering in file is done first by fixed 'v', varying 'u' @@ -214,7 +205,6 @@ bool ApparentHorizon::interpolate_ah( int old_number_of_v = old_coords[0].size() / old_number_of_u; const double dv = old_coords[1][old_number_of_u] - old_coords[1][old_number_of_u - 1]; - const double lo_v = m_interp.get_domain_v_min(); // int total_new_points = m_num_global_u * m_num_global_v; points_changed |= (old_number_of_v != m_num_global_v); @@ -246,9 +236,9 @@ bool ApparentHorizon::interpolate_ah( { #if CH_SPACEDIM == 3 int idx_global = v * m_num_global_u + u; + double val_v = m_interp.get_coord_system().v(v, m_num_global_v); int idx_local = (v - m_vmin) * m_nu + (u - m_umin); - if (std::abs((lo_v + v * m_dv) - old_coords[1][idx_global]) > - 1.e-7) + if (std::abs(val_v - old_coords[1][idx_global]) > 1.e-7) MayDay::Error( "'v' coordinates incompatible with restart data"); #elif CH_SPACEDIM == 2 @@ -256,8 +246,8 @@ bool ApparentHorizon::interpolate_ah( int idx_local = (u - m_umin); #endif - if (std::abs((lo_u + u * m_du) - old_coords[0][idx_global]) > - 1.e-7) + double val_u = m_interp.get_coord_system().u(u, m_num_global_u); + if (std::abs(val_u - old_coords[0][idx_global]) > 1.e-7) MayDay::Error( "'u' coordinates incompatible with restart data"); @@ -286,16 +276,18 @@ bool ApparentHorizon::interpolate_ah( #if CH_SPACEDIM == 3 SimpleArrayBox box( {old_number_of_u, old_number_of_v}, old_coords[CH_SPACEDIM - 1], - {m_interp.is_u_periodic(), m_interp.is_v_periodic()}); + {m_interp.get_coord_system().is_u_periodic(), + m_interp.get_coord_system().is_v_periodic()}); SimpleInterpSource source( {old_number_of_u, old_number_of_v}, - {m_interp.is_u_periodic(), m_interp.is_v_periodic()}); + {m_interp.get_coord_system().is_u_periodic(), + m_interp.get_coord_system().is_v_periodic()}); #elif CH_SPACEDIM == 2 - SimpleArrayBox box({old_number_of_u}, - old_coords[CH_SPACEDIM - 1], - {m_interp.is_u_periodic()}); - SimpleInterpSource source({old_number_of_u}, - {m_interp.is_u_periodic()}); + SimpleArrayBox box( + {old_number_of_u}, old_coords[CH_SPACEDIM - 1], + {m_interp.get_coord_system().is_u_periodic()}); + SimpleInterpSource source( + {old_number_of_u}, {m_interp.get_coord_system().is_u_periodic()}); #endif const int Order = 4; @@ -314,7 +306,7 @@ bool ApparentHorizon::interpolate_ah( #if CH_SPACEDIM == 3 for (int v = m_vmin; v < m_vmax; ++v) { - double v_val = lo_v + v * m_dv; + double v_val = m_interp.get_coord_system().v(v, m_num_global_v); double v_old_idx = v_val / dv; // round such that we don't get AMRInterpolator errors // (e.g. the last point idx=N-1 should be exact, and not being exact was @@ -327,7 +319,7 @@ bool ApparentHorizon::interpolate_ah( #endif for (int u = m_umin; u < m_umax; ++u) { - double u_val = lo_u + u * m_du; + double u_val = m_interp.get_coord_system().u(u, m_num_global_u); double u_old_idx = u_val / du; // round such that we don't get AMRInterpolator errors (same as for // 'v') diff --git a/Source/GRChomboCore/SimulationParametersBase.hpp b/Source/GRChomboCore/SimulationParametersBase.hpp index 7919295d1..727dfd38e 100644 --- a/Source/GRChomboCore/SimulationParametersBase.hpp +++ b/Source/GRChomboCore/SimulationParametersBase.hpp @@ -19,7 +19,7 @@ #include "SphericalExtraction.hpp" // add this type alias here for backwards compatibility -using extraction_params_t = SphericalExtraction::params_t; +using extraction_params_t = spherical_extraction_params_t; class SimulationParametersBase : public ChomboParameters { @@ -293,7 +293,7 @@ class SimulationParametersBase : public ChomboParameters // If you are using a different gauge, you need to load your parameters // in your own SimulationParameters class. CCZ4_params_t<> ccz4_params; - SphericalExtraction::params_t extraction_params; + spherical_extraction_params_t extraction_params; #ifdef USE_AHFINDER bool AH_activate; diff --git a/Source/TaggingCriteria/ChiExtractionTaggingCriterion.hpp b/Source/TaggingCriteria/ChiExtractionTaggingCriterion.hpp index f1cdc9181..d822013fb 100644 --- a/Source/TaggingCriteria/ChiExtractionTaggingCriterion.hpp +++ b/Source/TaggingCriteria/ChiExtractionTaggingCriterion.hpp @@ -20,7 +20,7 @@ class ChiExtractionTaggingCriterion protected: const double m_dx; const FourthOrderDerivatives m_deriv; - const SphericalExtraction::params_t m_params; + const spherical_extraction_params_t m_params; const int m_level; const bool m_activate_extraction; @@ -40,14 +40,14 @@ class ChiExtractionTaggingCriterion // The constructor ChiExtractionTaggingCriterion(const double dx, const int a_level, - const SphericalExtraction::params_t a_params, + const spherical_extraction_params_t a_params, const bool activate_extraction = false) : m_dx(dx), m_deriv(dx), m_params(a_params), m_level(a_level), m_activate_extraction(activate_extraction){}; // below is a constructor for backward compatibility ChiExtractionTaggingCriterion(const double dx, const int a_level, const int a_max_level, - const SphericalExtraction::params_t a_params, + const spherical_extraction_params_t a_params, const bool activate_extraction = false) : ChiExtractionTaggingCriterion(dx, a_level, a_params, activate_extraction){}; diff --git a/Source/TaggingCriteria/ChiPunctureExtractionTaggingCriterion.hpp b/Source/TaggingCriteria/ChiPunctureExtractionTaggingCriterion.hpp index f5d00b8c3..72c9476b6 100644 --- a/Source/TaggingCriteria/ChiPunctureExtractionTaggingCriterion.hpp +++ b/Source/TaggingCriteria/ChiPunctureExtractionTaggingCriterion.hpp @@ -26,7 +26,7 @@ class ChiPunctureExtractionTaggingCriterion const bool m_track_punctures; const bool m_activate_extraction; const FourthOrderDerivatives m_deriv; - const SphericalExtraction::params_t m_params; + const spherical_extraction_params_t m_params; const std::vector m_puncture_masses; const std::vector> &m_puncture_coords; @@ -47,7 +47,7 @@ class ChiPunctureExtractionTaggingCriterion // The constructor ChiPunctureExtractionTaggingCriterion( const double dx, const int a_level, const int a_max_level, - const SphericalExtraction::params_t a_params, + const spherical_extraction_params_t a_params, const std::vector> &a_puncture_coords, const bool activate_extraction = false, const bool track_punctures = false, diff --git a/Source/utils/WeylExtraction.hpp b/Source/utils/WeylExtraction.hpp index 64b809e69..89616a58f 100644 --- a/Source/utils/WeylExtraction.hpp +++ b/Source/utils/WeylExtraction.hpp @@ -19,7 +19,7 @@ class WeylExtraction : public SphericalExtraction { public: //! The constructor - WeylExtraction(SphericalExtraction::params_t &a_params, double a_dt, + WeylExtraction(spherical_extraction_params_t &a_params, double a_dt, double a_time, bool a_first_step, double a_restart_time = 0.0) : SphericalExtraction(a_params, a_dt, a_time, a_first_step, @@ -31,7 +31,7 @@ class WeylExtraction : public SphericalExtraction //! The old constructor which assumes it is called in specificPostTimeStep //! so the first time step is when m_time == m_dt - WeylExtraction(SphericalExtraction::params_t a_params, double a_dt, + WeylExtraction(spherical_extraction_params_t a_params, double a_dt, double a_time, double a_restart_time = 0.0) : WeylExtraction(a_params, a_dt, a_time, (a_dt == a_time), a_restart_time) diff --git a/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.cpp b/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.cpp index 43df00de3..d070b325d 100755 --- a/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.cpp +++ b/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.cpp @@ -60,7 +60,16 @@ int runApparentHorizonTest3D(int argc, char *argv[]) int status = 0; #ifdef USE_AHFINDER - AHFinder::params AH_params = {1, 15, 20, 1, 1, false, false, 0, 0., -1.}; + AHFinder::params AH_params = {1, + sim_params.AH_num_points_u, + sim_params.AH_num_points_v, + 1, + 1, + false, + false, + 0, + 0., + -1.}; AH_params.verbose = 3; // Set up interpolator and PETSc subcommunicator when AH extraction is @@ -70,7 +79,7 @@ int runApparentHorizonTest3D(int argc, char *argv[]) sim_params.verbosity); bh_amr.set_interpolator(&interpolator); - AHSphericalGeometry sph(sim_params.center); + AHSurfaceGeometry sph(sim_params.center); bh_amr.m_ah_finder.add_ah(sph, sim_params.initial_guess, AH_params); if (!bh_amr.m_ah_finder.get(0)->get_converged()) diff --git a/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.inputs b/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.inputs index 422bd65d5..083e4aafe 100755 --- a/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.inputs +++ b/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.inputs @@ -30,4 +30,7 @@ block_factor = 16 kerr_mass = 10. kerr_spin = 5. kerr_spin_direction = 1. 0. 0. -#initial_guess = 5 +#AH_initial_guess = 5 + +AH_num_points_u = 21 +AH_num_points_v = 20 \ No newline at end of file diff --git a/Tests/ApparentHorizonFinderTest3D/SimulationParameters.hpp b/Tests/ApparentHorizonFinderTest3D/SimulationParameters.hpp index 9d75d4708..5578ce6ca 100755 --- a/Tests/ApparentHorizonFinderTest3D/SimulationParameters.hpp +++ b/Tests/ApparentHorizonFinderTest3D/SimulationParameters.hpp @@ -30,13 +30,16 @@ class SimulationParameters : public ChomboParameters {0., 0., 1.}); #ifdef USE_AHFINDER - pp.load("initial_guess", initial_guess, kerr_params.mass * 0.5); + pp.load("AH_initial_guess", initial_guess, kerr_params.mass * 0.5); + pp.load("AH_num_points_u", AH_num_points_u); + pp.load("AH_num_points_v", AH_num_points_v); #endif } KerrBH::params_t kerr_params; #ifdef USE_AHFINDER double initial_guess; + int AH_num_points_u, AH_num_points_v; #endif }; diff --git a/Tests/SphericalExtractionTest/SimulationParameters.hpp b/Tests/SphericalExtractionTest/SimulationParameters.hpp index ada0151e1..61289c3c0 100644 --- a/Tests/SphericalExtractionTest/SimulationParameters.hpp +++ b/Tests/SphericalExtractionTest/SimulationParameters.hpp @@ -51,7 +51,7 @@ class SimulationParameters : public ChomboParameters } public: - SphericalExtraction::params_t extraction_params_lo; + spherical_extraction_params_t extraction_params_lo; int es, el, em; // spherical harmonic params }; diff --git a/Tests/SphericalExtractionTest/SphericalExtractionTest.cpp b/Tests/SphericalExtractionTest/SphericalExtractionTest.cpp index 2828e81b7..2275ccc57 100644 --- a/Tests/SphericalExtractionTest/SphericalExtractionTest.cpp +++ b/Tests/SphericalExtractionTest/SphericalExtractionTest.cpp @@ -75,7 +75,7 @@ int runSphericalExtractionTest(int argc, char *argv[]) spherical_extraction_lo.write_extraction("ExtractionOutLo_"); // high resolution spherical extraction - SphericalExtraction::params_t extraction_params_hi = + spherical_extraction_params_t extraction_params_hi = sim_params.extraction_params_lo; // we are only checking the converence in theta integration // extraction_params_hi.num_points_phi *= 2; @@ -98,7 +98,7 @@ int runSphericalExtractionTest(int argc, char *argv[]) }; // add the spherical harmonic mode integrands for each resolution and for - // the trapezium rule, Simpson's rule and Boole's rule + // the trapezium rule, Simpson's rule, Simpson's 3/8 rule and Boole's rule // Always use trapezium rule in phi as this is periodic bool broadcast_integral = true; std::pair, std::vector> integral_lo_trapezium, @@ -106,31 +106,41 @@ int runSphericalExtractionTest(int argc, char *argv[]) spherical_extraction_lo.add_mode_integrand( sim_params.es, sim_params.el, sim_params.em, extracted_harmonic, integral_lo_trapezium, IntegrationMethod::trapezium, - IntegrationMethod::trapezium, broadcast_integral); + IntegrationMethod::simpson, broadcast_integral); spherical_extraction_hi.add_mode_integrand( sim_params.es, sim_params.el, sim_params.em, extracted_harmonic, integral_hi_trapezium, IntegrationMethod::trapezium, - IntegrationMethod::trapezium, broadcast_integral); + IntegrationMethod::simpson, broadcast_integral); std::pair, std::vector> integral_lo_simpson, integral_hi_simpson; spherical_extraction_lo.add_mode_integrand( sim_params.es, sim_params.el, sim_params.em, extracted_harmonic, integral_lo_simpson, IntegrationMethod::simpson, - IntegrationMethod::trapezium, broadcast_integral); + IntegrationMethod::simpson, broadcast_integral); spherical_extraction_hi.add_mode_integrand( sim_params.es, sim_params.el, sim_params.em, extracted_harmonic, integral_hi_simpson, IntegrationMethod::simpson, - IntegrationMethod::trapezium, broadcast_integral); + IntegrationMethod::simpson, broadcast_integral); + std::pair, std::vector> integral_lo_simpson38, + integral_hi_simpson38; + spherical_extraction_lo.add_mode_integrand( + sim_params.es, sim_params.el, sim_params.em, extracted_harmonic, + integral_lo_simpson38, IntegrationMethod::simpson38, + IntegrationMethod::simpson, broadcast_integral); + spherical_extraction_hi.add_mode_integrand( + sim_params.es, sim_params.el, sim_params.em, extracted_harmonic, + integral_hi_simpson38, IntegrationMethod::simpson38, + IntegrationMethod::simpson, broadcast_integral); std::pair, std::vector> integral_lo_boole, integral_hi_boole; spherical_extraction_lo.add_mode_integrand( sim_params.es, sim_params.el, sim_params.em, extracted_harmonic, - integral_lo_boole, IntegrationMethod::boole, - IntegrationMethod::trapezium, broadcast_integral); + integral_lo_boole, IntegrationMethod::boole, IntegrationMethod::simpson, + broadcast_integral); spherical_extraction_hi.add_mode_integrand( sim_params.es, sim_params.el, sim_params.em, extracted_harmonic, - integral_hi_boole, IntegrationMethod::boole, - IntegrationMethod::trapezium, broadcast_integral); + integral_hi_boole, IntegrationMethod::boole, IntegrationMethod::simpson, + broadcast_integral); // do the surface integration spherical_extraction_lo.integrate(); @@ -150,6 +160,10 @@ int runSphericalExtractionTest(int argc, char *argv[]) (integral_hi_trapezium.first)[iradius]; double integral_re_lo_simpson = (integral_lo_simpson.first)[iradius]; double integral_re_hi_simpson = (integral_hi_simpson.first)[iradius]; + double integral_re_lo_simpson38 = + (integral_lo_simpson38.first)[iradius]; + double integral_re_hi_simpson38 = + (integral_hi_simpson38.first)[iradius]; double integral_re_lo_boole = (integral_lo_boole.first)[iradius]; double integral_re_hi_boole = (integral_hi_boole.first)[iradius]; double analytic_integral = 1.0; @@ -160,6 +174,9 @@ int runSphericalExtractionTest(int argc, char *argv[]) double convergence_factor_simpson = std::abs((integral_re_lo_simpson - analytic_integral) / (integral_re_hi_simpson - analytic_integral)); + double convergence_factor_simpson38 = + std::abs((integral_re_lo_simpson38 - analytic_integral) / + (integral_re_hi_simpson38 - analytic_integral)); double convergence_factor_boole = std::abs((integral_re_lo_boole - analytic_integral) / (integral_re_hi_boole - analytic_integral)); @@ -168,20 +185,39 @@ int runSphericalExtractionTest(int argc, char *argv[]) std::log2(convergence_factor_trapezium); double convergence_order_simpson = std::log2(convergence_factor_simpson); + double convergence_order_simpson38 = + std::log2(convergence_factor_simpson38); double convergence_order_boole = std::log2(convergence_factor_boole); // trapezium rule should have second order convergence status |= (convergence_order_trapezium < 1.5); // Simpson's rule should have fourth order convergence status |= (convergence_order_simpson < 3.5); + // Simpson 3/8's rule should have fourth order convergence + status |= (convergence_order_simpson38 < 3.5); // Boole's rule should have sixth order convergence status |= (convergence_order_boole < 5.5); pout() << "At r = " << r << ":\n"; + pout() << "analytic_integral = " << analytic_integral << "\n"; + pout() << "integral_re_lo_trapezium = " << integral_re_lo_trapezium + << "\n"; + pout() << "integral_re_hi_trapezium = " << integral_re_hi_trapezium + << "\n"; + pout() << "integral_re_lo_simpson = " << integral_re_lo_simpson << "\n"; + pout() << "integral_re_hi_simpson = " << integral_re_hi_simpson << "\n"; + pout() << "integral_re_lo_simpson38 = " << integral_re_lo_simpson38 + << "\n"; + pout() << "integral_re_hi_simpson38 = " << integral_re_hi_simpson38 + << "\n"; + pout() << "integral_re_lo_boole = " << integral_re_lo_boole << "\n"; + pout() << "integral_re_hi_boole = " << integral_re_hi_boole << "\n"; pout() << "convergence_order_trapezium = " << convergence_order_trapezium << "\n"; pout() << "convergence_order_simpson = " << convergence_order_simpson << "\n"; + pout() << "convergence_order_simpson38 = " + << convergence_order_simpson38 << "\n"; pout() << "convergence_order_boole = " << convergence_order_boole << "\n" << endl; diff --git a/Tests/SphericalExtractionTest/SphericalExtractionTest.inputs b/Tests/SphericalExtractionTest/SphericalExtractionTest.inputs index 4809398df..255e369cc 100644 --- a/Tests/SphericalExtractionTest/SphericalExtractionTest.inputs +++ b/Tests/SphericalExtractionTest/SphericalExtractionTest.inputs @@ -1,15 +1,12 @@ verbosity = 0 -N1 = 32 -N2 = 32 -N3 = 32 +N = 32 L = 16 chk_prefix = TestChk_ plot_prefix = TestPlt_ max_level = 1 -regrid_interval = 1 1 1 1 0 0 0 0 0 -ref_ratio = 2 2 2 2 2 2 2 2 2 +regrid_interval = 1 1 isPeriodic = 1 1 1 # Max and min box sizes @@ -22,7 +19,7 @@ checkpoint_interval = 1 num_extraction_radii = 2 extraction_radii = 4. 6. num_points_phi_lo = 12 -num_points_theta_lo = 13 +num_points_theta_lo = 13 # choose "multiple of 3 and of 4" + 1, to be compatible with Simpson, Simpson 3/8 and with Boole # Spherical Harmonic to set and extract es = 0 diff --git a/Tests/SphericalExtractionUniformTest/GNUmakefile b/Tests/SphericalExtractionUniformTest/GNUmakefile new file mode 100644 index 000000000..b513981a4 --- /dev/null +++ b/Tests/SphericalExtractionUniformTest/GNUmakefile @@ -0,0 +1,22 @@ +# -*- Mode: Makefile -*- + +### This makefile produces an executable for each name in the `ebase' +### variable using the libraries named in the `LibNames' variable. + +# Included makefiles need an absolute path to the Chombo installation +# CHOMBO_HOME := Please set the CHOMBO_HOME locally (e.g. export CHOMBO_HOME=... in bash) + +GRCHOMBO_SOURCE = $(shell pwd)/../../Source + +ebase := SphericalExtractionUniformTest + +LibNames := AMRTimeDependent AMRTools BoxTools + +src_dirs := $(GRCHOMBO_SOURCE)/utils \ + $(GRCHOMBO_SOURCE)/simd \ + $(GRCHOMBO_SOURCE)/BoxUtils \ + $(GRCHOMBO_SOURCE)/CCZ4 \ + $(GRCHOMBO_SOURCE)/GRChomboCore \ + $(GRCHOMBO_SOURCE)/AMRInterpolator + +include $(CHOMBO_HOME)/mk/Make.test diff --git a/Tests/SphericalExtractionUniformTest/SetHarmonic.hpp b/Tests/SphericalExtractionUniformTest/SetHarmonic.hpp new file mode 100644 index 000000000..d997017a0 --- /dev/null +++ b/Tests/SphericalExtractionUniformTest/SetHarmonic.hpp @@ -0,0 +1,42 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef SETHARMONIC_HPP_ +#define SETHARMONIC_HPP_ + +#include "BoxLoops.hpp" +#include "Cell.hpp" +#include "Coordinates.hpp" +#include "Tensor.hpp" +#include "UserVariables.hpp" //This files needs NUM_VARS - total number of components +#include "simd.hpp" + +// This compute class sets two vars to the real and imaginary parts of the +// s = a_es, l = a_el, m = a_em spin-weighted spherical harmonic +class SetHarmonic +{ + public: + SetHarmonic(int a_var_Re, int a_var_Im, int a_es, int a_el, int a_em, + std::array &a_center, double a_dx) + : m_var_Re(a_var_Re), m_var_Im(a_var_Im), m_es(a_es), m_el(a_el), + m_em(a_em), m_dx(a_dx), m_center(a_center) + { + } + + template void compute(Cell current_cell) const; + + protected: + const int m_var_Re; + const int m_var_Im; + const int m_es; + const int m_el; + const int m_em; + const double m_dx; + const std::array m_center; +}; + +#include "SetHarmonic.impl.hpp" + +#endif /* SETHARMONIC_HPP_ */ diff --git a/Tests/SphericalExtractionUniformTest/SetHarmonic.impl.hpp b/Tests/SphericalExtractionUniformTest/SetHarmonic.impl.hpp new file mode 100644 index 000000000..732909ff2 --- /dev/null +++ b/Tests/SphericalExtractionUniformTest/SetHarmonic.impl.hpp @@ -0,0 +1,33 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#if !defined(SETHARMONIC_HPP_) +#error "This file should only be included through SetHarmonic.hpp" +#endif + +#ifndef SETHARMONIC_IMPL_HPP_ +#define SETHARMONIC_IMPL_HPP_ + +#include "DebuggingTools.hpp" +#include "SetHarmonic.hpp" +#include "SphericalHarmonics.hpp" +#include "simd.hpp" + +template +void SetHarmonic::compute(Cell current_cell) const +{ + + Coordinates coords(current_cell, m_dx, m_center); + + using namespace SphericalHarmonics; + auto Y_lm = spin_Y_lm(coords.x, coords.y, coords.z, m_es, m_el, m_em); + data_t out_Re = Y_lm.Real; + data_t out_Im = Y_lm.Im; + + current_cell.store_vars(out_Re, m_var_Re); + current_cell.store_vars(out_Im, m_var_Im); +} + +#endif /* SETHARMONIC_IMPL_HPP_ */ diff --git a/Tests/SphericalExtractionUniformTest/SimulationParameters.hpp b/Tests/SphericalExtractionUniformTest/SimulationParameters.hpp new file mode 100644 index 000000000..61289c3c0 --- /dev/null +++ b/Tests/SphericalExtractionUniformTest/SimulationParameters.hpp @@ -0,0 +1,58 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef SIMULATIONPARAMETERS_HPP_ +#define SIMULATIONPARAMETERS_HPP_ + +// General includes +#include "ChomboParameters.hpp" +#include "GRParmParse.hpp" + +// Problem specific includes +#include "SphericalExtraction.hpp" + +class SimulationParameters : public ChomboParameters +{ + public: + SimulationParameters(GRParmParse &pp) : ChomboParameters(pp) + { + read_params(pp); + } + + private: + void read_params(GRParmParse &pp) + { + // Extraction params + pp.load("num_extraction_radii", + extraction_params_lo.num_extraction_radii, 1); + + if (pp.contains("extraction_radii")) + { + pp.load("extraction_radii", extraction_params_lo.extraction_radii, + extraction_params_lo.num_extraction_radii); + } + else + { + pp.load("extraction_radius", extraction_params_lo.extraction_radii, + 1, L / 4); + } + pp.load("num_points_phi_lo", extraction_params_lo.num_points_phi, 8); + pp.load("num_points_theta_lo", extraction_params_lo.num_points_theta, + 17); + pp.load("extraction_center", extraction_params_lo.center, center); + pp.load("write_extraction", extraction_params_lo.write_extraction, + false); + + pp.load("es", es, 0); + pp.load("el", el, 2); + pp.load("em", em, 0); + } + + public: + spherical_extraction_params_t extraction_params_lo; + int es, el, em; // spherical harmonic params +}; + +#endif /* SIMULATIONPARAMETERS_HPP_ */ diff --git a/Tests/SphericalExtractionUniformTest/SphericalExtractionUniformTest.cpp b/Tests/SphericalExtractionUniformTest/SphericalExtractionUniformTest.cpp new file mode 100644 index 000000000..877a84af4 --- /dev/null +++ b/Tests/SphericalExtractionUniformTest/SphericalExtractionUniformTest.cpp @@ -0,0 +1,248 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifdef CH_LANG_CC +/* + * _______ __ + * / ___/ / ___ __ _ / / ___ + * / /__/ _ \/ _ \/ V \/ _ \/ _ \ + * \___/_//_/\___/_/_/_/_.__/\___/ + * Please refer to LICENSE, in Chombo's root directory. + */ +#endif + +// Chombo includes +#include "parstream.H" //Gives us pout() + +// General includes: +#include +#include +#include +#include +#include +#include + +using std::endl; +#include "DefaultLevelFactory.hpp" +#include "GRAMR.hpp" + +#include "GRParmParse.hpp" +#include "SetupFunctions.hpp" +#include "SimulationParameters.hpp" + +// Problem specific includes: +#include "AMRInterpolator.hpp" +#include "Lagrange.hpp" +#include "SphericalExtraction.hpp" +#include "SphericalExtractionUniformTestLevel.hpp" + +#ifdef _OPENMP +#include +#endif + +// Chombo namespace +#include "UsingNamespace.H" + +int runSphericalExtractionUniformTest(int argc, char *argv[]) +{ + // Load the parameter file and construct the SimulationParameter class + // To add more parameters edit the SimulationParameters file. + std::string in_string = argv[argc - 1]; + pout() << in_string << std::endl; + char const *in_file = argv[argc - 1]; + GRParmParse pp(0, argv + argc, NULL, in_file); + SimulationParameters sim_params(pp); + + GRAMR gr_amr; + DefaultLevelFactory + surface_extraction_test_level_fact(gr_amr, sim_params); + // the initial data for the two variables is the spherical harmonic + // specified by params + setupAMRObject(gr_amr, surface_extraction_test_level_fact); + + AMRInterpolator> interpolator( + gr_amr, sim_params.origin, sim_params.dx, sim_params.boundary_params); + + // low resolution spherical extraction + SphericalExtractionUniform spherical_extraction_lo( + sim_params.extraction_params_lo, + sim_params.coarsest_dx * sim_params.dt_multiplier, 0.0, true, 0.0); + spherical_extraction_lo.add_var(c_phi_Re); + spherical_extraction_lo.add_var(c_phi_Im); + spherical_extraction_lo.extract(&interpolator); + spherical_extraction_lo.write_extraction("ExtractionOutLo_"); + + // high resolution spherical extraction + spherical_extraction_params_t extraction_params_hi = + sim_params.extraction_params_lo; + // we are only checking the converence in theta integration + // extraction_params_hi.num_points_phi *= 2; + extraction_params_hi.num_points_theta *= 2; + SphericalExtractionUniform spherical_extraction_hi( + extraction_params_hi, sim_params.coarsest_dx * sim_params.dt_multiplier, + 0.0, true, 0.0); + spherical_extraction_hi.add_var(c_phi_Re); + spherical_extraction_hi.add_var(c_phi_Im); + spherical_extraction_hi.extract(&interpolator); + spherical_extraction_hi.write_extraction("ExtractionOutHi_"); + + // real part is the zeroth componenent and imaginary part is first component + auto extracted_harmonic = [](std::vector &data, double, double, + double) { + return std::make_pair(data[0], data[1]); + }; + + // add the spherical harmonic mode integrands for each resolution and for + // the midpoint rule and Milne's regularized rule, Open 3rd and Open 4th + // order rules + // Always use trapezium rule in phi as this is periodic + bool broadcast_integral = true; + std::pair, std::vector> integral_lo_midpoint, + integral_hi_midpoint; + spherical_extraction_lo.add_mode_integrand( + sim_params.es, sim_params.el, sim_params.em, extracted_harmonic, + integral_lo_midpoint, IntegrationMethod::midpoint, + IntegrationMethod::trapezium, broadcast_integral); + spherical_extraction_hi.add_mode_integrand( + sim_params.es, sim_params.el, sim_params.em, extracted_harmonic, + integral_hi_midpoint, IntegrationMethod::midpoint, + IntegrationMethod::trapezium, broadcast_integral); + std::pair, std::vector> + integral_lo_milne_regularized, integral_hi_milne_regularized; + spherical_extraction_lo.add_mode_integrand( + sim_params.es, sim_params.el, sim_params.em, extracted_harmonic, + integral_lo_milne_regularized, IntegrationMethod::milne_regularized, + IntegrationMethod::trapezium, broadcast_integral); + spherical_extraction_hi.add_mode_integrand( + sim_params.es, sim_params.el, sim_params.em, extracted_harmonic, + integral_hi_milne_regularized, IntegrationMethod::milne_regularized, + IntegrationMethod::trapezium, broadcast_integral); + std::pair, std::vector> + integral_lo_open_3rd_order, integral_hi_open_3rd_order; + spherical_extraction_lo.add_mode_integrand( + sim_params.es, sim_params.el, sim_params.em, extracted_harmonic, + integral_lo_open_3rd_order, IntegrationMethod::open_3rd_order, + IntegrationMethod::trapezium, broadcast_integral); + spherical_extraction_hi.add_mode_integrand( + sim_params.es, sim_params.el, sim_params.em, extracted_harmonic, + integral_hi_open_3rd_order, IntegrationMethod::open_3rd_order, + IntegrationMethod::trapezium, broadcast_integral); + std::pair, std::vector> + integral_lo_open_4th_order, integral_hi_open_4th_order; + spherical_extraction_lo.add_mode_integrand( + sim_params.es, sim_params.el, sim_params.em, extracted_harmonic, + integral_lo_open_4th_order, IntegrationMethod::open_4th_order, + IntegrationMethod::trapezium, broadcast_integral); + spherical_extraction_hi.add_mode_integrand( + sim_params.es, sim_params.el, sim_params.em, extracted_harmonic, + integral_hi_open_4th_order, IntegrationMethod::open_4th_order, + IntegrationMethod::trapezium, broadcast_integral); + + // do the surface integration + spherical_extraction_lo.integrate(); + spherical_extraction_hi.integrate(); + + int status = 0; + pout() << std::setprecision(10); + + for (int iradius = 0; + iradius < sim_params.extraction_params_lo.num_extraction_radii; + ++iradius) + { + double r = sim_params.extraction_params_lo.extraction_radii[iradius]; + double integral_re_lo_midpoint = (integral_lo_midpoint.first)[iradius]; + double integral_re_hi_midpoint = (integral_hi_midpoint.first)[iradius]; + double integral_re_lo_milne_regularized = + (integral_lo_milne_regularized.first)[iradius]; + double integral_re_hi_milne_regularized = + (integral_hi_milne_regularized.first)[iradius]; + double integral_re_lo_open_3rd_order = + (integral_lo_open_3rd_order.first)[iradius]; + double integral_re_hi_open_3rd_order = + (integral_hi_open_3rd_order.first)[iradius]; + double integral_re_lo_open_4th_order = + (integral_lo_open_4th_order.first)[iradius]; + double integral_re_hi_open_4th_order = + (integral_hi_open_4th_order.first)[iradius]; + double analytic_integral = 1.0; + + double convergence_factor_midpoint = + std::abs((integral_re_lo_midpoint - analytic_integral) / + (integral_re_hi_midpoint - analytic_integral)); + double convergence_factor_milne_regularized = + std::abs((integral_re_lo_milne_regularized - analytic_integral) / + (integral_re_hi_milne_regularized - analytic_integral)); + double convergence_factor_open_3rd_order = + std::abs((integral_re_lo_open_3rd_order - analytic_integral) / + (integral_re_hi_open_3rd_order - analytic_integral)); + double convergence_factor_open_4th_order = + std::abs((integral_re_lo_open_4th_order - analytic_integral) / + (integral_re_hi_open_4th_order - analytic_integral)); + + double convergence_order_midpoint = + std::log2(convergence_factor_midpoint); + double convergence_order_milne_regularized = + std::log2(convergence_factor_milne_regularized); + double convergence_order_open_3rd_order = + std::log2(convergence_factor_open_3rd_order); + double convergence_order_open_4th_order = + std::log2(convergence_factor_open_4th_order); + + // midpoint rule should have second order convergence + status |= (convergence_order_midpoint < 1.5); + // Milne's adapted rule should have fourth order convergence + status |= (convergence_order_milne_regularized < 3.5); + // Open 3rd order rule should have fourth order convergence + status |= (convergence_order_open_3rd_order < 3.5); + // Open 4th order rule should have sixth order convergence + status |= (convergence_order_open_4th_order < 5.5); + + pout() << "At r = " << r << ":\n"; + pout() << "analytic_integral = " << analytic_integral << "\n"; + pout() << "integral_re_lo_midpoint = " << integral_re_lo_midpoint + << "\n"; + pout() << "integral_re_hi_midpoint = " << integral_re_hi_midpoint + << "\n"; + pout() << "integral_re_lo_milne_regularized = " + << integral_re_lo_milne_regularized << "\n"; + pout() << "integral_re_hi_milne_regularized = " + << integral_re_hi_milne_regularized << "\n"; + pout() << "integral_re_lo_open_3rd_order = " + << integral_re_lo_open_3rd_order << "\n"; + pout() << "integral_re_hi_open_3rd_order = " + << integral_re_hi_open_3rd_order << "\n"; + pout() << "integral_re_lo_open_4th_order = " + << integral_re_lo_open_4th_order << "\n"; + pout() << "integral_re_hi_open_4th_order = " + << integral_re_hi_open_4th_order << "\n"; + pout() << "convergence_order_midpoint = " << convergence_order_midpoint + << "\n"; + pout() << "convergence_order_milne_regularized = " + << convergence_order_milne_regularized << "\n"; + pout() << "convergence_order_open_3rd_order = " + << convergence_order_open_3rd_order << "\n"; + pout() << "convergence_order_open_4th_order = " + << convergence_order_open_4th_order << "\n" + << endl; + } + + return status; +} + +int main(int argc, char *argv[]) +{ + mainSetup(argc, argv); + + int status = runSphericalExtractionUniformTest(argc, argv); + + if (status == 0) + pout() << "SphericalExtractionUniformTest test passed." << endl; + else + pout() << "SphericalExtractionUniformTest test failed with return code " + << status << endl; + + mainFinalize(); + return status; +} diff --git a/Tests/SphericalExtractionUniformTest/SphericalExtractionUniformTest.inputs b/Tests/SphericalExtractionUniformTest/SphericalExtractionUniformTest.inputs new file mode 100644 index 000000000..cd11bf9f8 --- /dev/null +++ b/Tests/SphericalExtractionUniformTest/SphericalExtractionUniformTest.inputs @@ -0,0 +1,29 @@ +verbosity = 0 +N = 64 +L = 16 + +chk_prefix = TestChk_ +plot_prefix = TestPlt_ + +max_level = 1 +regrid_interval = 1 1 +isPeriodic = 1 1 1 + +# Max and min box sizes +max_grid_size = 32 +block_factor = 4 +tag_buffer_size = 3 +checkpoint_interval = 1 + +# extraction params +num_extraction_radii = 2 +extraction_radii = 4. 6. +num_points_phi_lo = 60 +num_points_theta_lo = 60 # multiple of 3,4,5 + +# Spherical Harmonic to set and extract +# needed to choose a complicated mode, otherwise computation +# is too precise (since we use 60 points) and doesn't converge +es = -1 +el = 6 +em = 1 diff --git a/Tests/SphericalExtractionUniformTest/SphericalExtractionUniformTestLevel.hpp b/Tests/SphericalExtractionUniformTest/SphericalExtractionUniformTestLevel.hpp new file mode 100644 index 000000000..959a00809 --- /dev/null +++ b/Tests/SphericalExtractionUniformTest/SphericalExtractionUniformTestLevel.hpp @@ -0,0 +1,38 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef SPHERICALEXTRACTIONUNIFORMTESTLEVEL_HPP_ +#define SPHERICALEXTRACTIONUNIFORMTESTLEVEL_HPP_ + +#include "BoxLoops.hpp" +#include "GRAMRLevel.hpp" +#include "SetHarmonic.hpp" +#include "SetValue.hpp" +#include "UserVariables.hpp" + +class SphericalExtractionUniformTestLevel : public GRAMRLevel +{ + friend class DefaultLevelFactory; + // Inherit the contructors from GRAMRLevel + using GRAMRLevel::GRAMRLevel; + + // initialize data + virtual void initialData() + { + BoxLoops::loop(SetHarmonic(c_phi_Re, c_phi_Im, m_p.es, m_p.el, m_p.em, + m_p.center, m_dx), + m_state_new, m_state_new, FILL_GHOST_CELLS); + } + + virtual void specificEvalRHS(GRLevelData &a_soln, GRLevelData &a_rhs, + const double a_time) + { + } + + virtual void computeTaggingCriterion(FArrayBox &tagging_criterion, + const FArrayBox ¤t_state){}; +}; + +#endif /* SPHERICALEXTRACTIONUNIFORMTESTLEVEL_HPP_ */ diff --git a/Tests/SphericalExtractionUniformTest/UserVariables.hpp b/Tests/SphericalExtractionUniformTest/UserVariables.hpp new file mode 100644 index 000000000..40ac01542 --- /dev/null +++ b/Tests/SphericalExtractionUniformTest/UserVariables.hpp @@ -0,0 +1,30 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef USERVARIABLES_HPP +#define USERVARIABLES_HPP + +#include "EmptyDiagnosticVariables.hpp" +#include +#include + +// assign enum to each variable +enum +{ + c_phi_Re, + c_phi_Im, + + NUM_VARS +}; + +namespace UserVariables +{ +static const std::array variable_names = {"phi_Re", + "phi_Im"}; +} + +#include "UserVariables.inc.hpp" + +#endif /* USERVARIABLES_HPP */ From 2574b7d7988f5ddf46513e455851e79c9ee39766 Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Fri, 23 Apr 2021 16:56:19 +0100 Subject: [PATCH 20/92] In reflective direction, origin should not move (e.g. stay at 0 for reflective lo_boundary) --- .../ApparentHorizon.impl.hpp | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp index 72f022ac3..6f5191fbf 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp @@ -286,8 +286,14 @@ void ApparentHorizon::predict_next_origin() { FOR1(a) { - new_center[a] += (m_old_centers[0][a] + m_old_centers[2][a] - - 2. * m_old_centers[1][a]); + if (!m_interp.get_interpolator()->get_boundary_reflective(Side::Lo, + a) && + !m_interp.get_interpolator()->get_boundary_reflective(Side::Hi, + a)) + { + new_center[a] += (m_old_centers[0][a] + m_old_centers[2][a] - + 2. * m_old_centers[1][a]); + } } if (m_params.verbose > AHFinder::SOME) { @@ -303,7 +309,13 @@ void ApparentHorizon::predict_next_origin() { FOR1(a) { - new_center[a] += (m_old_centers[0][a] - m_old_centers[1][a]); + if (!m_interp.get_interpolator()->get_boundary_reflective(Side::Lo, + a) && + !m_interp.get_interpolator()->get_boundary_reflective(Side::Hi, + a)) + { + new_center[a] += (m_old_centers[0][a] - m_old_centers[1][a]); + } } if (m_params.verbose > AHFinder::SOME) From e3e31dc0a627202fd2d6a92779d949ef85eddafd Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Tue, 27 Apr 2021 19:33:15 +0100 Subject: [PATCH 21/92] Simplify InterpSource (I wanted to remove the getLevelData() from InterpSource and use GRAMRLevel directly in AMRInterpolator, but that creates circular template dependency problems and I gave up) --- Source/AMRInterpolator/AMRInterpolator.hpp | 19 ++++++-- .../AMRInterpolator/AMRInterpolator.impl.hpp | 10 ++-- Source/AMRInterpolator/InterpSource.hpp | 10 +++- .../InterpolationAlgorithm.hpp | 35 -------------- .../AMRInterpolator/InterpolationLayout.hpp | 27 ----------- Source/AMRInterpolator/Lagrange.hpp | 7 ++- Source/AMRInterpolator/Lagrange.impl.hpp | 39 +++++++-------- Source/AMRInterpolator/QuinticConvolution.hpp | 5 +- .../QuinticConvolution.impl.hpp | 48 ++++++++++--------- Source/AMRInterpolator/SimpleInterpSource.hpp | 13 +++-- Source/AMRInterpolator/SurfaceExtraction.hpp | 2 +- Source/ApparentHorizonFinder/AHFinder.hpp | 4 +- .../ApparentHorizon.impl.hpp | 5 +- .../ApparentHorizon_petsc.impl.hpp | 22 ++++----- Source/GRChomboCore/GRAMRLevel.cpp | 2 - Source/GRChomboCore/GRAMRLevel.hpp | 3 +- Source/utils/WeylExtraction.hpp | 4 +- Tests/AMRInterpolatorTest/Polynomial.hpp | 4 +- Tests/InterpolatorTest/InterpolatorTest.cpp | 28 +++++++---- Tests/SphericalExtractionTest/SetHarmonic.hpp | 4 +- .../SetHarmonic.hpp | 4 +- 21 files changed, 130 insertions(+), 165 deletions(-) delete mode 100644 Source/AMRInterpolator/InterpolationAlgorithm.hpp delete mode 100644 Source/AMRInterpolator/InterpolationLayout.hpp diff --git a/Source/AMRInterpolator/AMRInterpolator.hpp b/Source/AMRInterpolator/AMRInterpolator.hpp index f68a0a143..5e14f983c 100644 --- a/Source/AMRInterpolator/AMRInterpolator.hpp +++ b/Source/AMRInterpolator/AMRInterpolator.hpp @@ -17,11 +17,7 @@ // Our includes #include "BoundaryConditions.hpp" #include "GRAMR.hpp" -#include "InterpSource.hpp" -#include "InterpolationAlgorithm.hpp" -#include "InterpolationLayout.hpp" #include "InterpolationQuery.hpp" - #include "MPIContext.hpp" #include "UserVariables.hpp" @@ -32,6 +28,19 @@ template class AMRInterpolator { + struct InterpolationLayout + { + std::vector rank; + std::vector level_idx; + std::vector box_idx; + + InterpolationLayout(int num_points) + : rank(num_points, -1), level_idx(num_points, -1), + box_idx(num_points, -1) + { + } + }; + public: // constructor for backward compatibility // (adds an artificial BC with only periodic BC) @@ -67,7 +76,7 @@ template class AMRInterpolator InterpolationLayout findBoxes(InterpolationQuery &query); void prepareMPI(InterpolationQuery &query, - const InterpolationLayout layout); + const InterpolationLayout &layout); void exchangeMPIQuery(); void calculateAnswers(InterpolationQuery &query); void exchangeMPIAnswer(); diff --git a/Source/AMRInterpolator/AMRInterpolator.impl.hpp b/Source/AMRInterpolator/AMRInterpolator.impl.hpp index 94d5be946..fa8ee0fca 100644 --- a/Source/AMRInterpolator/AMRInterpolator.impl.hpp +++ b/Source/AMRInterpolator/AMRInterpolator.impl.hpp @@ -45,9 +45,7 @@ void AMRInterpolator::refresh(const bool a_fill_ghosts) { CH_TIME("AMRInterpolator::refresh"); - const Vector &levels = - const_cast(m_gr_amr).getAMRLevels(); - m_num_levels = levels.size(); + m_num_levels = const_cast(m_gr_amr).getAMRLevels().size(); m_mem_level.clear(); m_mem_box.clear(); @@ -280,7 +278,7 @@ void AMRInterpolator::computeLevelLayouts() } template -InterpolationLayout +typename AMRInterpolator::InterpolationLayout AMRInterpolator::findBoxes(InterpolationQuery &query) { CH_TIME("AMRInterpolator::findBoxes"); @@ -518,7 +516,7 @@ AMRInterpolator::findBoxes(InterpolationQuery &query) template void AMRInterpolator::prepareMPI(InterpolationQuery &query, - const InterpolationLayout layout) + const InterpolationLayout &layout) { CH_TIME("AMRInterpolator::prepareMPI"); @@ -740,7 +738,7 @@ void AMRInterpolator::calculateAnswers(InterpolationQuery &query) typedef std::vector comps_t; comps_t &comps = deriv_it->second; - algo.setup(deriv, m_dx[level_idx], grid_coord); + algo.setup(deriv, grid_coord); for (typename comps_t::iterator it = comps.begin(); it != comps.end(); ++it) diff --git a/Source/AMRInterpolator/InterpSource.hpp b/Source/AMRInterpolator/InterpSource.hpp index 2a53c5848..61608438e 100644 --- a/Source/AMRInterpolator/InterpSource.hpp +++ b/Source/AMRInterpolator/InterpSource.hpp @@ -16,13 +16,21 @@ // Chombo namespace #include "UsingNamespace.H" -// Abstrace base class to get the FABs out of an AMRLevel +// Abstrace base class to identify points in an AMRLevel template class InterpSource { public: virtual const LevelData &getLevelData( const VariableType var_type = VariableType::evolution) const = 0; virtual bool contains(const std::array &point) const = 0; + virtual double get_dx(int dir) const = 0; + virtual std::array get_dxs() const + { + std::array dxs; + for (int i = 0; i < N_DIMS; ++i) + dxs[i] = get_dx(i); + return dxs; + } }; #endif /* INTERPSOURCE_H_ */ diff --git a/Source/AMRInterpolator/InterpolationAlgorithm.hpp b/Source/AMRInterpolator/InterpolationAlgorithm.hpp deleted file mode 100644 index ea8243ba0..000000000 --- a/Source/AMRInterpolator/InterpolationAlgorithm.hpp +++ /dev/null @@ -1,35 +0,0 @@ -/* GRChombo - * Copyright 2012 The GRChombo collaboration. - * Please refer to LICENSE in GRChombo's root directory. - */ - -#ifndef INTERPOLATIONALGORITHM_HPP_ -#define INTERPOLATIONALGORITHM_HPP_ - -// Chombo includes -#include "FArrayBox.H" - -// Other includes -#include - -// Chombo namespace -#include "UsingNamespace.H" - -class InterpolationAlgorithm -{ - public: - virtual ~InterpolationAlgorithm() = 0; -}; - -class NearestNeighbour : public InterpolationAlgorithm -{ - public: - static inline double - interpPoint(const std::array &gridCoord, - const FArrayBox &fab, int comps, const IntVect &nearest) - { - return fab.get(nearest, comps); - } -}; - -#endif /* INTERPOLATIONALGORITHM_HPP_ */ diff --git a/Source/AMRInterpolator/InterpolationLayout.hpp b/Source/AMRInterpolator/InterpolationLayout.hpp deleted file mode 100644 index bdc1f857d..000000000 --- a/Source/AMRInterpolator/InterpolationLayout.hpp +++ /dev/null @@ -1,27 +0,0 @@ -/* GRChombo - * Copyright 2012 The GRChombo collaboration. - * Please refer to LICENSE in GRChombo's root directory. - */ - -#ifndef INTERPOLATIONLAYOUT_HPP_ -#define INTERPOLATIONLAYOUT_HPP_ - -#include - -class InterpolationLayout -{ - private: - template friend class AMRInterpolator; - - std::vector rank; - std::vector level_idx; - std::vector box_idx; - - InterpolationLayout(int num_points) - : rank(num_points, -1), level_idx(num_points, -1), - box_idx(num_points, -1) - { - } -}; - -#endif /* INTERPOLATIONLAYOUT_HPP_ */ diff --git a/Source/AMRInterpolator/Lagrange.hpp b/Source/AMRInterpolator/Lagrange.hpp index 199ada2ea..838cf06d0 100644 --- a/Source/AMRInterpolator/Lagrange.hpp +++ b/Source/AMRInterpolator/Lagrange.hpp @@ -43,7 +43,7 @@ template class Lagrange pair, std::vector> generateStencil(const std::array &deriv, const std::array &dx, - const std::array &evalCoord, + const std::array &eval_index, int dim = N_DIMS - 1); std::vector m_interp_points; @@ -58,10 +58,9 @@ template class Lagrange public: Lagrange(const InterpSource &source, bool verbosity = false); - // evalCoord is in 'index' coordinates, not physical coordinates + // eval_index is in 'index' coordinates, not physical coordinates void setup(const std::array &deriv, - const std::array &dx, - const std::array &evalCoord); + const std::array &eval_index); // any class with a method: // Real get(const IntVect &a_iv, int a_comp) const diff --git a/Source/AMRInterpolator/Lagrange.impl.hpp b/Source/AMRInterpolator/Lagrange.impl.hpp index 8b0cb99ba..31cfe58ee 100644 --- a/Source/AMRInterpolator/Lagrange.impl.hpp +++ b/Source/AMRInterpolator/Lagrange.impl.hpp @@ -159,20 +159,21 @@ Lagrange::Lagrange(const InterpSource &source, } template -void Lagrange::setup(const std::array &deriv, - const std::array &dx, - const std::array &evalCoord) +void Lagrange::setup( + const std::array &deriv, + const std::array &eval_index) { + std::array dxs = m_source.get_dxs(); pair, std::vector> result = - generateStencil(deriv, dx, evalCoord); + generateStencil(deriv, dxs, eval_index); m_interp_points = result.first; m_interp_weights = result.second; /* - pout() << TAG << "Stencil: coord = { "; + pout() << TAG << "Stencil: point = { "; for (int i = 0; i < N_DIMS; ++i) { - pout() << evalCoord[i] << " "; + pout() << eval_index[i] << " "; } pout() << "}, weights = { "; for (int i = 0; i < m_interp_weights.size(); ++i) @@ -241,7 +242,7 @@ template pair, std::vector> Lagrange::generateStencil( const std::array &deriv, const std::array &dx, - const std::array &evalCoord, int dim) + const std::array &eval_index, int dim) { std::vector out_points; std::vector out_weights; @@ -265,20 +266,20 @@ Lagrange::generateStencil( int points_min = Order + deriv[dim]; int points_max = Order + deriv[dim]; - std::array interp_coord = evalCoord; + std::array interp_point = eval_index; - // TF: assume nearest point is at 'std::round(evalCoord[dim])' + // TF: assume nearest point is at 'std::round(eval_index[dim])' // this used to be an argument, but I think this assumption should always // hold - int candidate = std::round(evalCoord[dim]); - int grown_direction = (candidate - evalCoord[dim] < 0) ? DOWN : UP; + int candidate = std::round(eval_index[dim]); + int grown_direction = (candidate - eval_index[dim] < 0) ? DOWN : UP; while ((can_grow[DOWN] || can_grow[UP]) && (points_max - points_min < Order + deriv[dim])) { - interp_coord[dim] = candidate; + interp_point[dim] = candidate; - if (m_source.contains(interp_coord)) + if (m_source.contains(interp_point)) { int idx = (grown_direction == DOWN) ? (--points_min) : (points_max++); @@ -304,12 +305,12 @@ Lagrange::generateStencil( const Stencil my_weights = getStencil(stencil_width, deriv[dim], dx[dim], - evalCoord[dim] - my_points[points_min]); + eval_index[dim] - my_points[points_min]); if (m_verbosity) { pout() << TAG << "Stencil: dim = " << dim - << ", coord = " << evalCoord[dim] << ", points = { "; + << ", point = " << eval_index[dim] << ", points = { "; for (int i = points_min; i < points_max; ++i) { pout() << my_points[i] << " "; @@ -327,13 +328,13 @@ Lagrange::generateStencil( // first. for (int i = 0; i < stencil_width; ++i) { - interp_coord[dim] = my_points[i + points_min]; + interp_point[dim] = my_points[i + points_min]; if (dim > 0) { // Descend to the next dimension pair, std::vector> sub_result = - generateStencil(deriv, dx, interp_coord, dim - 1); + generateStencil(deriv, dx, interp_point, dim - 1); std::vector &sub_points = sub_result.first; std::vector &sub_weights = sub_result.second; @@ -353,8 +354,8 @@ Lagrange::generateStencil( if (my_weights[i] != 0) { out_points.push_back(IntVect(D_DECL6( - interp_coord[0], interp_coord[1], interp_coord[2], - interp_coord[3], interp_coord[4], interp_coord[5]))); + interp_point[0], interp_point[1], interp_point[2], + interp_point[3], interp_point[4], interp_point[5]))); out_weights.push_back(my_weights[i]); } } diff --git a/Source/AMRInterpolator/QuinticConvolution.hpp b/Source/AMRInterpolator/QuinticConvolution.hpp index ea92d574d..1e5cfcd95 100644 --- a/Source/AMRInterpolator/QuinticConvolution.hpp +++ b/Source/AMRInterpolator/QuinticConvolution.hpp @@ -21,10 +21,9 @@ template class QuinticConvolution QuinticConvolution(const InterpSource &source, bool verbosity = false); - // evalCoord is in 'index' coordinates, not physical coordinates + // eval_index is in 'index' coordinates, not physical coordinates void setup(const std::array &deriv, - const std::array &dx, - const std::array &evalCoord); + const std::array &eval_index); // any class with a method: // Real get(const IntVect &a_iv, int a_comp) const diff --git a/Source/AMRInterpolator/QuinticConvolution.impl.hpp b/Source/AMRInterpolator/QuinticConvolution.impl.hpp index b9c57a8a6..877be4d31 100644 --- a/Source/AMRInterpolator/QuinticConvolution.impl.hpp +++ b/Source/AMRInterpolator/QuinticConvolution.impl.hpp @@ -20,17 +20,19 @@ QuinticConvolution::QuinticConvolution( template void QuinticConvolution::setup( - const std::array &deriv, const std::array &dx, - const std::array &evalCoord) + const std::array &deriv, + const std::array &eval_index) { m_interp_points.clear(); m_interp_weights.clear(); + std::array dxs = m_source.get_dxs(); + double weights_1d[N_DIMS][6]; for (int dim = 0; dim < N_DIMS; ++dim) { - double s = evalCoord[dim] - floor(evalCoord[dim]); + double s = eval_index[dim] - floor(eval_index[dim]); if (deriv[dim] == 0) { @@ -59,40 +61,40 @@ void QuinticConvolution::setup( weights_1d[dim][0] = (0.046875 + s * (-0.375 + s * (0.84375 + (-0.75 + 0.234375 * s) * s))) / - dx[dim]; + dxs[dim]; weights_1d[dim][1] = (-0.59375 + s * (2.5 + s * (-1.6875 + s * (-1.1875 + 1.015625 * s)))) / - dx[dim]; + dxs[dim]; weights_1d[dim][2] = - (s * (-4.25 + (7.875 - 4.21875 * s) * (s * s))) / dx[dim]; + (s * (-4.25 + (7.875 - 4.21875 * s) * (s * s))) / dxs[dim]; weights_1d[dim][3] = (0.59375 + s * (2.5 + s * (1.6875 + s * (-9. + 4.21875 * s)))) / - dx[dim]; + dxs[dim]; weights_1d[dim][4] = (-0.046875 + s * (-0.375 + s * (-0.84375 + (2.875 - 1.015625 * s) * s))) / - dx[dim]; + dxs[dim]; weights_1d[dim][5] = - ((0.1875 - 0.234375 * s) * s * (s * s)) / dx[dim]; + ((0.1875 - 0.234375 * s) * s * (s * s)) / dxs[dim]; } else if (deriv[dim] == 2) { weights_1d[dim][0] = (-0.375 + s * (1.6875 + (-2.25 + 0.9375 * s) * s)) / - (dx[dim] * dx[dim]); + (dxs[dim] * dxs[dim]); weights_1d[dim][1] = (2.5 + s * (-3.375 + s * (-3.5625 + 4.0625 * s))) / - (dx[dim] * dx[dim]); - weights_1d[dim][2] = - (-4.25 + (23.625 - 16.875 * s) * (s * s)) / (dx[dim] * dx[dim]); + (dxs[dim] * dxs[dim]); + weights_1d[dim][2] = (-4.25 + (23.625 - 16.875 * s) * (s * s)) / + (dxs[dim] * dxs[dim]); weights_1d[dim][3] = (2.5 + s * (3.375 + s * (-27. + 16.875 * s))) / - (dx[dim] * dx[dim]); + (dxs[dim] * dxs[dim]); weights_1d[dim][4] = (-0.375 + s * (-1.6875 + (8.625 - 4.0625 * s) * s)) / - (dx[dim] * dx[dim]); + (dxs[dim] * dxs[dim]); weights_1d[dim][5] = - ((0.5625 - 0.9375 * s) * (s * s)) / (dx[dim] * dx[dim]); + ((0.5625 - 0.9375 * s) * (s * s)) / (dxs[dim] * dxs[dim]); } else { @@ -101,27 +103,27 @@ void QuinticConvolution::setup( } } - std::array interp_coord; + std::array interp_index; #if N_DIMS >= 3 for (int z = 0; z < 6; ++z) { - interp_coord[2] = floor(evalCoord[2]) + z - 2; + interp_index[2] = floor(eval_index[2]) + z - 2; #endif #if N_DIMS >= 2 for (int y = 0; y < 6; ++y) { - interp_coord[1] = floor(evalCoord[1]) + y - 2; + interp_index[1] = floor(eval_index[1]) + y - 2; #endif for (int x = 0; x < 6; ++x) { - interp_coord[0] = floor(evalCoord[0]) + x - 2; - CH_assert(m_source.contains(interp_coord)); + interp_index[0] = floor(eval_index[0]) + x - 2; + CH_assert(m_source.contains(interp_index)); m_interp_points.push_back(IntVect(D_DECL6( - interp_coord[0], interp_coord[1], interp_coord[2], - interp_coord[3], interp_coord[4], interp_coord[5]))); + interp_index[0], interp_index[1], interp_index[2], + interp_index[3], interp_index[4], interp_index[5]))); #if N_DIMS >= 3 m_interp_weights.push_back(weights_1d[0][x] * weights_1d[1][y] * weights_1d[2][z]); diff --git a/Source/AMRInterpolator/SimpleInterpSource.hpp b/Source/AMRInterpolator/SimpleInterpSource.hpp index 07f8f8d43..2e034a166 100644 --- a/Source/AMRInterpolator/SimpleInterpSource.hpp +++ b/Source/AMRInterpolator/SimpleInterpSource.hpp @@ -12,13 +12,15 @@ // Relevant to be able to use Interpolation classes by themselves template class SimpleInterpSource : public InterpSource { - LevelData m_fake; - std::array m_points_per_dir; std::array m_is_periodic; + std::array m_dxs; + + LevelData m_fake; public: SimpleInterpSource(std::array a_points_per_dir, + std::array a_dxs, std::array a_is_periodic = {false}) : m_points_per_dir(a_points_per_dir), m_is_periodic(a_is_periodic) { @@ -28,7 +30,8 @@ template class SimpleInterpSource : public InterpSource getLevelData(const VariableType var_type = VariableType::evolution) const { return m_fake; - }; + } + bool contains(const std::array &point) const { bool in = true; @@ -40,7 +43,9 @@ template class SimpleInterpSource : public InterpSource } return in; }; - void fillAllGhosts(const VariableType var_type = VariableType::evolution){}; + + double get_dx(int dir) const { return m_dxs[dir]; }; + std::array get_dxs() const { return m_dxs; } }; #endif /* SIMPLEINTERPSOURCE_H_ */ diff --git a/Source/AMRInterpolator/SurfaceExtraction.hpp b/Source/AMRInterpolator/SurfaceExtraction.hpp index aad518741..6d46e7215 100644 --- a/Source/AMRInterpolator/SurfaceExtraction.hpp +++ b/Source/AMRInterpolator/SurfaceExtraction.hpp @@ -48,7 +48,7 @@ struct surface_extraction_params_t std::string data_path, integral_file_prefix; std::string extraction_path, extraction_file_prefix; - int min_extraction_level() + int min_extraction_level() const { return *(std::min_element(extraction_levels.begin(), extraction_levels.end())); diff --git a/Source/ApparentHorizonFinder/AHFinder.hpp b/Source/ApparentHorizonFinder/AHFinder.hpp index a84023354..519d676d4 100644 --- a/Source/ApparentHorizonFinder/AHFinder.hpp +++ b/Source/ApparentHorizonFinder/AHFinder.hpp @@ -35,7 +35,6 @@ #include "AHData.hpp" #include "AHDeriv.hpp" #include "AHGeometryData.hpp" -#include "AMRInterpolator.hpp" #include "BoundaryConditions.hpp" #include "ChomboParameters.hpp" #include "Lagrange.hpp" @@ -48,6 +47,9 @@ template class AHInterpolation; template class ApparentHorizon; +// Forward declaration for AMRInterpolator +template class AMRInterpolator; + /* TF: A general note on the radius and the value of chi on the AH surface. diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp index 192557184..cc914793b 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp @@ -962,7 +962,7 @@ void ApparentHorizon::restart( } int rows = stats[0].size(); - SimpleInterpSource<1> source({rows}); + SimpleInterpSource<1> source({rows}, {old_print_dt}); SimpleArrayBox<1> box_x({rows}, stats[cols - CH_SPACEDIM]); SimpleArrayBox<1> box_y({rows}, stats[cols - CH_SPACEDIM + 1]); #if CH_SPACEDIM == 3 @@ -975,8 +975,7 @@ void ApparentHorizon::restart( { std::array old_center; - interpolator.setup({0}, {old_print_dt}, - {old_centers_time_index[i]}); + interpolator.setup({0}, {old_centers_time_index[i]}); old_center[0] = interpolator.interpData(box_x); old_center[1] = interpolator.interpData(box_y); #if CH_SPACEDIM == 3 diff --git a/Source/ApparentHorizonFinder/ApparentHorizon_petsc.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon_petsc.impl.hpp index c5a7bee6d..d2a4a0680 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon_petsc.impl.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon_petsc.impl.hpp @@ -274,35 +274,33 @@ bool ApparentHorizon::interpolate_ah( // start interpolating #if CH_SPACEDIM == 3 + // local, no derivative + std::array derivs = {0, 0}; + std::array dxs = {du, dv}; + SimpleArrayBox box( {old_number_of_u, old_number_of_v}, old_coords[CH_SPACEDIM - 1], {m_interp.get_coord_system().is_u_periodic(), m_interp.get_coord_system().is_v_periodic()}); SimpleInterpSource source( - {old_number_of_u, old_number_of_v}, + {old_number_of_u, old_number_of_v}, dxs, {m_interp.get_coord_system().is_u_periodic(), m_interp.get_coord_system().is_v_periodic()}); #elif CH_SPACEDIM == 2 + std::array derivs = {0}; + std::array dxs = {du}; + SimpleArrayBox box( {old_number_of_u}, old_coords[CH_SPACEDIM - 1], {m_interp.get_coord_system().is_u_periodic()}); SimpleInterpSource source( - {old_number_of_u}, {m_interp.get_coord_system().is_u_periodic()}); + {old_number_of_u}, dxs, {m_interp.get_coord_system().is_u_periodic()}); #endif const int Order = 4; bool verbose = false; Lagrange interpolator4(source, verbose); -#if CH_SPACEDIM == 3 - // local, no derivative - std::array derivs = {0, 0}; - std::array dxs = {du, dv}; -#elif CH_SPACEDIM == 2 - std::array derivs = {0}; - std::array dxs = {du}; -#endif - #if CH_SPACEDIM == 3 for (int v = m_vmin; v < m_vmax; ++v) { @@ -338,7 +336,7 @@ bool ApparentHorizon::interpolate_ah( std::array evalCoord = {u_old_idx}; #endif - interpolator4.setup(derivs, dxs, evalCoord); + interpolator4.setup(derivs, evalCoord); m_F[idx_local] = interpolator4.interpData(box); #if CH_SPACEDIM == 3 diff --git a/Source/GRChomboCore/GRAMRLevel.cpp b/Source/GRChomboCore/GRAMRLevel.cpp index 86edb91b6..ee6df819d 100644 --- a/Source/GRChomboCore/GRAMRLevel.cpp +++ b/Source/GRChomboCore/GRAMRLevel.cpp @@ -982,8 +982,6 @@ void GRAMRLevel::copySolnData(GRLevelData &dest, const GRLevelData &src) copyBdyGhosts(src, dest); } -double GRAMRLevel::get_dx() const { return m_dx; } - bool GRAMRLevel::at_level_timestep_multiple(int a_level) const { double target_dt = m_p.coarsest_dt; diff --git a/Source/GRChomboCore/GRAMRLevel.hpp b/Source/GRChomboCore/GRAMRLevel.hpp index e3f998eed..5b59e537c 100644 --- a/Source/GRChomboCore/GRAMRLevel.hpp +++ b/Source/GRChomboCore/GRAMRLevel.hpp @@ -165,7 +165,8 @@ class GRAMRLevel : public AMRLevel, public InterpSource<> { } - double get_dx() const; + // direction irrelevant, but relevant for InterpSource + ALWAYS_INLINE double get_dx(int dir = 0) const { return m_dx; }; /// Returns true if m_time is the same as the time at the end of the current /// timestep on level a_level and false otherwise diff --git a/Source/utils/WeylExtraction.hpp b/Source/utils/WeylExtraction.hpp index 505709e83..0130ebd09 100644 --- a/Source/utils/WeylExtraction.hpp +++ b/Source/utils/WeylExtraction.hpp @@ -19,7 +19,7 @@ class WeylExtraction : public SphericalExtraction { public: //! The constructor - WeylExtraction(spherical_extraction_params_t &a_params, double a_dt, + WeylExtraction(const spherical_extraction_params_t &a_params, double a_dt, double a_time, bool a_first_step, double a_restart_time = 0.0) : SphericalExtraction(a_params, a_dt, a_time, a_first_step, @@ -31,7 +31,7 @@ class WeylExtraction : public SphericalExtraction //! The old constructor which assumes it is called in specificPostTimeStep //! so the first time step is when m_time == m_dt - WeylExtraction(spherical_extraction_params_t a_params, double a_dt, + WeylExtraction(const spherical_extraction_params_t &a_params, double a_dt, double a_time, double a_restart_time = 0.0) : WeylExtraction(a_params, a_dt, a_time, (a_dt == a_time), a_restart_time) diff --git a/Tests/AMRInterpolatorTest/Polynomial.hpp b/Tests/AMRInterpolatorTest/Polynomial.hpp index 1ad556680..2b3e176f0 100644 --- a/Tests/AMRInterpolatorTest/Polynomial.hpp +++ b/Tests/AMRInterpolatorTest/Polynomial.hpp @@ -14,7 +14,7 @@ class Polynomial { public: - Polynomial(std::array &a_center, double a_dx) + Polynomial(const std::array &a_center, double a_dx) : m_dx(a_dx), m_center(a_center) { } @@ -38,7 +38,7 @@ class Polynomial protected: const double m_dx; - const std::array m_center; + const std::array &m_center; }; #endif /* POLYNOMIAL_HPP_ */ diff --git a/Tests/InterpolatorTest/InterpolatorTest.cpp b/Tests/InterpolatorTest/InterpolatorTest.cpp index 34dcd7143..d986f329f 100644 --- a/Tests/InterpolatorTest/InterpolatorTest.cpp +++ b/Tests/InterpolatorTest/InterpolatorTest.cpp @@ -13,6 +13,9 @@ */ #endif +// Chombo includes +#include "parstream.H" //Gives us pout() + // General includes: #include #include @@ -20,14 +23,15 @@ #include #include -#include "parstream.H" //Gives us pout() - #include "Lagrange.hpp" #include "QuinticConvolution.hpp" #include "SetupFunctions.hpp" #include "SimpleArrayBox.hpp" #include "SimpleInterpSource.hpp" +// Chombo namespace +#include "UsingNamespace.H" + // cell centered grid // double get_dx(double L, int num_points_u) { return L / num_points_u; } // double get_x(int i, double m_dx) { return m_dx * (i + 0.5); } @@ -67,18 +71,22 @@ int runInterpolatorTest(int argc, char *argv[]) f[i] = func(x, L); } - SimpleInterpSource<1> source({num_points_u}); + // Note that the 2nd argument of .setup 'dx' does not matter for 0th order + // interpolation -> set to 0 + SimpleInterpSource<1> source({num_points_u}, {0.}); SimpleArrayBox<1> box({num_points_u}, f); double test_point = M_PI; // just because it's irrational, hence for sure not in the grid + double test_index = get_idx(test_point, m_dx); - Lagrange<4, 1> interpolator1(source, true); - interpolator1.setup({0}, {m_dx}, {get_idx(test_point, m_dx)}); + bool verbosity = true; + Lagrange<4, 1> interpolator1(source, verbosity); + interpolator1.setup({0}, {test_index}); double val1 = interpolator1.interpData(box); - QuinticConvolution<1> interpolator2(source, true); - interpolator2.setup({0}, {m_dx}, {get_idx(test_point, m_dx)}); + QuinticConvolution<1> interpolator2(source, verbosity); + interpolator2.setup({0}, {test_index}); double val2 = interpolator2.interpData(box); double exact = func(test_point, L); @@ -105,10 +113,10 @@ int main(int argc, char *argv[]) int status = runInterpolatorTest(argc, argv); if (status == 0) - pout() << "Interpolator test passed." << endl; + std::cout << "Interpolator test passed." << endl; else - pout() << "Interpolator test failed with return code " << status - << endl; + std::cout << "Interpolator test failed with return code " << status + << endl; mainFinalize(); return status; diff --git a/Tests/SphericalExtractionTest/SetHarmonic.hpp b/Tests/SphericalExtractionTest/SetHarmonic.hpp index d997017a0..692c0be9c 100644 --- a/Tests/SphericalExtractionTest/SetHarmonic.hpp +++ b/Tests/SphericalExtractionTest/SetHarmonic.hpp @@ -19,7 +19,7 @@ class SetHarmonic { public: SetHarmonic(int a_var_Re, int a_var_Im, int a_es, int a_el, int a_em, - std::array &a_center, double a_dx) + const std::array &a_center, double a_dx) : m_var_Re(a_var_Re), m_var_Im(a_var_Im), m_es(a_es), m_el(a_el), m_em(a_em), m_dx(a_dx), m_center(a_center) { @@ -34,7 +34,7 @@ class SetHarmonic const int m_el; const int m_em; const double m_dx; - const std::array m_center; + const std::array &m_center; }; #include "SetHarmonic.impl.hpp" diff --git a/Tests/SphericalExtractionUniformTest/SetHarmonic.hpp b/Tests/SphericalExtractionUniformTest/SetHarmonic.hpp index d997017a0..692c0be9c 100644 --- a/Tests/SphericalExtractionUniformTest/SetHarmonic.hpp +++ b/Tests/SphericalExtractionUniformTest/SetHarmonic.hpp @@ -19,7 +19,7 @@ class SetHarmonic { public: SetHarmonic(int a_var_Re, int a_var_Im, int a_es, int a_el, int a_em, - std::array &a_center, double a_dx) + const std::array &a_center, double a_dx) : m_var_Re(a_var_Re), m_var_Im(a_var_Im), m_es(a_es), m_el(a_el), m_em(a_em), m_dx(a_dx), m_center(a_center) { @@ -34,7 +34,7 @@ class SetHarmonic const int m_el; const int m_em; const double m_dx; - const std::array m_center; + const std::array &m_center; }; #include "SetHarmonic.impl.hpp" From 242fbcb70da2bd1471dd8bb9035631513e02aa57 Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Tue, 27 Apr 2021 19:33:27 +0100 Subject: [PATCH 22/92] Fix minor bug affecting AH Tests --- Examples/SingleBH/params.txt | 4 +--- Source/ApparentHorizonFinder/AHFinder.hpp | 14 ++++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Examples/SingleBH/params.txt b/Examples/SingleBH/params.txt index a920c74c3..d4ba92118 100644 --- a/Examples/SingleBH/params.txt +++ b/Examples/SingleBH/params.txt @@ -10,7 +10,7 @@ verbosity = 0 # output_path = "" # Main path for all files. Must exist! chk_prefix = SingleBH_ plot_prefix = SingleBHPlot_ -#restart_file = SingleBH_000004.3d.hdf5 +# restart_file = SingleBH_000004.3d.hdf5 # HDF5files are written every dt = L/N*dt_multiplier*checkpoint_interval checkpoint_interval = 2 @@ -53,8 +53,6 @@ momentum = 0. -0.1 0.0 N_full = 32 L_full = 16 -L_full = 128 - # Maximum number of times you can regrid above coarsest level max_level = 4 # There are (max_level+1) grids, so min is zero diff --git a/Source/ApparentHorizonFinder/AHFinder.hpp b/Source/ApparentHorizonFinder/AHFinder.hpp index 519d676d4..11fa6d49d 100644 --- a/Source/ApparentHorizonFinder/AHFinder.hpp +++ b/Source/ApparentHorizonFinder/AHFinder.hpp @@ -154,12 +154,14 @@ class AHFinder int extra_contain_diagnostic; // not a parameter (set internally); // counts how many - std::string stats_path, - stats_prefix; //!< name for stats file with - //!< area, spin and AH origin/center - std::string coords_path, - coords_prefix; //!< name for coords file with AH - //!< coordinates at each time step + std::string stats_path = "", + stats_prefix = + "stats_AH"; //!< name for stats file with + //!< area, spin and AH origin/center + std::string coords_path = "", + coords_prefix = + "coords_AH"; //!< name for coords file with AH + //!< coordinates at each time step void read_params(GRParmParse &pp, const ChomboParameters &a_p); }; From 88bb72aa18e2e2d55ffd3d2e3e0e6e204e9ebb1f Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Tue, 27 Apr 2021 19:33:40 +0100 Subject: [PATCH 23/92] Rename AH new parameters --- Examples/BinaryBH/params.txt | 4 +++- Examples/BinaryBH/params_two_punctures.txt | 4 +++- Examples/KerrBH/params.txt | 4 +++- Examples/KerrBH/params_cheap.txt | 4 +++- Examples/SingleBH/params.txt | 4 +++- Source/ApparentHorizonFinder/AHFinder.cpp | 8 ++++---- 6 files changed, 19 insertions(+), 9 deletions(-) diff --git a/Examples/BinaryBH/params.txt b/Examples/BinaryBH/params.txt index 23b7a62a2..b7b1737f9 100644 --- a/Examples/BinaryBH/params.txt +++ b/Examples/BinaryBH/params.txt @@ -218,6 +218,8 @@ AH_num_points_v = 48 AH_set_origins_to_punctures = 1 -# coords_subpath = "data/coords" +# AH_coords_subpath = "data/coords" +# AH_stats_prefix = "stats_AH" +# AH_coords_prefix = "coords_AH" ################################################# diff --git a/Examples/BinaryBH/params_two_punctures.txt b/Examples/BinaryBH/params_two_punctures.txt index d0855e9ab..94f82eead 100644 --- a/Examples/BinaryBH/params_two_punctures.txt +++ b/Examples/BinaryBH/params_two_punctures.txt @@ -214,7 +214,9 @@ AH_num_points_v = 48 AH_set_origins_to_punctures = 1 -# coords_subpath = "data/coords" +# AH_coords_subpath = "data/coords" +# AH_stats_prefix = "stats_AH" +# AH_coords_prefix = "coords_AH" ################################################# # Two Punctures parameters diff --git a/Examples/KerrBH/params.txt b/Examples/KerrBH/params.txt index bc5e2d418..00805d6b3 100644 --- a/Examples/KerrBH/params.txt +++ b/Examples/KerrBH/params.txt @@ -171,7 +171,9 @@ AH_verbose = 2 AH_num_extra_vars = 1 AH_extra_vars = chi -# coords_subpath = "data/coords" +# AH_coords_subpath = "data/coords" +# AH_stats_prefix = "stats_AH" +# AH_coords_prefix = "coords_AH" ################################################# diff --git a/Examples/KerrBH/params_cheap.txt b/Examples/KerrBH/params_cheap.txt index 2a348d6d8..13c8c33f8 100644 --- a/Examples/KerrBH/params_cheap.txt +++ b/Examples/KerrBH/params_cheap.txt @@ -174,6 +174,8 @@ AH_verbose = 2 AH_num_extra_vars = 2 AH_extra_vars = chi d1_Ham -# coords_subpath = "data/coords" +# AH_coords_subpath = "data/coords" +# AH_stats_prefix = "stats_AH" +# AH_coords_prefix = "coords_AH" ################################################# diff --git a/Examples/SingleBH/params.txt b/Examples/SingleBH/params.txt index d4ba92118..3d84c0208 100644 --- a/Examples/SingleBH/params.txt +++ b/Examples/SingleBH/params.txt @@ -171,6 +171,8 @@ AH_verbose = 3 AH_num_extra_vars = 2 AH_extra_vars = chi d1_chi -# coords_subpath = "data/coords" +# AH_coords_subpath = "data/coords" +# AH_stats_prefix = "stats_AH" +# AH_coords_prefix = "coords_AH" ################################################# diff --git a/Source/ApparentHorizonFinder/AHFinder.cpp b/Source/ApparentHorizonFinder/AHFinder.cpp index 29fa90050..2821f3271 100644 --- a/Source/ApparentHorizonFinder/AHFinder.cpp +++ b/Source/ApparentHorizonFinder/AHFinder.cpp @@ -516,9 +516,9 @@ void AHFinder::params::read_params(GRParmParse &pp, const ChomboParameters &a_p) stats_path = a_p.data_path; - if (pp.contains("coords_subpath")) + if (pp.contains("AH_coords_subpath")) { - pp.load("coords_subpath", coords_path); + pp.load("AH_coords_subpath", coords_path); if (!coords_path.empty() && coords_path.back() != '/') coords_path += "/"; if (a_p.output_path != "./" && !a_p.output_path.empty()) @@ -527,8 +527,8 @@ void AHFinder::params::read_params(GRParmParse &pp, const ChomboParameters &a_p) else coords_path = stats_path; - pp.load("stats_prefix", stats_prefix, std::string("stats_AH")); - pp.load("coords_prefix", coords_prefix, std::string("coords_AH")); + pp.load("AH_stats_prefix", stats_prefix, std::string("stats_AH")); + pp.load("AH_coords_prefix", coords_prefix, std::string("coords_AH")); } ///////////////////////////////////////////////////////// From e3ae50b0194aa0ee7f0147690b912d50bc0fc495 Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Wed, 30 Jun 2021 00:42:30 +0100 Subject: [PATCH 24/92] Detach 2D AHFinder test from BHAMR First, it doesn't make sense, it was just convenient. Second, it was causing problems in the 2D code as it implied compiling the PunctureTracker.cpp file of the BlackHoles folder, which was giving an error due to no 'c_shift1' and 'c_shift2' variable existing in this Test. --- .../ApparentHorizonTest2D.cpp | 26 +++++++++++-------- Tests/ApparentHorizonFinderTest2D/GNUmakefile | 4 +-- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/Tests/ApparentHorizonFinderTest2D/ApparentHorizonTest2D.cpp b/Tests/ApparentHorizonFinderTest2D/ApparentHorizonTest2D.cpp index e92f39034..06b36feb7 100644 --- a/Tests/ApparentHorizonFinderTest2D/ApparentHorizonTest2D.cpp +++ b/Tests/ApparentHorizonFinderTest2D/ApparentHorizonTest2D.cpp @@ -22,7 +22,7 @@ recompile again for 2D when compiling this example) // General includes: #include -#include "BHAMR.hpp" +#include "GRAMR.hpp" #include "parstream.H" //Gives us pout() #include "SetupFunctions.hpp" @@ -35,6 +35,7 @@ recompile again for 2D when compiling this example) #include "SmallDataIO.hpp" #ifdef USE_AHFINDER +#include "AHFinder.hpp" #include "ApparentHorizon.hpp" #endif @@ -47,30 +48,33 @@ int runApparentHorizonTest2D(int argc, char *argv[]) GRParmParse pp(0, argv + argc, NULL, in_string.c_str()); SimulationParameters sim_params(pp); - BHAMR bh_amr; + GRAMR gr_amr; DefaultLevelFactory ah_test_level_fact( - bh_amr, sim_params); - setupAMRObject(bh_amr, ah_test_level_fact); + gr_amr, sim_params); + setupAMRObject(gr_amr, ah_test_level_fact); int status = 0; #ifdef USE_AHFINDER - AHFinder::params AH_params = {1, 20, 1, 1, false, false, 0, 0., -1.}; - AH_params.verbose = 3; - // Set up interpolator and PETSc subcommunicator when AH extraction is // active AMRInterpolator> interpolator( - bh_amr, sim_params.origin, sim_params.dx, sim_params.boundary_params, + gr_amr, sim_params.origin, sim_params.dx, sim_params.boundary_params, sim_params.verbosity); - bh_amr.set_interpolator(&interpolator); + gr_amr.set_interpolator(&interpolator); + AHFinder ah_finder; AHStringGeometry sph(sim_params.L); + + AHFinder::params AH_params = {1, 20, 1, 1, false, false, 0, 0., -1.}; + AH_params.verbose = 3; + // use 2. as initial guess, doesn't matter much (as long as it converges to // the correct solution) - bh_amr.m_ah_finder.add_ah(sph, 2., AH_params); + ah_finder.set_interpolator(&interpolator); + ah_finder.add_ah(sph, 2., AH_params); - if (!bh_amr.m_ah_finder.get(0)->get_converged()) + if (!ah_finder.get(0)->get_converged()) status = 1; else { diff --git a/Tests/ApparentHorizonFinderTest2D/GNUmakefile b/Tests/ApparentHorizonFinderTest2D/GNUmakefile index d91ab3fca..be4b5fe26 100644 --- a/Tests/ApparentHorizonFinderTest2D/GNUmakefile +++ b/Tests/ApparentHorizonFinderTest2D/GNUmakefile @@ -21,8 +21,6 @@ src_dirs := $(GRCHOMBO_SOURCE)/utils \ $(GRCHOMBO_SOURCE)/BoxUtils \ $(GRCHOMBO_SOURCE)/GRChomboCore \ $(GRCHOMBO_SOURCE)/AMRInterpolator \ - $(GRCHOMBO_SOURCE)/ApparentHorizonFinder \ - $(GRCHOMBO_SOURCE)/InitialConditions/BlackHoles \ - $(GRCHOMBO_SOURCE)/BlackHoles + $(GRCHOMBO_SOURCE)/ApparentHorizonFinder include $(CHOMBO_HOME)/mk/Make.test From ba85645a82199ed9203cb520baeb0f3be68680d1 Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Wed, 30 Jun 2021 00:42:47 +0100 Subject: [PATCH 25/92] Refactor AHFinder to template class --- Source/ApparentHorizonFinder/AHFinder.hpp | 163 ++------- .../{AHFinder.cpp => AHFinder.impl.hpp} | 346 +++--------------- .../AHInterpolation.impl.hpp | 6 +- .../ApparentHorizonFinder/ApparentHorizon.hpp | 106 +++++- .../ApparentHorizon.impl.hpp | 228 +++++++++--- .../ApparentHorizon_petsc.impl.hpp | 10 +- .../PETScCommunicator.cpp | 124 +++++++ .../PETScCommunicator.hpp | 45 +++ Source/BlackHoles/BHAMR.hpp | 2 +- .../GRChomboCore/SimulationParametersBase.hpp | 2 +- .../ApparentHorizonTest2D.cpp | 12 +- .../ApparentHorizonTest2D.inputs | 11 + .../SimulationParameters.hpp | 15 +- .../ApparentHorizonTest3D.cpp | 27 +- .../ApparentHorizonTest3D.inputs | 11 +- .../SimulationParameters.hpp | 9 +- 16 files changed, 588 insertions(+), 529 deletions(-) rename Source/ApparentHorizonFinder/{AHFinder.cpp => AHFinder.impl.hpp} (54%) create mode 100644 Source/ApparentHorizonFinder/PETScCommunicator.cpp create mode 100644 Source/ApparentHorizonFinder/PETScCommunicator.hpp diff --git a/Source/ApparentHorizonFinder/AHFinder.hpp b/Source/ApparentHorizonFinder/AHFinder.hpp index 11fa6d49d..ec93f0386 100644 --- a/Source/ApparentHorizonFinder/AHFinder.hpp +++ b/Source/ApparentHorizonFinder/AHFinder.hpp @@ -11,6 +11,21 @@ // Other includes: +#ifdef CH_MPI +#include +#endif +#include + +#include "ApparentHorizon.hpp" +#include "ChomboParameters.hpp" +#include "Lagrange.hpp" + +// Chombo namespace +#include "UsingNamespace.H" + +///////////////////////////////////////////////////////////////////////////// +// SurfaceGeometry and AHFunction defauls + // included so that user can re-define default AHSurfaceGeometry or AHFunction #include "UserVariables.hpp" @@ -25,30 +40,7 @@ #include "AHFunctions.hpp" #define AHFunction ExpansionFunction #endif - -#ifdef CH_MPI -#include -#endif - -#include - -#include "AHData.hpp" -#include "AHDeriv.hpp" -#include "AHGeometryData.hpp" -#include "BoundaryConditions.hpp" -#include "ChomboParameters.hpp" -#include "Lagrange.hpp" -#include "VariableType.hpp" - -// Chombo namespace -#include "UsingNamespace.H" - -// declare classes, define them later -template class AHInterpolation; -template class ApparentHorizon; - -// Forward declaration for AMRInterpolator -template class AMRInterpolator; +///////////////////////////////////////////////////////////////////////////// /* TF: @@ -83,95 +75,11 @@ is more senstitive). */ //! Class to manage AHs and its mergers + control PETSc MPI sub-communicator -class AHFinder +template class AHFinder { public: - struct params // prepend with 'AH_' in params file + struct params : ApparentHorizon::params { - int num_ranks; //!< number of ranks for PETSc sub-communicator (default - //!< 0, which is 'all') - int num_points_u; //!< number of points for 2D coordinate grid -#if CH_SPACEDIM == 3 - int num_points_v; //!< number of points for 2D coordinate grid -#endif - int solve_interval; //!< same as checkpoint_interval, for - //!< ApparentHorizon::solve (default 1) - int print_interval; //!< same as solve_interval, but for prints (default - //!< 1) - bool track_center; //!< whether or not to update the center - //!< (set to false if you know it won't move) - //!< (default true) - bool predict_origin; //!< whether or not to estimate where the next - //!< origin will be at (default = track_center) - - int level_to_run; // if negative, it will count backwards (e.g. -1 is - // 'last level') (default 0) - - double start_time; //!< time after which AH can start (default 0.) - //!< Useful for ScalarField collapse - double give_up_time; //!< stop if at this time nothing was found - //!< (<0 to never, which is default) - //!< Useful for ScalarField collapse - - //! mergers will be searched when distance between 'parent' BHs is - //! distance < merger_search_factor * 4. * (AH_initial_guess_1 + - //! AH_initial_guess_2) should be roughly '2M=2(m1+m2)' for initial - //! guess at m/2 (set to non-positive to 'always search') - double merger_search_factor; // see note above (default is 1) - //! initial guess for merger is 'merger_pre_factor * 4. * - //! (AH_initial_guess_1 + AH_initial_guess_2)' - //! set to somethig bigger to avoid finding the inner AH - double merger_pre_factor; // see note above (default to 1.) - - bool allow_re_attempt; //!< re-attempt with initial guess if - //!< previous convergence failed (default false) - int max_fails_after_lost; //!< number of time steps to try again after - //!< (-1 to never) the AH was lost - //!< (default is 0) - - int verbose; //!< print information during execution (default is 1) - - bool print_geometry_data; //!< print metric and extrinsic - //!< curvature of the AHs (default false) - - bool re_solve_at_restart; //!< whether or not to re-run even if AH - //!< already exists (useful in order to be - //!< able to provide an initial guess and - //!< re-run the AHFinder on that time step) - //!< (default false) - - bool stop_if_max_fails; //! breaks the run if AH doesn't converge - //! 'max_fails_after_lost' times or if - //! 'give_up_time' is reached without - //! convergence (default is 'false') - - std::map> - extra_vars; //! extra vars to write to coordinates file () - int num_extra_vars; // total number of extra vars (!=extra_vars.size() - // as derivative count for multiple vars) - - int extra_contain_diagnostic; // not a parameter (set internally); - // counts how many - - std::string stats_path = "", - stats_prefix = - "stats_AH"; //!< name for stats file with - //!< area, spin and AH origin/center - std::string coords_path = "", - coords_prefix = - "coords_AH"; //!< name for coords file with AH - //!< coordinates at each time step - - void read_params(GRParmParse &pp, const ChomboParameters &a_p); - }; - - enum verbose_level - { - NONE, - MIN, // minimal - SOME, // a bit technical - MORE // debug }; private: @@ -181,14 +89,14 @@ class AHFinder std::vector> m_merger_pairs; AMRInterpolator> *m_interpolator; //!< The interpolator pointer - std::vector *> + std::vector *> m_apparent_horizons; //!< public in case user wants to solve by himself public: AHFinder(){}; ~AHFinder(); - ALWAYS_INLINE ApparentHorizon * + ALWAYS_INLINE ApparentHorizon * get(unsigned AH_i) { CH_assert(AH_i < m_apparent_horizons.size()); @@ -197,7 +105,7 @@ class AHFinder //! returns the index of the AH in m_apparent_horizons int - add_ah(const AHSurfaceGeometry &a_coord_system, + add_ah(const SurfaceGeometry &a_coord_system, double a_initial_guess, //!< Initial guess for radius (or whatever //!< coordinate you're solving for) const params &a_params, //!< set of AH parameters @@ -207,7 +115,7 @@ class AHFinder //! returns the index of the AH in m_apparent_horizons //! allows for personalized optimizer that finds zero of function //! 'AHFunction' (that can have some ::params) - int add_ah(const AHSurfaceGeometry &a_coord_system, double a_initial_guess, + int add_ah(const SurfaceGeometry &a_coord_system, double a_initial_guess, const params &a_params, const typename AHFunction::params &a_func_params, bool solve_first_step = true); @@ -238,31 +146,8 @@ class AHFinder bool solve_merger(int ah1, int ah2, double &initial_guess_merger, std::array &origin_merged); - ///////////////////////////////////////////////////////// - ///////////////////////////////////////////////////////// - ///////////////// PETSc control methods ///////////////// - ///////////////////////////////////////////////////////// - ///////////////////////////////////////////////////////// - - static bool m_initialized; //!< is initialized? - -#ifdef CH_MPI - static MPI_Group m_mpi_group; //!< set of MPI ranks for PETSc - static MPI_Comm m_mpi_comm; //!< MPI sub-communicator -#endif - - //! define number of ranks of PETSc sub-communicator - static void set_num_ranks(int a_num_ranks); - - //! initialize PETSc and its MPI sub-communicator - static PetscErrorCode PETSc_initialize(int a_num_ranks); - - //! finalize PETSc - static PetscErrorCode PETSc_finalize(); - - public: - //! true if part of PETSc MPI sub-communicator - static bool is_rank_active(); }; // namespace AHFinder +#include "AHFinder.impl.hpp" + #endif /* _AHFINDER_HPP_ */ diff --git a/Source/ApparentHorizonFinder/AHFinder.cpp b/Source/ApparentHorizonFinder/AHFinder.impl.hpp similarity index 54% rename from Source/ApparentHorizonFinder/AHFinder.cpp rename to Source/ApparentHorizonFinder/AHFinder.impl.hpp index 2821f3271..f6b9e4805 100644 --- a/Source/ApparentHorizonFinder/AHFinder.cpp +++ b/Source/ApparentHorizonFinder/AHFinder.impl.hpp @@ -3,54 +3,58 @@ * Please refer to LICENSE in GRChombo's root directory. */ -#ifdef USE_AHFINDER +#ifndef _AHFINDER_HPP_ +#error "This file should only be included through AHFinder.hpp" +#endif -#include "AHFinder.hpp" +#ifndef _AHFINDER_IMPL_HPP_ +#define _AHFINDER_IMPL_HPP_ #include "AHInterpolation.hpp" #include "ApparentHorizon.hpp" #include "FilesystemTools.hpp" -AHFinder::~AHFinder() +template +AHFinder::~AHFinder() { // destroy horizon pointers and finalize PETSc for (auto &ah : m_apparent_horizons) delete ah; - PETSc_finalize(); + PETScCommunicator::finalize(); } -int AHFinder::add_ah(const AHSurfaceGeometry &a_coord_system, - double a_initial_guess, const AHFinder::params &a_params, - bool solve_first_step) +template +int AHFinder::add_ah( + const SurfaceGeometry &a_coord_system, double a_initial_guess, + const AHFinder::params &a_params, + bool solve_first_step) { return add_ah(a_coord_system, a_initial_guess, a_params, - AHFunction::params(), solve_first_step); + typename AHFunction::params(), solve_first_step); } -int AHFinder::add_ah(const AHSurfaceGeometry &a_coord_system, - double a_initial_guess, const AHFinder::params &a_params, - const typename AHFunction::params &a_func_params, - bool solve_first_step) +template +int AHFinder::add_ah( + const SurfaceGeometry &a_coord_system, double a_initial_guess, + const AHFinder::params &a_params, + const typename AHFunction::params &a_func_params, bool solve_first_step) { - if (!AHFinder::m_initialized) - { - AHFinder::PETSc_initialize(a_params.num_ranks); + PETScCommunicator::initialize(a_params.num_ranks); - if (!FilesystemTools::directory_exists(a_params.stats_path)) - FilesystemTools::mkdir_recursive(a_params.stats_path); + if (!FilesystemTools::directory_exists(a_params.stats_path)) + FilesystemTools::mkdir_recursive(a_params.stats_path); - if (!FilesystemTools::directory_exists(a_params.coords_path)) - FilesystemTools::mkdir_recursive(a_params.coords_path); - } + if (!FilesystemTools::directory_exists(a_params.coords_path)) + FilesystemTools::mkdir_recursive(a_params.coords_path); // determine how many AH there are already const int num_ah = m_apparent_horizons.size(); - AHInterpolation interp(a_coord_system, - m_interpolator); + AHInterpolation interp(a_coord_system, + m_interpolator); m_apparent_horizons.push_back( - new ApparentHorizon( + new ApparentHorizon( interp, a_initial_guess, a_params, a_func_params, a_params.stats_prefix + std::to_string(num_ah + 1), a_params.coords_prefix + std::to_string(num_ah + 1) + "_", @@ -60,7 +64,9 @@ int AHFinder::add_ah(const AHSurfaceGeometry &a_coord_system, return num_ah; } -int AHFinder::add_ah_merger(int ah1, int ah2, const params &a_params) +template +int AHFinder::add_ah_merger(int ah1, int ah2, + const params &a_params) { CH_assert(a_params.track_center); // center tracking must be on for mergers @@ -74,7 +80,7 @@ int AHFinder::add_ah_merger(int ah1, int ah2, const params &a_params) bool do_solve = solve_merger(ah1, ah2, initial_guess_merger, origin_merger); // assume same coordiante system between merging horizons - AHSurfaceGeometry coord_system = + SurfaceGeometry coord_system = m_apparent_horizons[ah1]->get_ah_interp().get_coord_system(); coord_system.set_origin(origin_merger); @@ -87,9 +93,11 @@ int AHFinder::add_ah_merger(int ah1, int ah2, const params &a_params) return num; } -void AHFinder::solve(double a_dt, double a_time, double a_restart_time) +template +void AHFinder::solve(double a_dt, double a_time, + double a_restart_time) { - CH_TIME("AHFinder::solve"); + CH_TIME("AHFinder::solve"); enum { @@ -160,7 +168,8 @@ void AHFinder::solve(double a_dt, double a_time, double a_restart_time) if (!(m_apparent_horizons[pair.first]->has_been_found() && m_apparent_horizons[pair.second]->has_been_found())) { - if (m_apparent_horizons[i]->m_params.verbose > NONE) + if (m_apparent_horizons[i]->m_params.verbose > + ApparentHorizon::NONE) { pout() << "Skipping BH #" << i << std::endl; } @@ -173,7 +182,8 @@ void AHFinder::solve(double a_dt, double a_time, double a_restart_time) ah_solved[pair.second] == TOO_FAR) { // if one of the 'parents' is a merger too far away to be // solved, skip children too - if (m_apparent_horizons[i]->m_params.verbose > NONE) + if (m_apparent_horizons[i]->m_params.verbose > + ApparentHorizon::NONE) { pout() << "Skipping BH #" << i << std::endl; } @@ -216,7 +226,8 @@ void AHFinder::solve(double a_dt, double a_time, double a_restart_time) // the past if (m_apparent_horizons[i]->good_to_go(a_dt, a_time)) { - if (m_apparent_horizons[i]->m_params.verbose > NONE) + if (m_apparent_horizons[i]->m_params.verbose > + ApparentHorizon::NONE) pout() << "Solving AH #" << i << std::endl; m_apparent_horizons[i]->solve(a_dt, a_time, a_restart_time); @@ -226,7 +237,8 @@ void AHFinder::solve(double a_dt, double a_time, double a_restart_time) } else { - if (m_apparent_horizons[i]->m_params.verbose > NONE) + if (m_apparent_horizons[i]->m_params.verbose > + ApparentHorizon::NONE) { pout() << "Skipping BH #" << i << ". Not good to go." << std::endl; @@ -240,7 +252,9 @@ void AHFinder::solve(double a_dt, double a_time, double a_restart_time) delete ah_solved; } -bool AHFinder::need_diagnostics(double a_dt, double a_time) const +template +bool AHFinder::need_diagnostics( + double a_dt, double a_time) const { bool out = false; for (auto &ah : m_apparent_horizons) @@ -249,8 +263,10 @@ bool AHFinder::need_diagnostics(double a_dt, double a_time) const return out; } -bool AHFinder::solve_merger(int ah1, int ah2, double &initial_guess_merger, - std::array ¢er_merger) +template +bool AHFinder::solve_merger( + int ah1, int ah2, double &initial_guess_merger, + std::array ¢er_merger) { // SKIP if 'parents' not yet close enough auto AH1 = m_apparent_horizons[ah1]; @@ -330,7 +346,8 @@ bool AHFinder::solve_merger(int ah1, int ah2, double &initial_guess_merger, // distance CH_assert(min_distance <= 2. * initial_guess_merger); - if (AH1->m_params.verbose > NONE) + if (AH1->m_params.verbose > + ApparentHorizon::NONE) { pout() << "BHs #" << ah1 << " and #" << ah2 << " at distance = " << distance; @@ -350,7 +367,8 @@ bool AHFinder::solve_merger(int ah1, int ah2, double &initial_guess_merger, } else { - if (AH1->m_params.verbose > NONE) + if (AH1->m_params.verbose > + ApparentHorizon::NONE) { pout() << "Original BH not yet found. Skipping solve for merger." << std::endl; @@ -360,7 +378,8 @@ bool AHFinder::solve_merger(int ah1, int ah2, double &initial_guess_merger, return do_solve; } -void AHFinder::set_origins( +template +void AHFinder::set_origins( const std::vector> &origins, bool includes_mergers) { @@ -398,255 +417,4 @@ void AHFinder::set_origins( } } -void AHFinder::params::read_params(GRParmParse &pp, const ChomboParameters &a_p) -{ - pp.load("AH_num_ranks", num_ranks, 0); // 0 means "all" - pp.load("AH_num_points_u", num_points_u); -#if CH_SPACEDIM == 3 - pp.load("AH_num_points_v", num_points_v); -#endif - pp.load("AH_solve_interval", solve_interval, 1); - pp.load("AH_print_interval", print_interval, 1); - pp.load("AH_track_center", track_center, true); - pp.load("AH_predict_origin", predict_origin, track_center); - // can't predict if center is not being tracked - CH_assert(!(predict_origin && !track_center)); - - pp.load("AH_level_to_run", level_to_run, 0); - CH_assert(level_to_run <= a_p.max_level && - level_to_run > -(a_p.max_level + 1)); - if (level_to_run < 0) // if negative, count backwards - level_to_run += a_p.max_level + 1; - - pp.load("AH_start_time", start_time, 0.0); - pp.load("AH_give_up_time", give_up_time, -1.0); - - pp.load("AH_merger_search_factor", merger_search_factor, 1.); - pp.load("AH_merger_pre_factor", merger_pre_factor, 1.); - - pp.load("AH_allow_re_attempt", allow_re_attempt, false); - pp.load("AH_max_fails_after_lost", max_fails_after_lost, 0); - pp.load("AH_stop_if_max_fails", stop_if_max_fails, false); - - pp.load("AH_verbose", verbose, (int)AHFinder::MIN); - pp.load("AH_print_geometry_data", print_geometry_data, false); - pp.load("AH_re_solve_at_restart", re_solve_at_restart, false); - - // sanity checks - CH_assert(solve_interval > 0 && print_interval > 0); - CH_assert(level_to_run >= 0 && level_to_run <= a_p.max_level); - - // if box division ends up with size less than 3, PETSc will - // complain (this only gives an estimate of the box side) - int size = 1; -#if CH_MPI - MPI_Comm_size(Chombo_MPI::comm, &size); -#endif - size = std::min(num_ranks, size); -#if CH_SPACEDIM == 3 - CH_assert(num_points_u > 0 && num_points_v > 0); - CH_assert(num_points_u / sqrt(size) >= 3); // make sure for size 'u' - CH_assert(num_points_v / sqrt(size) >= 3); // make sure for size 'v' -#elif CH_SPACEDIM == 2 - CH_assert(num_points_u > 0); - CH_assert(num_points_u / size >= 3); // make sure for size 'u' -#endif - - // load vars to write to coord files - num_extra_vars = 0; - extra_contain_diagnostic = 0; - - int AH_num_extra_vars; - pp.load("AH_num_extra_vars", AH_num_extra_vars, 0); - if (AH_num_extra_vars > 0) - { - std::vector AH_extra_var_names(AH_num_extra_vars, ""); - pp.load("AH_extra_vars", AH_extra_var_names, AH_num_extra_vars); - for (const std::string &full_name : AH_extra_var_names) - { - std::string var_name = full_name; - - // variable names might start with "d1_" or "d2_" to indicate - // the user wants derivatives - int der_type = 0; - std::string der = var_name.substr(0, 3); - if (der == "d1_") - { - der_type = 1; - var_name = var_name.substr(3); - } - else if (der == "d2_") - { - der_type = 2; - var_name = var_name.substr(3); - } - - // first assume extra_var is a normal evolution var - int var = UserVariables::variable_name_to_enum(var_name); - VariableType var_type = VariableType::evolution; - if (var < 0) - { - // if not an evolution var check if it's a diagnostic var - var = DiagnosticVariables::variable_name_to_enum(var_name); - if (var < 0) - { - // it's neither :( - pout() << "Variable with name " << var_name - << " not found.\n"; - } - else - { - var_type = VariableType::diagnostic; - ++extra_contain_diagnostic; - } - } - if (var >= 0) - { - extra_vars[full_name] = - std::tuple(var, var_type, der_type); - if (der_type == 0) - num_extra_vars += 1; - else if (der_type == 1) - num_extra_vars += CH_SPACEDIM; - else // if (der_type == 2) - num_extra_vars += CH_SPACEDIM * (CH_SPACEDIM + 1) / 2; - } - } - } - - stats_path = a_p.data_path; - - if (pp.contains("AH_coords_subpath")) - { - pp.load("AH_coords_subpath", coords_path); - if (!coords_path.empty() && coords_path.back() != '/') - coords_path += "/"; - if (a_p.output_path != "./" && !a_p.output_path.empty()) - coords_path = a_p.output_path + coords_path; - } - else - coords_path = stats_path; - - pp.load("AH_stats_prefix", stats_prefix, std::string("stats_AH")); - pp.load("AH_coords_prefix", coords_prefix, std::string("coords_AH")); -} - -///////////////////////////////////////////////////////// -// PETSc control methods -///////////////////////////////////////////////////////// - -void AHFinder::set_num_ranks(int a_num_ranks) -{ - if (m_initialized) - return; - -#ifdef CH_MPI - if (m_mpi_group != MPI_GROUP_NULL) - MPI_Group_free(&m_mpi_group); - - if (m_mpi_comm != MPI_COMM_NULL) - MPI_Comm_free(&m_mpi_comm); - - if (a_num_ranks > 0) // otherwise use the whole Chombo communicator - { - // don't make more ranks than there are processes - int size; - MPI_Comm_size(Chombo_MPI::comm, &size); - a_num_ranks = std::min(a_num_ranks, size); - - int rank; - MPI_Comm_rank(Chombo_MPI::comm, &rank); - if (rank == 0) - std::cout << "Using PETSc with " << a_num_ranks << " ranks" - << std::endl; - - MPI_Group MPI_GROUP_WORLD; - MPI_Comm_group(Chombo_MPI::comm, &MPI_GROUP_WORLD); - - // (TF): for things like 'SmallDataIO', rank 0 of Chombo_MPI::comm - // is the one that prints (not rank 0 from PETSC_COMM_WORLD) so pay - // attention before taking rank 0 of Chombo out of PETSc. As of - // 26/05/2019, I tested with 'int range[3] = { 1, a_num_ranks , 1 }' - // and all the AH stuff was compatible with doing so - // Not true anymore (19/08/2020) due to how - // ApparentHorizon::write_coords_file uses rank 0 of PETSc to transfer - // data and to write, but SmallDataIO uses rank 0 of Chombo to write. - int range[3] = {0, a_num_ranks - 1, 1}; - MPI_Group_range_incl(MPI_GROUP_WORLD, 1, - reinterpret_cast(&range), &m_mpi_group); - MPI_Group_free(&MPI_GROUP_WORLD); - - MPI_Comm_create(Chombo_MPI::comm, m_mpi_group, &m_mpi_comm); - } -#endif -} - -//! initialize PETSc and its MPI sub-communicator -PetscErrorCode AHFinder::PETSc_initialize(int a_num_ranks) -{ - set_num_ranks(a_num_ranks); - - PetscErrorCode err = 0; - if (!m_initialized) - { -#ifdef CH_MPI - if (m_mpi_comm != MPI_COMM_NULL) - PETSC_COMM_WORLD = m_mpi_comm; - else // use Chombo communicator if no 'set_num_ranks' was called (or - // if called with 'a_num_ranks'<=0) - PETSC_COMM_WORLD = Chombo_MPI::comm; -#endif - - if (AHFinder::is_rank_active()) - err = PetscInitializeNoArguments(); - - if (!err) - m_initialized = true; - } - return err; -} - -//! finalize PETSc -PetscErrorCode AHFinder::PETSc_finalize() -{ - PetscErrorCode err = 0; - if (m_initialized) - { - if (AHFinder::is_rank_active()) - err = PetscFinalize(); - - if (!err) - m_initialized = false; - } - return err; -} - -//! true if part of PETSc MPI sub-communicator -bool AHFinder::is_rank_active() -{ -#ifdef CH_MPI - if (m_mpi_group == MPI_GROUP_NULL) - return true; - else - { - int rank; - MPI_Group_rank(m_mpi_group, &rank); - return rank != MPI_UNDEFINED; - } -#else - return true; -#endif -} - -///////////////////////////////////////////////////////// -// initialize some "static" variables of AHFinder class -///////////////////////////////////////////////////////// - -bool AHFinder::m_initialized = false; - -#ifdef CH_MPI -MPI_Group AHFinder::m_mpi_group = MPI_GROUP_NULL; -MPI_Comm AHFinder::m_mpi_comm = MPI_COMM_NULL; -#endif - -#endif +#endif // _AHFINDER_IMPL_HPP_ diff --git a/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp b/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp index 940cb6cf2..608b83054 100644 --- a/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp +++ b/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp @@ -10,9 +10,7 @@ #ifndef _AHINTERPOLATION_IMPL_HPP_ #define _AHINTERPOLATION_IMPL_HPP_ -#include "AHFinder.hpp" -#include "DimensionDefinitions.hpp" // make sure GR_SPACEDIM exists -#include "TensorAlgebra.hpp" +#include "PETScCommunicator.hpp" template AHInterpolation::AHInterpolation( @@ -235,7 +233,7 @@ bool AHInterpolation // #include -#include "AHData.hpp" #include "AHDeriv.hpp" -#include "AHFinder.hpp" -#include "AHGeometryData.hpp" #include "AHInterpolation.hpp" +#include "ChomboParameters.hpp" #include "IntegrationMethod.hpp" // Class to manage ApparentHorizon for 2+1D and 3+1D simulations -// Class AHFinder manages it //! AHFunction defines the optimizing function (see AHFunction.hpp for //! expansion example calculation) template class ApparentHorizon { using Interpolation = AHInterpolation; + public: + // prepend with 'AH_' in params file + struct params + { + int num_ranks; //!< number of ranks for PETSc sub-communicator (default + //!< 0, which is 'all') + + int num_points_u; //!< number of points for 2D coordinate grid +#if CH_SPACEDIM == 3 + int num_points_v; //!< number of points for 2D coordinate grid +#endif + int solve_interval; //!< same as checkpoint_interval, for + //!< ApparentHorizon::solve (default 1) + int print_interval; //!< same as solve_interval, but for prints (default + //!< 1) + bool track_center; //!< whether or not to update the center + //!< (set to false if you know it won't move) + //!< (default true) + bool predict_origin; //!< whether or not to estimate where the next + //!< origin will be at (default = track_center) + + int level_to_run; // if negative, it will count backwards (e.g. -1 is + // 'last level') (default 0) + + double start_time; //!< time after which AH can start (default 0.) + //!< Useful for ScalarField collapse + double give_up_time; //!< stop if at this time nothing was found + //!< (<0 to never, which is default) + //!< Useful for ScalarField collapse + + bool allow_re_attempt; //!< re-attempt with initial guess if + //!< previous convergence failed (default false) + int max_fails_after_lost; //!< number of time steps to try again after + //!< (-1 to never) the AH was lost + //!< (default is 0) + + int verbose; //!< print information during execution (default is 1) + + bool print_geometry_data; //!< print metric and extrinsic + //!< curvature of the AHs (default false) + + bool re_solve_at_restart; //!< whether or not to re-run even if AH + //!< already exists (useful in order to be + //!< able to provide an initial guess and + //!< re-run the AH on that time step) + //!< (default false) + + bool stop_if_max_fails; //! breaks the run if AH doesn't converge + //! 'max_fails_after_lost' times or if + //! 'give_up_time' is reached without + //! convergence (default is 'false') + + std::map> + extra_vars; //! extra vars to write to coordinates file () + int num_extra_vars; // total number of extra vars (!=extra_vars.size() + // as derivative count for multiple vars) + + int extra_contain_diagnostic; // not a parameter (set internally); + // counts how many + + std::string stats_path = "", + stats_prefix = + "stats_AH"; //!< name for stats file with + //!< area, spin and AH origin/center + std::string coords_path = "", + coords_prefix = + "coords_AH"; //!< name for coords file with AH + //!< coordinates at each time step + + //! mergers will be searched when distance between 'parent' BHs is + //! distance < merger_search_factor * 4. * (AH_initial_guess_1 + + //! AH_initial_guess_2) should be roughly '2M=2(m1+m2)' for initial + //! guess at m/2 (set to non-positive to 'always search') + double merger_search_factor; // see note above (default is 1) + //! initial guess for merger is 'merger_pre_factor * 4. * + //! (AH_initial_guess_1 + AH_initial_guess_2)' + //! set to somethig bigger to avoid finding the inner AH + double merger_pre_factor; // see note above (default to 1.) + + void read_params(GRParmParse &pp, const ChomboParameters &a_p); + }; + + enum verbose_level + { + NONE, + MIN, // minimal + SOME, // a bit technical + MORE // debug + }; + public: //! AH that finds the zero of expansion ApparentHorizon( const Interpolation &a_interp, //!< Geometry class to exchange data double a_initial_guess, //!< Initial guess for radius (or whatever //!< coordinate you're solving for) - const AHFinder::params &a_params, //!< set of AH parameters + const params &a_params, //!< set of AH parameters const std::string &a_stats = "stats", //!< name for output file with area, spin and AH origin const std::string &a_coords = @@ -42,7 +130,7 @@ template class ApparentHorizon //! 'a_function_to_optimize' (a void* 'a_function_to_optimize_params' can be //! passed for auxiliary parameters passed to 'a_function_to_optimize') ApparentHorizon(const Interpolation &a_interp, double a_initial_guess, - const AHFinder::params &a_params, + const params &a_params, const typename AHFunction::params &a_func_params, const std::string &a_stats = "stats", const std::string &a_coords = "coords_", @@ -76,12 +164,12 @@ template class ApparentHorizon bool do_solve(double a_dt, double a_time) const; //!< decide (based times passed to 'solve') whether or not - //!< to print (uses AHFinder::params::solve_interval) + //!< to print (uses params::solve_interval) bool do_print(double a_dt, double a_time) - const; //!< decide when to print (only AHFinder::params::print_interval + const; //!< decide when to print (only params::print_interval //!< out of all 'solve's) - const AHFinder::params &m_params; //!< set of AH parameters + const params m_params; //!< set of AH parameters const std::string m_stats, m_coords; //!< public base names for output files (no need for a set as //!< they are const) diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp index cc914793b..e8f2d704f 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp @@ -11,6 +11,7 @@ #define _APPARENTHORIZON_IMPL_HPP_ #include "GRAMR.hpp" +#include "PETScCommunicator.hpp" #include "SmallDataIO.hpp" #include "TensorAlgebra.hpp" #include "UserVariables.hpp" @@ -26,9 +27,8 @@ template ApparentHorizon::ApparentHorizon( const AHInterpolation &a_interp, - double a_initial_guess, const AHFinder::params &a_params, - const std::string &a_stats, const std::string &a_coords, - bool solve_first_step) + double a_initial_guess, const params &a_params, const std::string &a_stats, + const std::string &a_coords, bool solve_first_step) : ApparentHorizon(a_interp, a_initial_guess, a_params, AHFunction::params(), a_stats, a_coords, solve_first_step) { @@ -37,7 +37,7 @@ ApparentHorizon::ApparentHorizon( template ApparentHorizon::ApparentHorizon( const AHInterpolation &a_interp, - double a_initial_guess, const AHFinder::params &a_params, + double a_initial_guess, const params &a_params, const typename AHFunction::params &a_func_params, const std::string &a_stats, const std::string &a_coords, bool solve_first_step) @@ -98,6 +98,143 @@ ApparentHorizon::~ApparentHorizon() finalise_PETSc(); } +template +void ApparentHorizon::params::read_params( + GRParmParse &pp, const ChomboParameters &a_p) +{ + pp.load("AH_num_ranks", num_ranks, 0); // 0 means "all" + + pp.load("AH_num_points_u", num_points_u); +#if CH_SPACEDIM == 3 + pp.load("AH_num_points_v", num_points_v); +#endif + + // if box division ends up with size less than 3, PETSc will + // complain (this only gives an estimate of the box side) + int size = 1; +#if CH_MPI + MPI_Comm_size(Chombo_MPI::comm, &size); +#endif + size = std::min(num_ranks, size); +#if CH_SPACEDIM == 3 + CH_assert(this->num_points_u > 0 && this->num_points_v > 0); + CH_assert(this->num_points_u / sqrt(size) >= 3); // make sure for size 'u' + CH_assert(this->num_points_v / sqrt(size) >= 3); // make sure for size 'v' +#elif CH_SPACEDIM == 2 + CH_assert(this->num_points_u > 0); + CH_assert(this->num_points_u / size >= 3); // make sure for size 'u' +#endif + + pp.load("AH_solve_interval", solve_interval, 1); + pp.load("AH_print_interval", print_interval, 1); + pp.load("AH_track_center", track_center, true); + pp.load("AH_predict_origin", predict_origin, track_center); + // can't predict if center is not being tracked + CH_assert(!(predict_origin && !track_center)); + + pp.load("AH_level_to_run", level_to_run, 0); + CH_assert(level_to_run <= a_p.max_level && + level_to_run > -(a_p.max_level + 1)); + if (level_to_run < 0) // if negative, count backwards + level_to_run += a_p.max_level + 1; + + pp.load("AH_start_time", start_time, 0.0); + pp.load("AH_give_up_time", give_up_time, -1.0); + + pp.load("AH_allow_re_attempt", allow_re_attempt, false); + pp.load("AH_max_fails_after_lost", max_fails_after_lost, 0); + pp.load("AH_stop_if_max_fails", stop_if_max_fails, false); + + pp.load("AH_verbose", verbose, (int)MIN); + pp.load("AH_print_geometry_data", print_geometry_data, false); + pp.load("AH_re_solve_at_restart", re_solve_at_restart, false); + + // sanity checks + CH_assert(solve_interval > 0 && print_interval > 0); + CH_assert(level_to_run >= 0 && level_to_run <= a_p.max_level); + + // load vars to write to coord files + num_extra_vars = 0; + extra_contain_diagnostic = 0; + + int AH_num_extra_vars; + pp.load("AH_num_extra_vars", AH_num_extra_vars, 0); + if (AH_num_extra_vars > 0) + { + std::vector AH_extra_var_names(AH_num_extra_vars, ""); + pp.load("AH_extra_vars", AH_extra_var_names, AH_num_extra_vars); + for (const std::string &full_name : AH_extra_var_names) + { + std::string var_name = full_name; + + // variable names might start with "d1_" or "d2_" to indicate + // the user wants derivatives + int der_type = 0; + std::string der = var_name.substr(0, 3); + if (der == "d1_") + { + der_type = 1; + var_name = var_name.substr(3); + } + else if (der == "d2_") + { + der_type = 2; + var_name = var_name.substr(3); + } + + // first assume extra_var is a normal evolution var + int var = UserVariables::variable_name_to_enum(var_name); + VariableType var_type = VariableType::evolution; + if (var < 0) + { + // if not an evolution var check if it's a diagnostic var + var = DiagnosticVariables::variable_name_to_enum(var_name); + if (var < 0) + { + // it's neither :( + pout() << "Variable with name " << var_name + << " not found.\n"; + } + else + { + var_type = VariableType::diagnostic; + ++extra_contain_diagnostic; + } + } + if (var >= 0) + { + extra_vars[full_name] = + std::tuple(var, var_type, der_type); + if (der_type == 0) + num_extra_vars += 1; + else if (der_type == 1) + num_extra_vars += CH_SPACEDIM; + else // if (der_type == 2) + num_extra_vars += CH_SPACEDIM * (CH_SPACEDIM + 1) / 2; + } + } + } + + stats_path = a_p.data_path; + + if (pp.contains("AH_coords_subpath")) + { + pp.load("AH_coords_subpath", coords_path); + if (!coords_path.empty() && coords_path.back() != '/') + coords_path += "/"; + if (a_p.output_path != "./" && !a_p.output_path.empty()) + coords_path = a_p.output_path + coords_path; + } + else + coords_path = stats_path; + + pp.load("AH_stats_prefix", stats_prefix, std::string("stats_AH")); + pp.load("AH_coords_prefix", coords_prefix, std::string("coords_AH")); + + pp.load("AH_merger_search_factor", merger_search_factor, 1.); + pp.load("AH_merger_pre_factor", merger_pre_factor, 1.); +} + template bool ApparentHorizon::good_to_go( double a_dt, double a_time) const @@ -157,7 +294,7 @@ void ApparentHorizon::set_origin( m_interp_minus.set_origin(a_origin); origin_already_updated = true; - if (m_params.verbose > AHFinder::SOME) + if (m_params.verbose > SOME) { pout() << "Setting origin to (" << a_origin[0] << "," << a_origin[1] #if CH_SPACEDIM == 3 @@ -236,7 +373,7 @@ void ApparentHorizon::reset_initial_guess() { CH_TIME("ApparentHorizon::reset_initial_guess"); - if (!AHFinder::is_rank_active()) + if (!PETScCommunicator::is_rank_active()) return; auto origin = get_origin(); @@ -245,7 +382,7 @@ void ApparentHorizon::reset_initial_guess() bool out_of_grid = m_interp.is_in_grid(origin, m_initial_guess); CH_assert(!out_of_grid); - if (m_params.verbose > AHFinder::NONE) + if (m_params.verbose > NONE) { pout() << "Setting Initial Guess to f=" << m_initial_guess << " centered at (" << origin[0] << "," << origin[1] @@ -295,7 +432,7 @@ void ApparentHorizon::predict_next_origin() 2. * m_old_centers[1][a]); } } - if (m_params.verbose > AHFinder::SOME) + if (m_params.verbose > SOME) { pout() << "OLD[-2]: (" << m_old_centers[2][0] << "," << m_old_centers[2][1] @@ -318,7 +455,7 @@ void ApparentHorizon::predict_next_origin() } } - if (m_params.verbose > AHFinder::SOME) + if (m_params.verbose > SOME) { pout() << "OLD[-1]: (" << m_old_centers[1][0] << "," << m_old_centers[1][1] @@ -333,7 +470,7 @@ void ApparentHorizon::predict_next_origin() #endif << ")" << std::endl; } - if (m_params.verbose > AHFinder::MIN) + if (m_params.verbose > MIN) { pout() << "Estimated center: (" << new_center[0] << "," << new_center[1] @@ -401,12 +538,12 @@ void ApparentHorizon::solve(double a_dt, PetscInt its; SNESGetIterationNumber(m_snes, &its); - if (m_params.verbose > AHFinder::MIN) + if (m_params.verbose > MIN) { pout() << "SNES Iteration number " << its << endl; } SNESGetLinearSolveIterations(m_snes, &its); - if (m_params.verbose > AHFinder::MIN) + if (m_params.verbose > MIN) { pout() << "KSP Iteration number " << its << endl; } @@ -414,7 +551,7 @@ void ApparentHorizon::solve(double a_dt, } { - if (m_params.verbose > AHFinder::SOME) + if (m_params.verbose > SOME) { pout() << "In [ApparentHorizon::solve::post-solving]" << endl; } @@ -433,7 +570,7 @@ void ApparentHorizon::solve(double a_dt, if (save_converged && !get_converged() && m_params.allow_re_attempt) { --m_num_failed_convergences; // reduce failed and try again - if (m_params.verbose > AHFinder::NONE) + if (m_params.verbose > NONE) { pout() << "Re-attempting to solve using initial guess." << std::endl; @@ -468,13 +605,13 @@ void ApparentHorizon::solve(double a_dt, m_spin_z_alt = calculate_spin_dimensionless(m_area); #endif - if (m_params.verbose > AHFinder::NONE) + if (m_params.verbose > NONE) { pout() << "mass = " << m_mass << endl; #if CH_SPACEDIM == 3 pout() << "spin = " << m_spin << endl; #endif - if (m_params.verbose > AHFinder::MIN) + if (m_params.verbose > MIN) { pout() << "irreducible mass = " << m_irreducible_mass << endl; #if CH_SPACEDIM == 3 @@ -510,7 +647,7 @@ void ApparentHorizon::solve(double a_dt, MayDay::Error("Reached max fails. Stopping. Parameter " "'stop_if_max_fails' is set to true."); - if (m_params.verbose > AHFinder::SOME) + if (m_params.verbose > SOME) { pout() << "ApparentHorizon::solve finished successfully!" << endl; } @@ -525,7 +662,7 @@ void ApparentHorizon::write_outputs( // the frequency of writing outputs) and the class will still be able to // restart from the correct file (this only applies for frequency increase, // for which the file number will suddenly jump, as for frequency decrease - // the AHFinder may override old coord files) + // the AH may override old coord files) // print stats (area, spin, origin, center) and coordinates // stop printing if it stopped converging @@ -533,7 +670,7 @@ void ApparentHorizon::write_outputs( if (do_print(a_dt, a_time)) { CH_TIME("ApparentHorizon::solve::printing"); - if (m_params.verbose > AHFinder::MIN) + if (m_params.verbose > MIN) { pout() << "Printing statistics and coordinates." << std::endl; } @@ -636,7 +773,7 @@ void ApparentHorizon::check_convergence() // https://www.mcs.anl.gov/petsc/petsc-current/docs/manualpages/SNES/SNESConvergedReason.html#SNESConvergedReason int result; - if (AHFinder::is_rank_active()) + if (PETScCommunicator::is_rank_active()) { SNESConvergedReason reason; SNESGetConvergedReason(m_snes, &reason); @@ -673,13 +810,13 @@ void ApparentHorizon::check_convergence() ++m_num_failed_convergences; } - if (m_params.verbose > AHFinder::NONE) + if (m_params.verbose > NONE) { pout() << (m_converged ? "Solver converged. Horizon FOUND." : "Solver diverged. Horizon NOT found.") << std::endl; - if (m_params.verbose > AHFinder::MIN) + if (m_params.verbose > MIN) { pout() << "SNESConvergedReason = " << result << std::endl; } @@ -733,7 +870,7 @@ void ApparentHorizon::restart( double old_print_dt = 0.; if (stats.size() == 0) { // case when it never ran the AH or the file doesn't exist - if (m_params.verbose > AHFinder::NONE && current_step != 0) + if (m_params.verbose > NONE && current_step != 0) { pout() << "Empty stats file '" << file << "'. Assuming AHFinder was not run yet for this AH." @@ -764,7 +901,7 @@ void ApparentHorizon::restart( else // if we can't know, just default to the current one old_print_dt = current_solve_dt * m_params.print_interval; - if (m_params.verbose > AHFinder::SOME) + if (m_params.verbose > SOME) { if (idx >= 0) pout() << "stats[0][idx] = " << stats[0][idx] << std::endl; @@ -778,7 +915,7 @@ void ApparentHorizon::restart( if (idx < 0) { // case when job was restarted at a point before the AH was first ever // found - if (m_params.verbose > AHFinder::NONE) + if (m_params.verbose > NONE) { pout() << "First time step is after restart in '" << file @@ -795,7 +932,7 @@ void ApparentHorizon::restart( -(old_print_dt == 0. ? eps : old_print_dt - eps)) { // case when the PETSc stopped converging and stopped printing // so there is a mismatch with the last time in the file - if (m_params.verbose > AHFinder::NONE) + if (m_params.verbose > NONE) { pout() << "Last time step not found in '" << file << "'. Assuming AH stopped converging." << std::endl; @@ -811,7 +948,7 @@ void ApparentHorizon::restart( // having converged // OR when job was restarted at a point before the AH was first ever // found - if (m_params.verbose > AHFinder::NONE) + if (m_params.verbose > NONE) { pout() << "Last time step is NAN in '" << file << "'. Assuming AH wasn't found and PETSc didn't " @@ -826,7 +963,7 @@ void ApparentHorizon::restart( } else { // case when the AH was found and there is a coordinate file - if (m_params.verbose > AHFinder::NONE) + if (m_params.verbose > NONE) { pout() << "Last time step found in '" << file << "'. Reading coordinates file." << std::endl; @@ -875,7 +1012,7 @@ void ApparentHorizon::restart( int offset = CH_SPACEDIM + was_center_tracked * CH_SPACEDIM; FOR1(a) { origin[a] = stats[cols - offset + a][idx]; } - if (m_params.verbose > AHFinder::NONE) + if (m_params.verbose > NONE) { pout() << "Setting origin from stats file '" << m_params.stats_path + m_stats << "' at (" << origin[0] @@ -898,8 +1035,7 @@ void ApparentHorizon::restart( // SET OLD CENTERS directly if old 'dt' matches current 'dt' if (std::abs(old_print_dt - current_solve_dt) < eps) { - if (m_params.verbose > AHFinder::NONE && - m_params.predict_origin) + if (m_params.verbose > NONE && m_params.predict_origin) { pout() << "Reading old centers to predict next origin." << std::endl; @@ -942,8 +1078,7 @@ void ApparentHorizon::restart( old_centers_time_index.push_back(index); } - if (m_params.verbose > AHFinder::NONE && - m_params.predict_origin) + if (m_params.verbose > NONE && m_params.predict_origin) { if (old_centers_time_index.size() == 0) pout() << "AH time step changed and no old centers " @@ -985,7 +1120,7 @@ void ApparentHorizon::restart( m_converged++; update_old_centers(old_center); - if (m_params.verbose > AHFinder::SOME) + if (m_params.verbose > SOME) { pout() << "OLD[-" << i + 1 << "] = (" << old_center[0] << "," << old_center[1] @@ -1019,14 +1154,14 @@ void ApparentHorizon::restart( auto coords = SmallDataIO::read(coords_filename); - if (m_params.verbose > AHFinder::NONE) + if (m_params.verbose > NONE) { pout() << "Setting Initial Guess to previous file '" << coords_filename << "' found." << std::endl; } // now write local points based on file (or interpolated file) - if (AHFinder::is_rank_active()) + if (PETScCommunicator::is_rank_active()) { // read PETSc array to 'f' dmda_arr_t f; @@ -1073,7 +1208,7 @@ void ApparentHorizon::write_coords_file( return; } - if (m_params.verbose > AHFinder::NONE && write_geometry_data) + if (m_params.verbose > NONE && write_geometry_data) pout() << "Writing geometry data." << std::endl; CH_assert(a_dt != 0); // Check if time was set!! @@ -1152,7 +1287,7 @@ void ApparentHorizon::write_coords_file( num_components + m_params.num_extra_vars + CH_SPACEDIM; // step 1 - if (AHFinder::is_rank_active()) + if (PETScCommunicator::is_rank_active()) { #ifdef CH_MPI @@ -1775,7 +1910,7 @@ double ApparentHorizon::calculate_area() area = integral; #endif - if (m_params.verbose > AHFinder::MIN) + if (m_params.verbose > MIN) { pout() << "area = " << area << endl; } @@ -1811,9 +1946,10 @@ ApparentHorizon::calculate_center() std::array min_temp = max_temp; int idx = 0; - if (AHFinder::is_rank_active()) // in principle this wouldn't be needed, as - // non-PETSc processes wouldn't even enter - // the loop anyways + if (PETScCommunicator::is_rank_active()) // in principle this wouldn't be + // needed, as non-PETSc processes + // wouldn't even enter the loop + // anyways { #if CH_SPACEDIM == 3 for (int v = m_vmin; v < m_vmax; ++v) @@ -1870,7 +2006,7 @@ ApparentHorizon::calculate_center() center[i] = (max_point[i] + min_point[i]) / 2.; } - if (m_params.verbose > AHFinder::SOME) + if (m_params.verbose > SOME) { pout() << "max_point: (" << max_point[0] << ", " << max_point[1] #if CH_SPACEDIM == 3 @@ -1886,7 +2022,7 @@ ApparentHorizon::calculate_center() update_old_centers(center); // set new - if (m_params.verbose > AHFinder::MIN) + if (m_params.verbose > MIN) { pout() << "center: (" << center[0] << ", " << center[1] #if CH_SPACEDIM == 3 @@ -1906,7 +2042,7 @@ void ApparentHorizon::calculate_minmax_F() const double local_max = std::numeric_limits::min(); double local_min = std::numeric_limits::max(); - if (AHFinder::is_rank_active()) + if (PETScCommunicator::is_rank_active()) { auto local_minmax = (std::minmax_element(m_F.begin(), m_F.end())); local_min = *(local_minmax.first); @@ -1935,7 +2071,7 @@ void ApparentHorizon::calculate_average_F() const int local_size = 0; double local_sum = 0.; double local_sum_sq = 0.; - if (AHFinder::is_rank_active()) + if (PETScCommunicator::is_rank_active()) { std::pair sums(0., 0.); auto lambda_sum = [](std::pair sums, double r) { diff --git a/Source/ApparentHorizonFinder/ApparentHorizon_petsc.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon_petsc.impl.hpp index d2a4a0680..ef85723d2 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon_petsc.impl.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon_petsc.impl.hpp @@ -30,7 +30,7 @@ void ApparentHorizon::initialise_PETSc() { CH_TIME("ApparentHorizon::initialise_PETSc"); - if (!AHFinder::is_rank_active()) + if (!PETScCommunicator::is_rank_active()) return; #if PETSC_VERSION_LT(3, 5, 0) @@ -134,7 +134,7 @@ void ApparentHorizon::initialise_PETSc() SNESSetFromOptions(m_snes); - if (m_params.verbose > AHFinder::MIN) + if (m_params.verbose > MIN) { SNESType snes_type; SNESGetType(m_snes, &snes_type); @@ -177,7 +177,7 @@ void ApparentHorizon::initialise_PETSc() template void ApparentHorizon::finalise_PETSc() { - if (!AHFinder::is_rank_active()) + if (!PETScCommunicator::is_rank_active()) return; SNESDestroy(&m_snes); @@ -265,7 +265,7 @@ bool ApparentHorizon::interpolate_ah( return false; } - if (m_params.verbose > AHFinder::NONE) + if (m_params.verbose > NONE) { pout() << "Number of AH points changed. Interpolating old ones." << std::endl; @@ -348,7 +348,7 @@ bool ApparentHorizon::interpolate_ah( } } - if (m_params.verbose > AHFinder::NONE) + if (m_params.verbose > NONE) { pout() << "Interpolation Successfull." << std::endl; } diff --git a/Source/ApparentHorizonFinder/PETScCommunicator.cpp b/Source/ApparentHorizonFinder/PETScCommunicator.cpp new file mode 100644 index 000000000..9593b2586 --- /dev/null +++ b/Source/ApparentHorizonFinder/PETScCommunicator.cpp @@ -0,0 +1,124 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifdef USE_AHFINDER + +#include "PETScCommunicator.hpp" + +void PETScCommunicator::set_num_ranks(int a_num_ranks) +{ + if (m_initialized) + return; + +#ifdef CH_MPI + if (m_mpi_group != MPI_GROUP_NULL) + MPI_Group_free(&m_mpi_group); + + if (m_mpi_comm != MPI_COMM_NULL) + MPI_Comm_free(&m_mpi_comm); + + if (a_num_ranks > 0) // otherwise use the whole Chombo communicator + { + // don't make more ranks than there are processes + int size; + MPI_Comm_size(Chombo_MPI::comm, &size); + a_num_ranks = std::min(a_num_ranks, size); + + int rank; + MPI_Comm_rank(Chombo_MPI::comm, &rank); + if (rank == 0) + std::cout << "Using PETSc with " << a_num_ranks << " ranks" + << std::endl; + + MPI_Group MPI_GROUP_WORLD; + MPI_Comm_group(Chombo_MPI::comm, &MPI_GROUP_WORLD); + + // (TF): for things like 'SmallDataIO', rank 0 of Chombo_MPI::comm + // is the one that prints (not rank 0 from PETSC_COMM_WORLD) so pay + // attention before taking rank 0 of Chombo out of PETSc. As of + // 26/05/2019, I tested with 'int range[3] = { 1, a_num_ranks , 1 }' + // and all the AH stuff was compatible with doing so + // Not true anymore (19/08/2020) due to how + // ApparentHorizon::write_coords_file uses rank 0 of PETSc to transfer + // data and to write, but SmallDataIO uses rank 0 of Chombo to write. + int range[3] = {0, a_num_ranks - 1, 1}; + MPI_Group_range_incl(MPI_GROUP_WORLD, 1, + reinterpret_cast(&range), &m_mpi_group); + MPI_Group_free(&MPI_GROUP_WORLD); + + MPI_Comm_create(Chombo_MPI::comm, m_mpi_group, &m_mpi_comm); + } +#endif +} + +//! initialize PETSc and its MPI sub-communicator +PetscErrorCode PETScCommunicator::initialize(int a_num_ranks) +{ + set_num_ranks(a_num_ranks); + + PetscErrorCode err = 0; + if (!m_initialized) + { +#ifdef CH_MPI + if (m_mpi_comm != MPI_COMM_NULL) + PETSC_COMM_WORLD = m_mpi_comm; + else // use Chombo communicator if no 'set_num_ranks' was called (or + // if called with 'a_num_ranks'<=0) + PETSC_COMM_WORLD = Chombo_MPI::comm; +#endif + + if (PETScCommunicator::is_rank_active()) + err = PetscInitializeNoArguments(); + + if (!err) + m_initialized = true; + } + return err; +} + +//! finalize PETSc +PetscErrorCode PETScCommunicator::finalize() +{ + PetscErrorCode err = 0; + if (m_initialized) + { + if (PETScCommunicator::is_rank_active()) + err = PetscFinalize(); + + if (!err) + m_initialized = false; + } + return err; +} + +//! true if part of PETSc MPI sub-communicator +bool PETScCommunicator::is_rank_active() +{ +#ifdef CH_MPI + if (m_mpi_group == MPI_GROUP_NULL) + return true; + else + { + int rank; + MPI_Group_rank(m_mpi_group, &rank); + return rank != MPI_UNDEFINED; + } +#else + return true; +#endif +} + +///////////////////////////////////////////////////////// +// initialize some "static" variables of PETScCommunicator class +///////////////////////////////////////////////////////// + +bool PETScCommunicator::m_initialized = false; + +#ifdef CH_MPI +MPI_Group PETScCommunicator::m_mpi_group = MPI_GROUP_NULL; +MPI_Comm PETScCommunicator::m_mpi_comm = MPI_COMM_NULL; +#endif + +#endif diff --git a/Source/ApparentHorizonFinder/PETScCommunicator.hpp b/Source/ApparentHorizonFinder/PETScCommunicator.hpp new file mode 100644 index 000000000..155441e47 --- /dev/null +++ b/Source/ApparentHorizonFinder/PETScCommunicator.hpp @@ -0,0 +1,45 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef _PETSCCOMMUNICATOR_HPP_ +#define _PETSCCOMMUNICATOR_HPP_ + +// Chombo includes +#include "SPMD.H" //Chombo_MPI::comm + +#ifdef CH_MPI +#include +#endif +#include + +// Chombo namespace +#include "UsingNamespace.H" + +//! Class to control PETSc MPI sub-communicator +class PETScCommunicator +{ + public: + //! true if part of PETSc MPI sub-communicator + static bool is_rank_active(); + + //! initialize PETSc and its MPI sub-communicator + static PetscErrorCode initialize(int a_num_ranks); + + //! finalize PETSc + static PetscErrorCode finalize(); + + private: + static bool m_initialized; //!< is initialized? + +#ifdef CH_MPI + static MPI_Group m_mpi_group; //!< set of MPI ranks for PETSc + static MPI_Comm m_mpi_comm; //!< MPI sub-communicator +#endif + + //! define number of ranks of PETSc sub-communicator + static void set_num_ranks(int a_num_ranks); +}; // namespace AHFinder + +#endif /* _PETSCCOMMUNICATOR_HPP_ */ diff --git a/Source/BlackHoles/BHAMR.hpp b/Source/BlackHoles/BHAMR.hpp index 9febe1ddc..43989941f 100644 --- a/Source/BlackHoles/BHAMR.hpp +++ b/Source/BlackHoles/BHAMR.hpp @@ -28,7 +28,7 @@ class BHAMR : public GRAMR #endif #ifdef USE_AHFINDER - AHFinder m_ah_finder; + AHFinder m_ah_finder; #endif BHAMR() {} diff --git a/Source/GRChomboCore/SimulationParametersBase.hpp b/Source/GRChomboCore/SimulationParametersBase.hpp index b01a9e029..d50b72078 100644 --- a/Source/GRChomboCore/SimulationParametersBase.hpp +++ b/Source/GRChomboCore/SimulationParametersBase.hpp @@ -327,7 +327,7 @@ class SimulationParametersBase : public ChomboParameters #ifdef USE_AHFINDER bool AH_activate; - AHFinder::params AH_params; + AHFinder::params AH_params; #endif }; diff --git a/Tests/ApparentHorizonFinderTest2D/ApparentHorizonTest2D.cpp b/Tests/ApparentHorizonFinderTest2D/ApparentHorizonTest2D.cpp index 06b36feb7..401a29b09 100644 --- a/Tests/ApparentHorizonFinderTest2D/ApparentHorizonTest2D.cpp +++ b/Tests/ApparentHorizonFinderTest2D/ApparentHorizonTest2D.cpp @@ -36,7 +36,6 @@ recompile again for 2D when compiling this example) #ifdef USE_AHFINDER #include "AHFinder.hpp" -#include "ApparentHorizon.hpp" #endif int runApparentHorizonTest2D(int argc, char *argv[]) @@ -63,25 +62,22 @@ int runApparentHorizonTest2D(int argc, char *argv[]) sim_params.verbosity); gr_amr.set_interpolator(&interpolator); - AHFinder ah_finder; + AHFinder ah_finder; AHStringGeometry sph(sim_params.L); - AHFinder::params AH_params = {1, 20, 1, 1, false, false, 0, 0., -1.}; - AH_params.verbose = 3; - // use 2. as initial guess, doesn't matter much (as long as it converges to // the correct solution) ah_finder.set_interpolator(&interpolator); - ah_finder.add_ah(sph, 2., AH_params); + ah_finder.add_ah(sph, 2., sim_params.AH_params); if (!ah_finder.get(0)->get_converged()) - status = 1; + status = 3; else { // get area from file to determine status auto stats = SmallDataIO::read("stats_AH1.dat"); if (stats.size() == 0) - status = 1; + status = 2; else { double area = stats[2][0]; diff --git a/Tests/ApparentHorizonFinderTest2D/ApparentHorizonTest2D.inputs b/Tests/ApparentHorizonFinderTest2D/ApparentHorizonTest2D.inputs index 4c0106cf0..8572e23e6 100644 --- a/Tests/ApparentHorizonFinderTest2D/ApparentHorizonTest2D.inputs +++ b/Tests/ApparentHorizonFinderTest2D/ApparentHorizonTest2D.inputs @@ -23,3 +23,14 @@ max_grid_size = 32 block_factor = 4 tag_buffer_size = 3 checkpoint_interval = 1 + +AH_num_ranks = 1 +AH_num_points_u = 20 +AH_solve_interval = 1 +AH_print_interval = 1 +AH_track_center = false +AH_predict_origin = false +AH_level_to_run = 0 +AH_start_time = 0 +AH_give_up_time = -1. +AH_verbose = 3 \ No newline at end of file diff --git a/Tests/ApparentHorizonFinderTest2D/SimulationParameters.hpp b/Tests/ApparentHorizonFinderTest2D/SimulationParameters.hpp index adc9824b2..a4b7685b0 100755 --- a/Tests/ApparentHorizonFinderTest2D/SimulationParameters.hpp +++ b/Tests/ApparentHorizonFinderTest2D/SimulationParameters.hpp @@ -9,10 +9,23 @@ // General includes #include "ChomboParameters.hpp" +#ifdef USE_AHFINDER +#include "AHFinder.hpp" +#endif + class SimulationParameters : public ChomboParameters { public: - SimulationParameters(GRParmParse &pp) : ChomboParameters(pp) {} + SimulationParameters(GRParmParse &pp) : ChomboParameters(pp) + { +#ifdef USE_AHFINDER + AH_params.read_params(pp, *this); +#endif + } + +#ifdef USE_AHFINDER + AHFinder::params AH_params; +#endif }; #endif /* SIMULATIONPARAMETERS_HPP_ */ diff --git a/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.cpp b/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.cpp index d070b325d..4761ffaa3 100755 --- a/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.cpp +++ b/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.cpp @@ -30,17 +30,11 @@ using std::endl; // Problem specific includes: #include "AMRInterpolator.hpp" #include "ApparentHorizonTest3DLevel.hpp" -#include "InterpolationQuery.hpp" #include "Lagrange.hpp" #include "SmallDataIO.hpp" -#include "UserVariables.hpp" - -#ifdef _OPENMP -#include -#endif #ifdef USE_AHFINDER -#include "ApparentHorizon.hpp" +#include "AHFinder.hpp" #endif int runApparentHorizonTest3D(int argc, char *argv[]) @@ -60,18 +54,6 @@ int runApparentHorizonTest3D(int argc, char *argv[]) int status = 0; #ifdef USE_AHFINDER - AHFinder::params AH_params = {1, - sim_params.AH_num_points_u, - sim_params.AH_num_points_v, - 1, - 1, - false, - false, - 0, - 0., - -1.}; - AH_params.verbose = 3; - // Set up interpolator and PETSc subcommunicator when AH extraction is // active AMRInterpolator> interpolator( @@ -80,17 +62,18 @@ int runApparentHorizonTest3D(int argc, char *argv[]) bh_amr.set_interpolator(&interpolator); AHSurfaceGeometry sph(sim_params.center); - bh_amr.m_ah_finder.add_ah(sph, sim_params.initial_guess, AH_params); + bh_amr.m_ah_finder.add_ah(sph, sim_params.initial_guess, + sim_params.AH_params); if (!bh_amr.m_ah_finder.get(0)->get_converged()) - status = 1; + status = 3; else { // get area from file to determine status auto stats = SmallDataIO::read("stats_AH1.dat"); if (stats.size() == 0) - status = 1; + status = 2; else { double mass = stats[3][0]; diff --git a/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.inputs b/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.inputs index 083e4aafe..58e3d33b1 100755 --- a/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.inputs +++ b/Tests/ApparentHorizonFinderTest3D/ApparentHorizonTest3D.inputs @@ -32,5 +32,14 @@ kerr_spin = 5. kerr_spin_direction = 1. 0. 0. #AH_initial_guess = 5 +AH_num_ranks = 1 AH_num_points_u = 21 -AH_num_points_v = 20 \ No newline at end of file +AH_num_points_v = 20 +AH_solve_interval = 1 +AH_print_interval = 1 +AH_track_center = false +AH_predict_origin = false +AH_level_to_run = 0 +AH_start_time = 0 +AH_give_up_time = -1. +AH_verbose = 3 \ No newline at end of file diff --git a/Tests/ApparentHorizonFinderTest3D/SimulationParameters.hpp b/Tests/ApparentHorizonFinderTest3D/SimulationParameters.hpp index 5578ce6ca..e7f39c181 100755 --- a/Tests/ApparentHorizonFinderTest3D/SimulationParameters.hpp +++ b/Tests/ApparentHorizonFinderTest3D/SimulationParameters.hpp @@ -12,6 +12,10 @@ #include "KerrBH.hpp" +#ifdef USE_AHFINDER +#include "AHFinder.hpp" +#endif + class SimulationParameters : public ChomboParameters { public: @@ -31,15 +35,14 @@ class SimulationParameters : public ChomboParameters #ifdef USE_AHFINDER pp.load("AH_initial_guess", initial_guess, kerr_params.mass * 0.5); - pp.load("AH_num_points_u", AH_num_points_u); - pp.load("AH_num_points_v", AH_num_points_v); + AH_params.read_params(pp, *this); #endif } KerrBH::params_t kerr_params; #ifdef USE_AHFINDER double initial_guess; - int AH_num_points_u, AH_num_points_v; + AHFinder::params AH_params; #endif }; From 1c60880d5546eef6edfac52f51af2b364f182352 Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Wed, 30 Jun 2021 00:43:06 +0100 Subject: [PATCH 26/92] Move PETSc specifics of ApparentHorizon to separate class --- Source/ApparentHorizonFinder/AHFinder.hpp | 38 +- .../ApparentHorizonFinder/AHFinder.impl.hpp | 39 +- .../ApparentHorizonFinder/AHInterpolation.hpp | 6 +- .../AHInterpolation.impl.hpp | 44 +- Source/ApparentHorizonFinder/AHParams.hpp | 237 +++++++ .../ApparentHorizonFinder/ApparentHorizon.hpp | 255 +------- .../ApparentHorizon.impl.hpp | 594 +++++------------- .../ApparentHorizonFinder/PETScAHSolver.hpp | 178 ++++++ ..._petsc.impl.hpp => PETScAHSolver.impl.hpp} | 386 ++++++++---- .../GRChomboCore/SimulationParametersBase.hpp | 2 +- .../SimulationParameters.hpp | 2 +- .../SimulationParameters.hpp | 2 +- 12 files changed, 943 insertions(+), 840 deletions(-) create mode 100644 Source/ApparentHorizonFinder/AHParams.hpp create mode 100644 Source/ApparentHorizonFinder/PETScAHSolver.hpp rename Source/ApparentHorizonFinder/{ApparentHorizon_petsc.impl.hpp => PETScAHSolver.impl.hpp} (73%) diff --git a/Source/ApparentHorizonFinder/AHFinder.hpp b/Source/ApparentHorizonFinder/AHFinder.hpp index ec93f0386..029efbe07 100644 --- a/Source/ApparentHorizonFinder/AHFinder.hpp +++ b/Source/ApparentHorizonFinder/AHFinder.hpp @@ -16,6 +16,8 @@ #endif #include +#include "AHInterpolation.hpp" +#include "AHParams.hpp" #include "ApparentHorizon.hpp" #include "ChomboParameters.hpp" #include "Lagrange.hpp" @@ -77,20 +79,8 @@ is more senstitive). //! Class to manage AHs and its mergers + control PETSc MPI sub-communicator template class AHFinder { - public: - struct params : ApparentHorizon::params - { - }; - - private: - //! if this AH is supposed to track the formation of a merger - //! this pair indicates the indices of the 2 AHs in the vector - //! 'm_apparent_horizons' - std::vector> m_merger_pairs; - AMRInterpolator> *m_interpolator; //!< The interpolator pointer - - std::vector *> - m_apparent_horizons; //!< public in case user wants to solve by himself + using AHInterpolation = AHInterpolation_t; + using AHParams = AHParams_t; public: AHFinder(){}; @@ -106,9 +96,9 @@ template class AHFinder //! returns the index of the AH in m_apparent_horizons int add_ah(const SurfaceGeometry &a_coord_system, - double a_initial_guess, //!< Initial guess for radius (or whatever - //!< coordinate you're solving for) - const params &a_params, //!< set of AH parameters + double a_initial_guess, //!< Initial guess for radius (or whatever + //!< coordinate you're solving for) + const AHParams &a_params, //!< set of AH parameters bool solve_first_step = true //!< whether or not to solve if t=0 ); @@ -116,12 +106,12 @@ template class AHFinder //! allows for personalized optimizer that finds zero of function //! 'AHFunction' (that can have some ::params) int add_ah(const SurfaceGeometry &a_coord_system, double a_initial_guess, - const params &a_params, + const AHParams &a_params, const typename AHFunction::params &a_func_params, bool solve_first_step = true); // returns the index of the AH in m_apparent_horizons - int add_ah_merger(int ah1, int ah2, const params &a_params); + int add_ah_merger(int ah1, int ah2, const AHParams &a_params); //! Find AH; Calculate area and spin; Update center; Print outputs void solve(double a_dt, double a_time, double a_restart_time); @@ -146,6 +136,16 @@ template class AHFinder bool solve_merger(int ah1, int ah2, double &initial_guess_merger, std::array &origin_merged); + private: + //! if this AH is supposed to track the formation of a merger + //! this pair indicates the indices of the 2 AHs in the vector + //! 'm_apparent_horizons' + std::vector> m_merger_pairs; + AMRInterpolator> *m_interpolator; //!< The interpolator pointer + + std::vector *> + m_apparent_horizons; //!< public in case user wants to solve by himself + }; // namespace AHFinder #include "AHFinder.impl.hpp" diff --git a/Source/ApparentHorizonFinder/AHFinder.impl.hpp b/Source/ApparentHorizonFinder/AHFinder.impl.hpp index f6b9e4805..50e457e86 100644 --- a/Source/ApparentHorizonFinder/AHFinder.impl.hpp +++ b/Source/ApparentHorizonFinder/AHFinder.impl.hpp @@ -10,8 +10,6 @@ #ifndef _AHFINDER_IMPL_HPP_ #define _AHFINDER_IMPL_HPP_ -#include "AHInterpolation.hpp" -#include "ApparentHorizon.hpp" #include "FilesystemTools.hpp" template @@ -26,8 +24,7 @@ AHFinder::~AHFinder() template int AHFinder::add_ah( const SurfaceGeometry &a_coord_system, double a_initial_guess, - const AHFinder::params &a_params, - bool solve_first_step) + const AHParams &a_params, bool solve_first_step) { return add_ah(a_coord_system, a_initial_guess, a_params, typename AHFunction::params(), solve_first_step); @@ -36,8 +33,8 @@ int AHFinder::add_ah( template int AHFinder::add_ah( const SurfaceGeometry &a_coord_system, double a_initial_guess, - const AHFinder::params &a_params, - const typename AHFunction::params &a_func_params, bool solve_first_step) + const AHParams &a_params, const typename AHFunction::params &a_func_params, + bool solve_first_step) { PETScCommunicator::initialize(a_params.num_ranks); @@ -50,8 +47,7 @@ int AHFinder::add_ah( // determine how many AH there are already const int num_ah = m_apparent_horizons.size(); - AHInterpolation interp(a_coord_system, - m_interpolator); + AHInterpolation interp(a_coord_system, m_interpolator); m_apparent_horizons.push_back( new ApparentHorizon( @@ -65,8 +61,8 @@ int AHFinder::add_ah( } template -int AHFinder::add_ah_merger(int ah1, int ah2, - const params &a_params) +int AHFinder::add_ah_merger( + int ah1, int ah2, const AHParams &a_params) { CH_assert(a_params.track_center); // center tracking must be on for mergers @@ -84,7 +80,8 @@ int AHFinder::add_ah_merger(int ah1, int ah2, m_apparent_horizons[ah1]->get_ah_interp().get_coord_system(); coord_system.set_origin(origin_merger); - auto &function_to_optimize_params = m_apparent_horizons[ah1]->m_func_params; + auto &function_to_optimize_params = + m_apparent_horizons[ah1]->get_petsc_solver().m_func_params; int num = add_ah(coord_system, initial_guess_merger, a_params, function_to_optimize_params, do_solve); @@ -169,7 +166,7 @@ void AHFinder::solve(double a_dt, double a_time, m_apparent_horizons[pair.second]->has_been_found())) { if (m_apparent_horizons[i]->m_params.verbose > - ApparentHorizon::NONE) + AHParams::NONE) { pout() << "Skipping BH #" << i << std::endl; } @@ -183,7 +180,7 @@ void AHFinder::solve(double a_dt, double a_time, { // if one of the 'parents' is a merger too far away to be // solved, skip children too if (m_apparent_horizons[i]->m_params.verbose > - ApparentHorizon::NONE) + AHParams::NONE) { pout() << "Skipping BH #" << i << std::endl; } @@ -226,8 +223,7 @@ void AHFinder::solve(double a_dt, double a_time, // the past if (m_apparent_horizons[i]->good_to_go(a_dt, a_time)) { - if (m_apparent_horizons[i]->m_params.verbose > - ApparentHorizon::NONE) + if (m_apparent_horizons[i]->m_params.verbose > AHParams::NONE) pout() << "Solving AH #" << i << std::endl; m_apparent_horizons[i]->solve(a_dt, a_time, a_restart_time); @@ -237,8 +233,7 @@ void AHFinder::solve(double a_dt, double a_time, } else { - if (m_apparent_horizons[i]->m_params.verbose > - ApparentHorizon::NONE) + if (m_apparent_horizons[i]->m_params.verbose > AHParams::NONE) { pout() << "Skipping BH #" << i << ". Not good to go." << std::endl; @@ -274,8 +269,8 @@ bool AHFinder::solve_merger( auto center1 = AH1->get_center(); auto center2 = AH2->get_center(); - auto initial_guess1 = AH1->get_initial_guess(); - auto initial_guess2 = AH2->get_initial_guess(); + auto initial_guess1 = AH1->get_petsc_solver().get_initial_guess(); + auto initial_guess2 = AH2->get_petsc_solver().get_initial_guess(); if (m_merger_pairs[ah1].first >= 0) { @@ -346,8 +341,7 @@ bool AHFinder::solve_merger( // distance CH_assert(min_distance <= 2. * initial_guess_merger); - if (AH1->m_params.verbose > - ApparentHorizon::NONE) + if (AH1->m_params.verbose > AHParams::NONE) { pout() << "BHs #" << ah1 << " and #" << ah2 << " at distance = " << distance; @@ -367,8 +361,7 @@ bool AHFinder::solve_merger( } else { - if (AH1->m_params.verbose > - ApparentHorizon::NONE) + if (AH1->m_params.verbose > AHParams::NONE) { pout() << "Original BH not yet found. Skipping solve for merger." << std::endl; diff --git a/Source/ApparentHorizonFinder/AHInterpolation.hpp b/Source/ApparentHorizonFinder/AHInterpolation.hpp index 4c238b97c..5ba956993 100644 --- a/Source/ApparentHorizonFinder/AHInterpolation.hpp +++ b/Source/ApparentHorizonFinder/AHInterpolation.hpp @@ -16,7 +16,7 @@ //! Class used for interpolation of the variables needed to calculate expansion //! with the data from a given 'SurfaceGeometry' -template class AHInterpolation +template class AHInterpolation_t { private: SurfaceGeometry m_coord_system; @@ -55,8 +55,8 @@ template class AHInterpolation ); public: - AHInterpolation(const SurfaceGeometry &a_coordSystem, - AMRInterpolator> *a_interpolator); + AHInterpolation_t(const SurfaceGeometry &a_coordSystem, + AMRInterpolator> *a_interpolator); const AMRInterpolator> *get_interpolator() const; const SurfaceGeometry &get_coord_system() const; diff --git a/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp b/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp index 608b83054..f1ab2138b 100644 --- a/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp +++ b/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp @@ -13,7 +13,7 @@ #include "PETScCommunicator.hpp" template -AHInterpolation::AHInterpolation( +AHInterpolation_t::AHInterpolation_t( const SurfaceGeometry &a_coord_system, AMRInterpolator> *a_interpolator) : m_coord_system(a_coord_system), m_interpolator(a_interpolator) @@ -76,21 +76,21 @@ AHInterpolation::AHInterpolation( template const AMRInterpolator> * -AHInterpolation::get_interpolator() const +AHInterpolation_t::get_interpolator() const { return m_interpolator; } template const SurfaceGeometry & -AHInterpolation::get_coord_system() const +AHInterpolation_t::get_coord_system() const { return m_coord_system; } template std::vector -AHInterpolation::get_labels() const +AHInterpolation_t::get_labels() const { return { @@ -103,14 +103,14 @@ AHInterpolation::get_labels() const } template -void AHInterpolation::set_origin( +void AHInterpolation_t::set_origin( const std::array &origin) { m_coord_system.set_origin(origin); } template -bool AHInterpolation::is_in_grid( +bool AHInterpolation_t::is_in_grid( const std::array &a_origin, double a_initial_guess) { bool out_of_grid = false; @@ -162,11 +162,11 @@ bool AHInterpolation::is_in_grid( return out_of_grid; } template -bool AHInterpolation::fit_in_grid(double &x, - double &y +bool AHInterpolation_t::fit_in_grid(double &x, + double &y #if CH_SPACEDIM == 3 - , - double &z + , + double &z #endif ) { @@ -228,8 +228,8 @@ bool AHInterpolation::fit_in_grid(double &x, * } */ template -bool AHInterpolation::keep_interpolating_if_inactive() +bool AHInterpolation_t::keep_interpolating_if_inactive() { CH_TIME("AMRInterpolator::keep_interpolating_if_inactive"); @@ -245,7 +245,7 @@ bool AHInterpolation -void AHInterpolation::break_interpolation_loop() +void AHInterpolation_t::break_interpolation_loop() const { CH_TIME("AMRInterpolator::break_interpolation_loop"); @@ -260,7 +260,7 @@ void AHInterpolation::break_interpolation_loop() #endif } template -int AHInterpolation::interpolate() +int AHInterpolation_t::interpolate() { CH_TIME("AHInterpolation::interpolate"); @@ -303,7 +303,7 @@ int AHInterpolation::interpolate() } template -void AHInterpolation::refresh_interpolator( +void AHInterpolation_t::refresh_interpolator( bool printing_step, const std::map> &extra_vars) { @@ -367,7 +367,7 @@ void AHInterpolation::refresh_interpolator( // 'set_coordinates' calls 'interpolate'. All Chombo_MPI:comm need to run // 'interpolate' for it to work template -bool AHInterpolation::set_coordinates( +bool AHInterpolation_t::set_coordinates( const vector &f, const vector &u, #if CH_SPACEDIM == 3 const vector &v, @@ -431,7 +431,7 @@ bool AHInterpolation::set_coordinates( template const AHGeometryData -AHInterpolation::get_geometry_data(int idx) const +AHInterpolation_t::get_geometry_data(int idx) const { CH_TIME("AHInterpolation::get_geometry_data"); @@ -445,7 +445,7 @@ AHInterpolation::get_geometry_data(int idx) const template const Tensor<1, double> -AHInterpolation::get_cartesian_coords( +AHInterpolation_t::get_cartesian_coords( int idx) const { return @@ -460,7 +460,7 @@ AHInterpolation::get_cartesian_coords( template const Tensor<1, double> -AHInterpolation::get_coords(int idx) const +AHInterpolation_t::get_coords(int idx) const { return { @@ -474,13 +474,13 @@ AHInterpolation::get_coords(int idx) const template const AHData -AHInterpolation::get_data(int idx) const +AHInterpolation_t::get_data(int idx) const { return get_AHData_idx(idx, m_data); } template -void AHInterpolation::interpolate_extra_vars( +void AHInterpolation_t::interpolate_extra_vars( const std::map> &extra_vars) { CH_TIME("AHInterpolation::interpolate_extra_vars"); @@ -517,7 +517,7 @@ void AHInterpolation::interpolate_extra_vars( template const AHData -AHInterpolation::get_extra_data(int idx) const +AHInterpolation_t::get_extra_data(int idx) const { return get_AHData_idx(idx, m_extra); } diff --git a/Source/ApparentHorizonFinder/AHParams.hpp b/Source/ApparentHorizonFinder/AHParams.hpp new file mode 100644 index 000000000..a262ede44 --- /dev/null +++ b/Source/ApparentHorizonFinder/AHParams.hpp @@ -0,0 +1,237 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef _AHPARAMS_HPP_ +#define _AHPARAMS_HPP_ + +#include "ChomboParameters.hpp" +#include "GRParmParse.hpp" + +// prepend with 'AH_' in params file +template struct AHParams_t +{ + int num_ranks; //!< number of ranks for PETSc sub-communicator (default + //!< 0, which is 'all') + + int num_points_u; //!< number of points for 2D coordinate grid +#if CH_SPACEDIM == 3 + int num_points_v; //!< number of points for 2D coordinate grid +#endif + int solve_interval; //!< same as checkpoint_interval, for + //!< ApparentHorizon::solve (default 1) + int print_interval; //!< same as solve_interval, but for prints (default + //!< 1) + bool track_center; //!< whether or not to update the center + //!< (set to false if you know it won't move) + //!< (default true) + bool predict_origin; //!< whether or not to estimate where the next + //!< origin will be at (default = track_center) + + int level_to_run; // if negative, it will count backwards (e.g. -1 is + // 'last level') (default 0) + + double start_time; //!< time after which AH can start (default 0.) + //!< Useful for ScalarField collapse + double give_up_time; //!< stop if at this time nothing was found + //!< (<0 to never, which is default) + //!< Useful for ScalarField collapse + + bool allow_re_attempt; //!< re-attempt with initial guess if + //!< previous convergence failed (default false) + int max_fails_after_lost; //!< number of time steps to try again after + //!< (-1 to never) the AH was lost + //!< (default is 0) + + int verbose; //!< print information during execution (default is 1) + + bool print_geometry_data; //!< print metric and extrinsic + //!< curvature of the AHs (default false) + + bool re_solve_at_restart; //!< whether or not to re-run even if AH + //!< already exists (useful in order to be + //!< able to provide an initial guess and + //!< re-run the AH on that time step) + //!< (default false) + + bool stop_if_max_fails; //! breaks the run if AH doesn't converge + //! 'max_fails_after_lost' times or if + //! 'give_up_time' is reached without + //! convergence (default is 'false') + + std::map> + extra_vars; //! extra vars to write to coordinates file () + int num_extra_vars; // total number of extra vars (!=extra_vars.size() + // as derivative count for multiple vars) + + int extra_contain_diagnostic; // not a parameter (set internally); + // counts how many + + std::string stats_path = "", + stats_prefix = "stats_AH"; //!< name for stats file with + //!< area, spin and AH origin/center + std::string coords_path = "", + coords_prefix = "coords_AH"; //!< name for coords file with AH + //!< coordinates at each time step + + //! mergers will be searched when distance between 'parent' BHs is + //! distance < merger_search_factor * 4. * (AH_initial_guess_1 + + //! AH_initial_guess_2) should be roughly '2M=2(m1+m2)' for initial + //! guess at m/2 (set to non-positive to 'always search') + double merger_search_factor; // see note above (default is 1) + //! initial guess for merger is 'merger_pre_factor * 4. * + //! (AH_initial_guess_1 + AH_initial_guess_2)' + //! set to somethig bigger to avoid finding the inner AH + double merger_pre_factor; // see note above (default to 1.) + + enum verbose_level + { + NONE, + MIN, // minimal + SOME, // a bit technical + MORE // debug + }; + + void read_params(GRParmParse &pp, const ChomboParameters &a_p); +}; + +template +void AHParams_t::read_params( + GRParmParse &pp, const ChomboParameters &a_p) +{ + pp.load("AH_num_ranks", num_ranks, 0); // 0 means "all" + + pp.load("AH_num_points_u", num_points_u); +#if CH_SPACEDIM == 3 + pp.load("AH_num_points_v", num_points_v); +#endif + + // if box division ends up with size less than 3, PETSc will + // complain (this only gives an estimate of the box side) + int size = 1; +#if CH_MPI + MPI_Comm_size(Chombo_MPI::comm, &size); +#endif + size = std::min(num_ranks, size); +#if CH_SPACEDIM == 3 + CH_assert(this->num_points_u > 0 && this->num_points_v > 0); + CH_assert(this->num_points_u / sqrt(size) >= 3); // make sure for size 'u' + CH_assert(this->num_points_v / sqrt(size) >= 3); // make sure for size 'v' +#elif CH_SPACEDIM == 2 + CH_assert(this->num_points_u > 0); + CH_assert(this->num_points_u / size >= 3); // make sure for size 'u' +#endif + + pp.load("AH_solve_interval", solve_interval, 1); + pp.load("AH_print_interval", print_interval, 1); + pp.load("AH_track_center", track_center, true); + pp.load("AH_predict_origin", predict_origin, track_center); + // can't predict if center is not being tracked + CH_assert(!(predict_origin && !track_center)); + + pp.load("AH_level_to_run", level_to_run, 0); + CH_assert(level_to_run <= a_p.max_level && + level_to_run > -(a_p.max_level + 1)); + if (level_to_run < 0) // if negative, count backwards + level_to_run += a_p.max_level + 1; + + pp.load("AH_start_time", start_time, 0.0); + pp.load("AH_give_up_time", give_up_time, -1.0); + + pp.load("AH_allow_re_attempt", allow_re_attempt, false); + pp.load("AH_max_fails_after_lost", max_fails_after_lost, 0); + pp.load("AH_stop_if_max_fails", stop_if_max_fails, false); + + pp.load("AH_verbose", verbose, (int)MIN); + pp.load("AH_print_geometry_data", print_geometry_data, false); + pp.load("AH_re_solve_at_restart", re_solve_at_restart, false); + + // sanity checks + CH_assert(solve_interval > 0 && print_interval > 0); + CH_assert(level_to_run >= 0 && level_to_run <= a_p.max_level); + + // load vars to write to coord files + num_extra_vars = 0; + extra_contain_diagnostic = 0; + + int AH_num_extra_vars; + pp.load("AH_num_extra_vars", AH_num_extra_vars, 0); + if (AH_num_extra_vars > 0) + { + std::vector AH_extra_var_names(AH_num_extra_vars, ""); + pp.load("AH_extra_vars", AH_extra_var_names, AH_num_extra_vars); + for (const std::string &full_name : AH_extra_var_names) + { + std::string var_name = full_name; + + // variable names might start with "d1_" or "d2_" to indicate + // the user wants derivatives + int der_type = 0; + std::string der = var_name.substr(0, 3); + if (der == "d1_") + { + der_type = 1; + var_name = var_name.substr(3); + } + else if (der == "d2_") + { + der_type = 2; + var_name = var_name.substr(3); + } + + // first assume extra_var is a normal evolution var + int var = UserVariables::variable_name_to_enum(var_name); + VariableType var_type = VariableType::evolution; + if (var < 0) + { + // if not an evolution var check if it's a diagnostic var + var = DiagnosticVariables::variable_name_to_enum(var_name); + if (var < 0) + { + // it's neither :( + pout() << "Variable with name " << var_name + << " not found.\n"; + } + else + { + var_type = VariableType::diagnostic; + ++extra_contain_diagnostic; + } + } + if (var >= 0) + { + extra_vars[full_name] = + std::tuple(var, var_type, der_type); + if (der_type == 0) + num_extra_vars += 1; + else if (der_type == 1) + num_extra_vars += CH_SPACEDIM; + else // if (der_type == 2) + num_extra_vars += CH_SPACEDIM * (CH_SPACEDIM + 1) / 2; + } + } + } + + stats_path = a_p.data_path; + + if (pp.contains("AH_coords_subpath")) + { + pp.load("AH_coords_subpath", coords_path); + if (!coords_path.empty() && coords_path.back() != '/') + coords_path += "/"; + if (a_p.output_path != "./" && !a_p.output_path.empty()) + coords_path = a_p.output_path + coords_path; + } + else + coords_path = stats_path; + + pp.load("AH_stats_prefix", stats_prefix, std::string("stats_AH")); + pp.load("AH_coords_prefix", coords_prefix, std::string("coords_AH")); + + pp.load("AH_merger_search_factor", merger_search_factor, 1.); + pp.load("AH_merger_pre_factor", merger_pre_factor, 1.); +} + +#endif /* _AHPARAMS_HPP_ */ diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.hpp index 8c547986f..231085021 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.hpp @@ -6,119 +6,26 @@ #ifndef _APPARENTHORIZON_HPP_ #define _APPARENTHORIZON_HPP_ -#include -// #include - -#include "AHDeriv.hpp" #include "AHInterpolation.hpp" -#include "ChomboParameters.hpp" +#include "AHParams.hpp" #include "IntegrationMethod.hpp" +#include "PETScAHSolver.hpp" // Class to manage ApparentHorizon for 2+1D and 3+1D simulations //! AHFunction defines the optimizing function (see AHFunction.hpp for //! expansion example calculation) template class ApparentHorizon { - using Interpolation = AHInterpolation; - - public: - // prepend with 'AH_' in params file - struct params - { - int num_ranks; //!< number of ranks for PETSc sub-communicator (default - //!< 0, which is 'all') - - int num_points_u; //!< number of points for 2D coordinate grid -#if CH_SPACEDIM == 3 - int num_points_v; //!< number of points for 2D coordinate grid -#endif - int solve_interval; //!< same as checkpoint_interval, for - //!< ApparentHorizon::solve (default 1) - int print_interval; //!< same as solve_interval, but for prints (default - //!< 1) - bool track_center; //!< whether or not to update the center - //!< (set to false if you know it won't move) - //!< (default true) - bool predict_origin; //!< whether or not to estimate where the next - //!< origin will be at (default = track_center) - - int level_to_run; // if negative, it will count backwards (e.g. -1 is - // 'last level') (default 0) - - double start_time; //!< time after which AH can start (default 0.) - //!< Useful for ScalarField collapse - double give_up_time; //!< stop if at this time nothing was found - //!< (<0 to never, which is default) - //!< Useful for ScalarField collapse - - bool allow_re_attempt; //!< re-attempt with initial guess if - //!< previous convergence failed (default false) - int max_fails_after_lost; //!< number of time steps to try again after - //!< (-1 to never) the AH was lost - //!< (default is 0) - - int verbose; //!< print information during execution (default is 1) - - bool print_geometry_data; //!< print metric and extrinsic - //!< curvature of the AHs (default false) - - bool re_solve_at_restart; //!< whether or not to re-run even if AH - //!< already exists (useful in order to be - //!< able to provide an initial guess and - //!< re-run the AH on that time step) - //!< (default false) - - bool stop_if_max_fails; //! breaks the run if AH doesn't converge - //! 'max_fails_after_lost' times or if - //! 'give_up_time' is reached without - //! convergence (default is 'false') - - std::map> - extra_vars; //! extra vars to write to coordinates file () - int num_extra_vars; // total number of extra vars (!=extra_vars.size() - // as derivative count for multiple vars) - - int extra_contain_diagnostic; // not a parameter (set internally); - // counts how many - - std::string stats_path = "", - stats_prefix = - "stats_AH"; //!< name for stats file with - //!< area, spin and AH origin/center - std::string coords_path = "", - coords_prefix = - "coords_AH"; //!< name for coords file with AH - //!< coordinates at each time step - - //! mergers will be searched when distance between 'parent' BHs is - //! distance < merger_search_factor * 4. * (AH_initial_guess_1 + - //! AH_initial_guess_2) should be roughly '2M=2(m1+m2)' for initial - //! guess at m/2 (set to non-positive to 'always search') - double merger_search_factor; // see note above (default is 1) - //! initial guess for merger is 'merger_pre_factor * 4. * - //! (AH_initial_guess_1 + AH_initial_guess_2)' - //! set to somethig bigger to avoid finding the inner AH - double merger_pre_factor; // see note above (default to 1.) - - void read_params(GRParmParse &pp, const ChomboParameters &a_p); - }; - - enum verbose_level - { - NONE, - MIN, // minimal - SOME, // a bit technical - MORE // debug - }; + using AHInterpolation = AHInterpolation_t; + using AHParams = AHParams_t; public: //! AH that finds the zero of expansion ApparentHorizon( - const Interpolation &a_interp, //!< Geometry class to exchange data - double a_initial_guess, //!< Initial guess for radius (or whatever - //!< coordinate you're solving for) - const params &a_params, //!< set of AH parameters + const AHInterpolation &a_interp, //!< Geometry class to exchange data + double a_initial_guess, //!< Initial guess for radius (or whatever + //!< coordinate you're solving for) + const AHParams &a_params, //!< set of AH parameters const std::string &a_stats = "stats", //!< name for output file with area, spin and AH origin const std::string &a_coords = @@ -129,13 +36,12 @@ template class ApparentHorizon //! personalized optimizer that finds zero of function //! 'a_function_to_optimize' (a void* 'a_function_to_optimize_params' can be //! passed for auxiliary parameters passed to 'a_function_to_optimize') - ApparentHorizon(const Interpolation &a_interp, double a_initial_guess, - const params &a_params, + ApparentHorizon(const AHInterpolation &a_interp, double a_initial_guess, + const AHParams &a_params, const typename AHFunction::params &a_func_params, const std::string &a_stats = "stats", const std::string &a_coords = "coords_", bool solve_first_step = true); - ~ApparentHorizon(); void solve(double a_dt, double a_time, double a_restart_time); @@ -149,9 +55,8 @@ template class ApparentHorizon const std::array &get_origin() const; const std::array &get_center() const; - double get_initial_guess() const; - void set_initial_guess(double a_initial_guess); - const Interpolation &get_ah_interp() const; + const AHInterpolation &get_ah_interp() const; + PETScAHSolver &get_petsc_solver(); // set origin to whatever you want (e.g. punctures) before solving if you // want, otherwise we use the last center or the estimate next center @@ -169,15 +74,13 @@ template class ApparentHorizon const; //!< decide when to print (only params::print_interval //!< out of all 'solve's) - const params m_params; //!< set of AH parameters + // variables + public: + const AHParams m_params; //!< set of AH parameters const std::string m_stats, m_coords; //!< public base names for output files (no need for a set as //!< they are const) - // any parameters that want to be saved to be passed to optimization - // function - typename AHFunction::params m_func_params; - private: void write_outputs(double a_dt, double a_time, double a_restart_time); @@ -207,8 +110,6 @@ template class ApparentHorizon void calculate_minmax_F() const; void calculate_average_F() const; - void reset_initial_guess(); - void update_old_centers(std::array); void predict_next_origin(); std::array @@ -230,9 +131,6 @@ template class ApparentHorizon // default to simpson rule and change if invalid void check_integration_methods(); - void initialise_PETSc(); //!< initialise automatically done in constructor - void finalise_PETSc(); //!< finalise automatically done in destructor - // variables private: bool m_printed_once; @@ -244,9 +142,6 @@ template class ApparentHorizon int m_num_failed_convergences; //!< the number of failed consecutive //!< convergences - double m_initial_guess; //!< initial guess for AH (saved so that it can be - //!< re-used when atempting to solve again) - //! used to estimate where the new center will be before solving //! (currently last 2 centers saved to make a parabolic regression) //! (elements ordered in reverse time - older with higher index) @@ -269,127 +164,9 @@ template class ApparentHorizon // before 'solve' bool origin_already_updated; - ///////////////////////////////////////////////////////////////////////// - /////////////////////////// PETSc stuff below /////////////////////////// - ///////////////////////////////////////////////////////////////////////// - -#if CH_SPACEDIM == 3 - typedef PetscScalar **dmda_arr_t; -#elif CH_SPACEDIM == 2 - typedef PetscScalar *dmda_arr_t; -#endif - - //! Geometries of the AH - //! 'm_geom_plus' and 'm_geom_minus' are used to calculate the - //! jacobian of the expansion using a 'delta' numerical differentiation - Interpolation m_interp; - Interpolation m_interp_plus; - Interpolation m_interp_minus; - - //!< used to compute jacobian of expansion (numerical differentiation) - static constexpr double eps = 1e-7; - - const bool m_periodic_u; //!< is 'u' periodic? - PetscInt m_num_global_u; //!< total number of grid points in 'u' coordinate - double m_du; //!< physical 'delta' in 'u' coordinate - -#if CH_SPACEDIM == 3 - const bool m_periodic_v; //!< is 'v' periodic? - PetscInt m_num_global_v; //!< total number of grid points in 'u' coordinate - double m_dv; //!< physical 'delta' in 'v' coordinate -#endif - - //! vectors to store and manipulate 'F', u', and 'v' - //! internally, in interaction with the 'Interpolation's - std::vector m_F; - std::vector m_u; -#if CH_SPACEDIM == 3 - std::vector m_v; -#endif - - //! minimums and maximums of coordinates 'u' and 'v' - //! of the PETSc grid specific to the current rank - PetscInt m_umin; - PetscInt m_umax; - -#if CH_SPACEDIM == 3 - PetscInt m_vmin; - PetscInt m_vmax; -#endif - - //! number of points in 'u' and 'v' direction - //! (m_nu = m_umax - m_umin) - PetscInt m_nu; -#if CH_SPACEDIM == 3 - PetscInt m_nv; -#endif - - // PETSc main object - DM m_dmda; - //! Scalable Nonlinear Equations Solvers - SNES m_snes; - - Vec m_snes_soln; - Vec m_snes_rhs; - Mat m_snes_jac; - - //! interpolate (u,v) 2D grid points at restart if number of points in - //! either direction changed - //! This only interpolates the points that the PETSc rank that called it has - //! returns whether or not points were interpolated - bool interpolate_ah(dmda_arr_t f, - const std::vector> &old_coords); - - //! set the default stencils of AHDeriv at position {u,v} - void set_stencils(AHDeriv &out, int u -#if CH_SPACEDIM == 3 - , - int v -#endif - ); - - //! function to calculate 1st and 2nd derivatives of 'in' - //! (tipically corresponds to our 'f' function) - //! in the 'u' and 'v' directions - AHDeriv diff(const dmda_arr_t in, int u -#if CH_SPACEDIM == 3 - , - int v -#endif - ); - - //! private functions used to compute the RHS (the expansion) and it's - //! jacobian - void form_function(Vec F, Vec Rhs); - void form_jacobian(Vec F, Mat J); - - //! helper for 'form_jacobian' - double point_jacobian(int u, int u_stencil, -#if CH_SPACEDIM == 3 - int v, int v_stencil, -#endif - dmda_arr_t in, int idx, - const Interpolation &interp_plus, - const Interpolation &interp_minus); - - //! functions used by PETSc based on 'form_function' and 'form_jacobian' - static PetscErrorCode Petsc_form_function(SNES snes, Vec F, Vec Rhs, - void *ptr); - - static PetscErrorCode -#if PETSC_VERSION_GE(3, 5, 0) - Petsc_form_jacobian(SNES snes, Vec F, Mat Amat, Mat Pmat, void *ptr); -#else - Petsc_form_jacobian(SNES snes, Vec F, Mat *Amat, Mat *Pmat, - MatStructure *flag, void *ptr); -#endif - - // monitor function required for SNES - static PetscErrorCode Petsc_SNES_monitor(SNES snes, PetscInt its, - PetscReal norm, void *ptr); + PETScAHSolver solver; }; #include "ApparentHorizon.impl.hpp" -#include "ApparentHorizon_petsc.impl.hpp" #endif /* _APPARENTHORIZON_HPP_ */ diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp index e8f2d704f..1a8196e67 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp @@ -21,13 +21,10 @@ #include "SimpleArrayBox.hpp" #include "SimpleInterpSource.hpp" -#include // for 'setw' in stringstream -#include // infinity for min and max calculation - template ApparentHorizon::ApparentHorizon( - const AHInterpolation &a_interp, - double a_initial_guess, const params &a_params, const std::string &a_stats, + const AHInterpolation &a_interp, double a_initial_guess, + const AHParams &a_params, const std::string &a_stats, const std::string &a_coords, bool solve_first_step) : ApparentHorizon(a_interp, a_initial_guess, a_params, AHFunction::params(), a_stats, a_coords, solve_first_step) @@ -36,16 +33,15 @@ ApparentHorizon::ApparentHorizon( template ApparentHorizon::ApparentHorizon( - const AHInterpolation &a_interp, - double a_initial_guess, const params &a_params, - const typename AHFunction::params &a_func_params, + const AHInterpolation &a_interp, double a_initial_guess, + const AHParams &a_params, const typename AHFunction::params &a_func_params, const std::string &a_stats, const std::string &a_coords, bool solve_first_step) : m_params(a_params), m_stats(a_stats), m_coords(a_coords), - m_func_params(a_func_params), + solver(a_interp, a_initial_guess, a_params, a_func_params), m_printed_once(false), @@ -55,8 +51,6 @@ ApparentHorizon::ApparentHorizon( m_num_failed_convergences(0), - m_initial_guess(a_initial_guess), - m_old_centers(3, a_interp.get_coord_system().get_origin()), m_max_F(0.), m_min_F(0.), m_ave_F(0.), m_std_F(0.), @@ -74,166 +68,12 @@ ApparentHorizon::ApparentHorizon( m_area(NAN), m_spin(NAN), m_mass(NAN), m_irreducible_mass(NAN), m_spin_z_alt(NAN), m_dimensionless_spin_vector({NAN}), - origin_already_updated(false), - - m_interp(a_interp), m_interp_plus(a_interp), m_interp_minus(a_interp), - - m_periodic_u(a_interp.get_coord_system().is_u_periodic()), - m_num_global_u(a_params.num_points_u) - -#if CH_SPACEDIM == 3 - , - m_periodic_v(a_interp.get_coord_system().is_v_periodic()), - m_num_global_v(a_params.num_points_v) -#endif + origin_already_updated(false) { - initialise_PETSc(); set_origin(a_interp.get_coord_system().get_origin()); check_integration_methods(); restart(solve_first_step); } -template -ApparentHorizon::~ApparentHorizon() -{ - finalise_PETSc(); -} - -template -void ApparentHorizon::params::read_params( - GRParmParse &pp, const ChomboParameters &a_p) -{ - pp.load("AH_num_ranks", num_ranks, 0); // 0 means "all" - - pp.load("AH_num_points_u", num_points_u); -#if CH_SPACEDIM == 3 - pp.load("AH_num_points_v", num_points_v); -#endif - - // if box division ends up with size less than 3, PETSc will - // complain (this only gives an estimate of the box side) - int size = 1; -#if CH_MPI - MPI_Comm_size(Chombo_MPI::comm, &size); -#endif - size = std::min(num_ranks, size); -#if CH_SPACEDIM == 3 - CH_assert(this->num_points_u > 0 && this->num_points_v > 0); - CH_assert(this->num_points_u / sqrt(size) >= 3); // make sure for size 'u' - CH_assert(this->num_points_v / sqrt(size) >= 3); // make sure for size 'v' -#elif CH_SPACEDIM == 2 - CH_assert(this->num_points_u > 0); - CH_assert(this->num_points_u / size >= 3); // make sure for size 'u' -#endif - - pp.load("AH_solve_interval", solve_interval, 1); - pp.load("AH_print_interval", print_interval, 1); - pp.load("AH_track_center", track_center, true); - pp.load("AH_predict_origin", predict_origin, track_center); - // can't predict if center is not being tracked - CH_assert(!(predict_origin && !track_center)); - - pp.load("AH_level_to_run", level_to_run, 0); - CH_assert(level_to_run <= a_p.max_level && - level_to_run > -(a_p.max_level + 1)); - if (level_to_run < 0) // if negative, count backwards - level_to_run += a_p.max_level + 1; - - pp.load("AH_start_time", start_time, 0.0); - pp.load("AH_give_up_time", give_up_time, -1.0); - - pp.load("AH_allow_re_attempt", allow_re_attempt, false); - pp.load("AH_max_fails_after_lost", max_fails_after_lost, 0); - pp.load("AH_stop_if_max_fails", stop_if_max_fails, false); - - pp.load("AH_verbose", verbose, (int)MIN); - pp.load("AH_print_geometry_data", print_geometry_data, false); - pp.load("AH_re_solve_at_restart", re_solve_at_restart, false); - - // sanity checks - CH_assert(solve_interval > 0 && print_interval > 0); - CH_assert(level_to_run >= 0 && level_to_run <= a_p.max_level); - - // load vars to write to coord files - num_extra_vars = 0; - extra_contain_diagnostic = 0; - - int AH_num_extra_vars; - pp.load("AH_num_extra_vars", AH_num_extra_vars, 0); - if (AH_num_extra_vars > 0) - { - std::vector AH_extra_var_names(AH_num_extra_vars, ""); - pp.load("AH_extra_vars", AH_extra_var_names, AH_num_extra_vars); - for (const std::string &full_name : AH_extra_var_names) - { - std::string var_name = full_name; - - // variable names might start with "d1_" or "d2_" to indicate - // the user wants derivatives - int der_type = 0; - std::string der = var_name.substr(0, 3); - if (der == "d1_") - { - der_type = 1; - var_name = var_name.substr(3); - } - else if (der == "d2_") - { - der_type = 2; - var_name = var_name.substr(3); - } - - // first assume extra_var is a normal evolution var - int var = UserVariables::variable_name_to_enum(var_name); - VariableType var_type = VariableType::evolution; - if (var < 0) - { - // if not an evolution var check if it's a diagnostic var - var = DiagnosticVariables::variable_name_to_enum(var_name); - if (var < 0) - { - // it's neither :( - pout() << "Variable with name " << var_name - << " not found.\n"; - } - else - { - var_type = VariableType::diagnostic; - ++extra_contain_diagnostic; - } - } - if (var >= 0) - { - extra_vars[full_name] = - std::tuple(var, var_type, der_type); - if (der_type == 0) - num_extra_vars += 1; - else if (der_type == 1) - num_extra_vars += CH_SPACEDIM; - else // if (der_type == 2) - num_extra_vars += CH_SPACEDIM * (CH_SPACEDIM + 1) / 2; - } - } - } - - stats_path = a_p.data_path; - - if (pp.contains("AH_coords_subpath")) - { - pp.load("AH_coords_subpath", coords_path); - if (!coords_path.empty() && coords_path.back() != '/') - coords_path += "/"; - if (a_p.output_path != "./" && !a_p.output_path.empty()) - coords_path = a_p.output_path + coords_path; - } - else - coords_path = stats_path; - - pp.load("AH_stats_prefix", stats_prefix, std::string("stats_AH")); - pp.load("AH_coords_prefix", coords_prefix, std::string("coords_AH")); - - pp.load("AH_merger_search_factor", merger_search_factor, 1.); - pp.load("AH_merger_pre_factor", merger_pre_factor, 1.); -} template bool ApparentHorizon::good_to_go( @@ -275,33 +115,30 @@ template const std::array & ApparentHorizon::get_origin() const { - return m_interp.get_coord_system().get_origin(); + return solver.get_origin(); } template -const AHInterpolation & +const AHInterpolation_t & ApparentHorizon::get_ah_interp() const { - return m_interp; + return solver.m_interp; +} + +template +PETScAHSolver & +ApparentHorizon::get_petsc_solver() +{ + return solver; } template void ApparentHorizon::set_origin( const std::array &a_origin) { - m_interp.set_origin(a_origin); - m_interp_plus.set_origin(a_origin); - m_interp_minus.set_origin(a_origin); - origin_already_updated = true; + solver.set_origin(a_origin); - if (m_params.verbose > SOME) - { - pout() << "Setting origin to (" << a_origin[0] << "," << a_origin[1] -#if CH_SPACEDIM == 3 - << "," << a_origin[2] -#endif - << ")" << std::endl; - } + origin_already_updated = true; } template @@ -356,65 +193,6 @@ double ApparentHorizon::get_std_F() const return m_std_F; } -template -double ApparentHorizon::get_initial_guess() const -{ - return m_initial_guess; -} - -template -void ApparentHorizon::set_initial_guess( - double a_initial_guess) -{ - m_initial_guess = a_initial_guess; -} -template -void ApparentHorizon::reset_initial_guess() -{ - CH_TIME("ApparentHorizon::reset_initial_guess"); - - if (!PETScCommunicator::is_rank_active()) - return; - - auto origin = get_origin(); - - // verify origin +- initial guess is inside the grid - bool out_of_grid = m_interp.is_in_grid(origin, m_initial_guess); - CH_assert(!out_of_grid); - - if (m_params.verbose > NONE) - { - pout() << "Setting Initial Guess to f=" << m_initial_guess - << " centered at (" << origin[0] << "," << origin[1] -#if CH_SPACEDIM == 3 - << "," << origin[2] -#endif - << ")" << std::endl; - } - - // read PETSc array to 'f' - dmda_arr_t f; - DMDAVecGetArray(m_dmda, m_snes_soln, &f); - -#if CH_SPACEDIM == 3 - for (int v = m_vmin; v < m_vmax; ++v) -#endif - { - for (int u = m_umin; u < m_umax; ++u) - { -#if CH_SPACEDIM == 3 - double &f_point = f[v][u]; -#else - double &f_point = f[u]; -#endif - - f_point = m_initial_guess; - } - } - - // write PETSc array back - DMDAVecRestoreArray(m_dmda, m_snes_soln, &f); -} template void ApparentHorizon::predict_next_origin() { @@ -423,16 +201,16 @@ void ApparentHorizon::predict_next_origin() { FOR1(a) { - if (!m_interp.get_interpolator()->get_boundary_reflective(Side::Lo, - a) && - !m_interp.get_interpolator()->get_boundary_reflective(Side::Hi, - a)) + if (!solver.m_interp.get_interpolator()->get_boundary_reflective( + Side::Lo, a) && + !solver.m_interp.get_interpolator()->get_boundary_reflective( + Side::Hi, a)) { new_center[a] += (m_old_centers[0][a] + m_old_centers[2][a] - 2. * m_old_centers[1][a]); } } - if (m_params.verbose > SOME) + if (m_params.verbose > AHParams::SOME) { pout() << "OLD[-2]: (" << m_old_centers[2][0] << "," << m_old_centers[2][1] @@ -446,16 +224,16 @@ void ApparentHorizon::predict_next_origin() { FOR1(a) { - if (!m_interp.get_interpolator()->get_boundary_reflective(Side::Lo, - a) && - !m_interp.get_interpolator()->get_boundary_reflective(Side::Hi, - a)) + if (!solver.m_interp.get_interpolator()->get_boundary_reflective( + Side::Lo, a) && + !solver.m_interp.get_interpolator()->get_boundary_reflective( + Side::Hi, a)) { new_center[a] += (m_old_centers[0][a] - m_old_centers[1][a]); } } - if (m_params.verbose > SOME) + if (m_params.verbose > AHParams::SOME) { pout() << "OLD[-1]: (" << m_old_centers[1][0] << "," << m_old_centers[1][1] @@ -470,7 +248,7 @@ void ApparentHorizon::predict_next_origin() #endif << ")" << std::endl; } - if (m_params.verbose > MIN) + if (m_params.verbose > AHParams::MIN) { pout() << "Estimated center: (" << new_center[0] << "," << new_center[1] @@ -490,9 +268,10 @@ void ApparentHorizon::update_old_centers( { FOR1(a) { - if (!m_interp.get_interpolator()->get_boundary_reflective(Side::Lo, - a) && - !m_interp.get_interpolator()->get_boundary_reflective(Side::Hi, a)) + if (!solver.m_interp.get_interpolator()->get_boundary_reflective( + Side::Lo, a) && + !solver.m_interp.get_interpolator()->get_boundary_reflective( + Side::Hi, a)) { m_old_centers[2][a] = m_old_centers[1][a]; m_old_centers[1][a] = m_old_centers[0][a]; @@ -510,7 +289,7 @@ void ApparentHorizon::solve(double a_dt, if (!good_to_go(a_dt, a_time)) return; - m_interp.refresh_interpolator( + solver.m_interp.refresh_interpolator( do_print(a_dt, a_time), m_params.extra_vars); // (ALL CHOMBO ranks do it!!) @@ -524,34 +303,21 @@ void ApparentHorizon::solve(double a_dt, } // PETSc processes go inside 'if', others "wait" until 'if' gets to - // 'm_interp.break_interpolation_loop()' - if (m_interp.keep_interpolating_if_inactive()) + // 'solver.m_interp.break_interpolation_loop()' + if (solver.m_interp.keep_interpolating_if_inactive()) { CH_TIME("ApparentHorizon::solve::solving"); if (!get_converged()) - reset_initial_guess(); // reset initial guess if diverged (or in - // first attempt) - - // actual solve happens here! - SNESSolve(m_snes, NULL, m_snes_soln); + solver.reset_initial_guess(); // reset initial guess if diverged (or + // in first attempt) - PetscInt its; - SNESGetIterationNumber(m_snes, &its); - if (m_params.verbose > MIN) - { - pout() << "SNES Iteration number " << its << endl; - } - SNESGetLinearSolveIterations(m_snes, &its); - if (m_params.verbose > MIN) - { - pout() << "KSP Iteration number " << its << endl; - } - m_interp.break_interpolation_loop(); + solver.solve(); + solver.m_interp.break_interpolation_loop(); } { - if (m_params.verbose > SOME) + if (m_params.verbose > AHParams::SOME) { pout() << "In [ApparentHorizon::solve::post-solving]" << endl; } @@ -570,7 +336,7 @@ void ApparentHorizon::solve(double a_dt, if (save_converged && !get_converged() && m_params.allow_re_attempt) { --m_num_failed_convergences; // reduce failed and try again - if (m_params.verbose > NONE) + if (m_params.verbose > AHParams::NONE) { pout() << "Re-attempting to solve using initial guess." << std::endl; @@ -605,13 +371,13 @@ void ApparentHorizon::solve(double a_dt, m_spin_z_alt = calculate_spin_dimensionless(m_area); #endif - if (m_params.verbose > NONE) + if (m_params.verbose > AHParams::NONE) { pout() << "mass = " << m_mass << endl; #if CH_SPACEDIM == 3 pout() << "spin = " << m_spin << endl; #endif - if (m_params.verbose > MIN) + if (m_params.verbose > AHParams::MIN) { pout() << "irreducible mass = " << m_irreducible_mass << endl; #if CH_SPACEDIM == 3 @@ -647,7 +413,7 @@ void ApparentHorizon::solve(double a_dt, MayDay::Error("Reached max fails. Stopping. Parameter " "'stop_if_max_fails' is set to true."); - if (m_params.verbose > SOME) + if (m_params.verbose > AHParams::SOME) { pout() << "ApparentHorizon::solve finished successfully!" << endl; } @@ -670,7 +436,7 @@ void ApparentHorizon::write_outputs( if (do_print(a_dt, a_time)) { CH_TIME("ApparentHorizon::solve::printing"); - if (m_params.verbose > MIN) + if (m_params.verbose > AHParams::MIN) { pout() << "Printing statistics and coordinates." << std::endl; } @@ -758,7 +524,7 @@ void ApparentHorizon::write_outputs( file.write_time_data_line(values); // write coordinates - m_interp.interpolate_extra_vars(m_params.extra_vars); + solver.m_interp.interpolate_extra_vars(m_params.extra_vars); write_coords_file(a_dt, a_time, a_restart_time, m_params.coords_path + m_coords, m_params.print_geometry_data); @@ -775,8 +541,7 @@ void ApparentHorizon::check_convergence() int result; if (PETScCommunicator::is_rank_active()) { - SNESConvergedReason reason; - SNESGetConvergedReason(m_snes, &reason); + SNESConvergedReason reason = solver.getConvergedReason(); // result will be 0 if any of the PETSc ranks says 'reason <=0' (PETSc // convergence error) @@ -810,13 +575,13 @@ void ApparentHorizon::check_convergence() ++m_num_failed_convergences; } - if (m_params.verbose > NONE) + if (m_params.verbose > AHParams::NONE) { pout() << (m_converged ? "Solver converged. Horizon FOUND." : "Solver diverged. Horizon NOT found.") << std::endl; - if (m_params.verbose > MIN) + if (m_params.verbose > AHParams::MIN) { pout() << "SNESConvergedReason = " << result << std::endl; } @@ -837,8 +602,8 @@ void ApparentHorizon::restart( // GUIDING PRINCIPLE : restart should not depend on m_params, as these might // have changed! We should be able to figure everything out without them - const GRAMR &gramr = - (static_cast(m_interp.get_interpolator()->getAMR())); + const GRAMR &gramr = (static_cast( + solver.m_interp.get_interpolator()->getAMR())); int current_step = AMR::s_step; int restart_step = gramr.get_restart_step(); @@ -870,7 +635,7 @@ void ApparentHorizon::restart( double old_print_dt = 0.; if (stats.size() == 0) { // case when it never ran the AH or the file doesn't exist - if (m_params.verbose > NONE && current_step != 0) + if (m_params.verbose > AHParams::NONE && current_step != 0) { pout() << "Empty stats file '" << file << "'. Assuming AHFinder was not run yet for this AH." @@ -901,7 +666,7 @@ void ApparentHorizon::restart( else // if we can't know, just default to the current one old_print_dt = current_solve_dt * m_params.print_interval; - if (m_params.verbose > SOME) + if (m_params.verbose > AHParams::SOME) { if (idx >= 0) pout() << "stats[0][idx] = " << stats[0][idx] << std::endl; @@ -915,7 +680,7 @@ void ApparentHorizon::restart( if (idx < 0) { // case when job was restarted at a point before the AH was first ever // found - if (m_params.verbose > NONE) + if (m_params.verbose > AHParams::NONE) { pout() << "First time step is after restart in '" << file @@ -932,7 +697,7 @@ void ApparentHorizon::restart( -(old_print_dt == 0. ? eps : old_print_dt - eps)) { // case when the PETSc stopped converging and stopped printing // so there is a mismatch with the last time in the file - if (m_params.verbose > NONE) + if (m_params.verbose > AHParams::NONE) { pout() << "Last time step not found in '" << file << "'. Assuming AH stopped converging." << std::endl; @@ -948,7 +713,7 @@ void ApparentHorizon::restart( // having converged // OR when job was restarted at a point before the AH was first ever // found - if (m_params.verbose > NONE) + if (m_params.verbose > AHParams::NONE) { pout() << "Last time step is NAN in '" << file << "'. Assuming AH wasn't found and PETSc didn't " @@ -963,7 +728,7 @@ void ApparentHorizon::restart( } else { // case when the AH was found and there is a coordinate file - if (m_params.verbose > NONE) + if (m_params.verbose > AHParams::NONE) { pout() << "Last time step found in '" << file << "'. Reading coordinates file." << std::endl; @@ -1012,7 +777,7 @@ void ApparentHorizon::restart( int offset = CH_SPACEDIM + was_center_tracked * CH_SPACEDIM; FOR1(a) { origin[a] = stats[cols - offset + a][idx]; } - if (m_params.verbose > NONE) + if (m_params.verbose > AHParams::NONE) { pout() << "Setting origin from stats file '" << m_params.stats_path + m_stats << "' at (" << origin[0] @@ -1035,7 +800,8 @@ void ApparentHorizon::restart( // SET OLD CENTERS directly if old 'dt' matches current 'dt' if (std::abs(old_print_dt - current_solve_dt) < eps) { - if (m_params.verbose > NONE && m_params.predict_origin) + if (m_params.verbose > AHParams::NONE && + m_params.predict_origin) { pout() << "Reading old centers to predict next origin." << std::endl; @@ -1078,7 +844,8 @@ void ApparentHorizon::restart( old_centers_time_index.push_back(index); } - if (m_params.verbose > NONE && m_params.predict_origin) + if (m_params.verbose > AHParams::NONE && + m_params.predict_origin) { if (old_centers_time_index.size() == 0) pout() << "AH time step changed and no old centers " @@ -1120,7 +887,7 @@ void ApparentHorizon::restart( m_converged++; update_old_centers(old_center); - if (m_params.verbose > SOME) + if (m_params.verbose > AHParams::SOME) { pout() << "OLD[-" << i + 1 << "] = (" << old_center[0] << "," << old_center[1] @@ -1154,7 +921,7 @@ void ApparentHorizon::restart( auto coords = SmallDataIO::read(coords_filename); - if (m_params.verbose > NONE) + if (m_params.verbose > AHParams::NONE) { pout() << "Setting Initial Guess to previous file '" << coords_filename << "' found." << std::endl; @@ -1163,15 +930,8 @@ void ApparentHorizon::restart( // now write local points based on file (or interpolated file) if (PETScCommunicator::is_rank_active()) { - // read PETSc array to 'f' - dmda_arr_t f; - DMDAVecGetArray(m_dmda, m_snes_soln, &f); - // doesn't change anything if number of points remained the same - force_restart |= interpolate_ah(f, coords); - - // write PETSc array back - DMDAVecRestoreArray(m_dmda, m_snes_soln, &f); + force_restart |= solver.interpolate_ah(coords); } #ifdef CH_MPI @@ -1208,7 +968,7 @@ void ApparentHorizon::write_coords_file( return; } - if (m_params.verbose > NONE && write_geometry_data) + if (m_params.verbose > AHParams::NONE && write_geometry_data) pout() << "Writing geometry data." << std::endl; CH_assert(a_dt != 0); // Check if time was set!! @@ -1265,7 +1025,7 @@ void ApparentHorizon::write_coords_file( // geometry headers if (write_geometry_data) AHFunction::write_headers(&components[el]); - file.write_header_line(components, m_interp.get_labels()); + file.write_header_line(components, solver.m_interp.get_labels()); ////////////////// // method: @@ -1309,9 +1069,10 @@ void ApparentHorizon::write_coords_file( #endif #if CH_SPACEDIM == 3 - int local_total = (m_vmax - m_vmin) * (m_umax - m_umin); + int local_total = + (solver.m_vmax - solver.m_vmin) * (solver.m_umax - solver.m_umin); #elif CH_SPACEDIM == 2 - int local_total = (m_umax - m_umin); + int local_total = (solver.m_umax - solver.m_umin); #endif #ifdef CH_MPI @@ -1321,21 +1082,21 @@ void ApparentHorizon::write_coords_file( int idx = 0; #if CH_SPACEDIM == 3 - for (int v = m_vmin; v < m_vmax; ++v) + for (int v = solver.m_vmin; v < solver.m_vmax; ++v) #endif { - for (int u = m_umin; u < m_umax; ++u) + for (int u = solver.m_umin; u < solver.m_umax; ++u) { #if CH_SPACEDIM == 3 - output[idx * num_components_total] = m_u[idx]; - output[idx * num_components_total + 1] = m_v[idx]; - output[idx * num_components_total + 2] = m_F[idx]; + output[idx * num_components_total] = solver.m_u[idx]; + output[idx * num_components_total + 1] = solver.m_v[idx]; + output[idx * num_components_total + 2] = solver.m_F[idx]; #elif CH_SPACEDIM == 2 - output[idx * num_components_total] = m_u[idx]; - output[idx * num_components_total + 1] = m_F[idx]; + output[idx * num_components_total] = solver.m_u[idx]; + output[idx * num_components_total + 1] = solver.m_F[idx]; #endif - auto extra = m_interp.get_extra_data(idx); + auto extra = solver.m_interp.get_extra_data(idx); int el = 0; @@ -1369,9 +1130,10 @@ void ApparentHorizon::write_coords_file( // geometry vars if (write_geometry_data) { - const auto data = m_interp.get_data(idx); - const auto coords = m_interp.get_coords(idx); - const auto coords_cart = m_interp.get_cartesian_coords(idx); + const auto data = solver.m_interp.get_data(idx); + const auto coords = solver.m_interp.get_coords(idx); + const auto coords_cart = + solver.m_interp.get_cartesian_coords(idx); AHFunction func(data, coords, coords_cart); func.write_vars( &output[idx * num_components_total + CH_SPACEDIM + el]); @@ -1384,7 +1146,7 @@ void ApparentHorizon::write_coords_file( // has) sends are tagged by global index, so that receives // can be unique and writes indexed correctly #if CH_SPACEDIM == 3 - int idx_global = v * m_num_global_u + u; + int idx_global = v * solver.m_num_global_u + u; #elif CH_SPACEDIM == 2 int idx_global = u; #endif @@ -1410,9 +1172,9 @@ void ApparentHorizon::write_coords_file( // step 2 #ifdef CH_MPI #if CH_SPACEDIM == 3 - int total = m_num_global_u * m_num_global_v; + int total = solver.m_num_global_u * solver.m_num_global_v; #elif CH_SPACEDIM == 2 - int total = m_num_global_u; + int total = solver.m_num_global_u; #endif double temp[total * num_components_total]; if (rank_petsc == 0) @@ -1430,10 +1192,10 @@ void ApparentHorizon::write_coords_file( // step 3 idx = 0; #if CH_SPACEDIM == 3 - for (int v = m_vmin; v < m_vmax; ++v) + for (int v = solver.m_vmin; v < solver.m_vmax; ++v) #endif { - for (int u = m_umin; u < m_umax; ++u) + for (int u = solver.m_umin; u < solver.m_umax; ++u) { MPI_Wait(&(requests[idx]), MPI_STATUS_IGNORE); idx++; @@ -1467,11 +1229,12 @@ void ApparentHorizon::check_integration_methods() // check if integration methods are valid given periodicity and number of // points bool valid_u = m_integration_methods[0].is_valid( - m_params.num_points_u, m_interp.get_coord_system().is_u_periodic()); + m_params.num_points_u, + solver.m_interp.get_coord_system().is_u_periodic()); const IntegrationMethod &method_default_u = - m_interp.get_coord_system().get_recommended_integration_method_u( - m_num_global_u); + solver.m_interp.get_coord_system().get_recommended_integration_method_u( + solver.m_num_global_u); if (!valid_u) { std::string warn = @@ -1479,16 +1242,17 @@ void ApparentHorizon::check_integration_methods() "not valid with this num_points_u.\n" "Reverting to trapezium rule."; MayDay::Warning(warn.c_str()); - pout() << warn << std::endl; + pout() << warn << std ::endl; m_integration_methods[0] = method_default_u; } #if CH_SPACEDIM == 3 bool valid_v = m_integration_methods[1].is_valid( - m_params.num_points_v, m_interp.get_coord_system().is_v_periodic()); + m_params.num_points_v, + solver.m_interp.get_coord_system().is_v_periodic()); const IntegrationMethod &method_default_v = - m_interp.get_coord_system().get_recommended_integration_method_v( - m_num_global_v); + solver.m_interp.get_coord_system().get_recommended_integration_method_v( + solver.m_num_global_v); if (!valid_v) { std::string warn = @@ -1523,36 +1287,33 @@ ApparentHorizon::calculate_spin_dimensionless( // non-PETSc processes // PETSc processes go inside 'if', others "wait" until 'if' gets to - // 'm_interp.break_interpolation_loop()' - if (m_interp.keep_interpolating_if_inactive()) + // 'solver.m_interp.break_interpolation_loop()' + if (solver.m_interp.keep_interpolating_if_inactive()) { int idx = 0; Vec localF; - - DMGetLocalVector(m_dmda, &localF); - DMGlobalToLocalBegin(m_dmda, m_snes_soln, INSERT_VALUES, localF); - DMGlobalToLocalEnd(m_dmda, m_snes_soln, INSERT_VALUES, localF); - dmda_arr_t in; - DMDAVecGetArray(m_dmda, localF, &in); + solver.get_dmda_arr_t(localF, in); - // m_F is already set from solve + // solver.m_F is already set from solve - int u_equator = std::round(M_PI / 2.0 / m_du); + int u_equator = std::round(M_PI / 2.0 / solver.m_du); - for (int v = m_vmin; v < m_vmax; ++v) + for (int v = solver.m_vmin; v < solver.m_vmax; ++v) { - for (int u = m_umin; u < m_umax; ++u) + for (int u = solver.m_umin; u < solver.m_umax; ++u) { if (u_equator == u) { - AHDeriv deriv = diff(in, u, v); - const auto geometry_data = m_interp.get_geometry_data(idx); - const auto data = m_interp.get_data(idx); - const auto coords = m_interp.get_coords(idx); - const auto coords_cart = m_interp.get_cartesian_coords(idx); + AHDeriv deriv = solver.diff(in, u, v); + const auto geometry_data = + solver.m_interp.get_geometry_data(idx); + const auto data = solver.m_interp.get_data(idx); + const auto coords = solver.m_interp.get_coords(idx); + const auto coords_cart = + solver.m_interp.get_cartesian_coords(idx); AHFunction func(data, coords, coords_cart); auto &g = func.get_metric(); @@ -1570,19 +1331,17 @@ ApparentHorizon::calculate_spin_dimensionless( double weight = m_integration_methods[1].weight( v, m_params.num_points_v, - m_interp.get_coord_system().is_v_periodic()); + solver.m_interp.get_coord_system().is_v_periodic()); - integral += sqrt(norm2) * weight * m_dv; + integral += sqrt(norm2) * weight * solver.m_dv; } idx++; } } - DMDAVecRestoreArray(m_dmda, localF, &in); - DMRestoreLocalVector(m_dmda, &localF); - - m_interp.break_interpolation_loop(); + solver.restore_dmda_arr_t(localF, in); + solver.m_interp.break_interpolation_loop(); } // reduction across all Chombo processes (note that 'integral' is 0 for @@ -1627,35 +1386,32 @@ ApparentHorizon::calculate_angular_momentum_J() // in scope for non-PETSc processes // PETSc processes go inside 'if', others "wait" until 'if' gets to - // 'm_interp.break_interpolation_loop()' - if (m_interp.keep_interpolating_if_inactive()) + // 'solver.m_interp.break_interpolation_loop()' + if (solver.m_interp.keep_interpolating_if_inactive()) { int idx = 0; Vec localF; - - DMGetLocalVector(m_dmda, &localF); - DMGlobalToLocalBegin(m_dmda, m_snes_soln, INSERT_VALUES, localF); - DMGlobalToLocalEnd(m_dmda, m_snes_soln, INSERT_VALUES, localF); - dmda_arr_t in; - DMDAVecGetArray(m_dmda, localF, &in); + solver.get_dmda_arr_t(localF, in); - // m_F is already set from solve + // solver.m_F is already set from solve const std::array ¢er = get_center(); - for (int v = m_vmin; v < m_vmax; ++v) + for (int v = solver.m_vmin; v < solver.m_vmax; ++v) { Tensor<1, double> inner_integral = {0.}; - for (int u = m_umin; u < m_umax; ++u) + for (int u = solver.m_umin; u < solver.m_umax; ++u) { - AHDeriv deriv = diff(in, u, v); - const auto geometric_data = m_interp.get_geometry_data(idx); - const auto data = m_interp.get_data(idx); - const auto coords = m_interp.get_coords(idx); - const auto coords_cart = m_interp.get_cartesian_coords(idx); + AHDeriv deriv = solver.diff(in, u, v); + const auto geometric_data = + solver.m_interp.get_geometry_data(idx); + const auto data = solver.m_interp.get_data(idx); + const auto coords = solver.m_interp.get_coords(idx); + const auto coords_cart = + solver.m_interp.get_cartesian_coords(idx); AHFunction func(data, coords, coords_cart); Tensor<2, double> g = func.get_metric(); Tensor<2, double> K = func.get_extrinsic_curvature(); @@ -1722,12 +1478,12 @@ ApparentHorizon::calculate_angular_momentum_J() double weight = m_integration_methods[0].weight( u, m_params.num_points_u, - m_interp.get_coord_system().is_u_periodic()); + solver.m_interp.get_coord_system().is_u_periodic()); FOR1(a) { double element = spin_integrand[a] / (8. * M_PI) * - sqrt(det) * weight * m_du; + sqrt(det) * weight * solver.m_du; // hack for the poles (theta=0,\pi) // where nans appear in 'spin_integrand', but @@ -1741,14 +1497,15 @@ ApparentHorizon::calculate_angular_momentum_J() } double weight = m_integration_methods[1].weight( v, m_params.num_points_v, - m_interp.get_coord_system().is_v_periodic()); - FOR1(a) { integrals[a] += weight * m_dv * inner_integral[a]; } + solver.m_interp.get_coord_system().is_v_periodic()); + FOR1(a) + { + integrals[a] += weight * solver.m_dv * inner_integral[a]; + } } - DMDAVecRestoreArray(m_dmda, localF, &in); - DMRestoreLocalVector(m_dmda, &localF); - - m_interp.break_interpolation_loop(); + solver.restore_dmda_arr_t(localF, in); + solver.m_interp.break_interpolation_loop(); } // reduction across all Chombo processes (note that 'integral' is 0 for @@ -1779,34 +1536,31 @@ double ApparentHorizon::calculate_area() 0.; // temporary, but defined outside to exist for non-PETSc processes // PETSc processes go inside 'if', others "wait" until 'if' gets to - // 'm_interp.break_interpolation_loop()' - if (m_interp.keep_interpolating_if_inactive()) + // 'solver.m_interp.break_interpolation_loop()' + if (solver.m_interp.keep_interpolating_if_inactive()) { int idx = 0; Vec localF; - - DMGetLocalVector(m_dmda, &localF); - DMGlobalToLocalBegin(m_dmda, m_snes_soln, INSERT_VALUES, localF); - DMGlobalToLocalEnd(m_dmda, m_snes_soln, INSERT_VALUES, localF); - dmda_arr_t in; - DMDAVecGetArray(m_dmda, localF, &in); + solver.get_dmda_arr_t(localF, in); - // m_F is already set from solve + // solver.m_F is already set from solve #if CH_SPACEDIM == 3 - for (int v = m_vmin; v < m_vmax; ++v) + for (int v = solver.m_vmin; v < solver.m_vmax; ++v) #endif { double inner_integral = 0.; - for (int u = m_umin; u < m_umax; ++u) + for (int u = solver.m_umin; u < solver.m_umax; ++u) { - const auto geometric_data = m_interp.get_geometry_data(idx); - const auto data = m_interp.get_data(idx); - const auto coords = m_interp.get_coords(idx); - const auto coords_cart = m_interp.get_cartesian_coords(idx); + const auto geometric_data = + solver.m_interp.get_geometry_data(idx); + const auto data = solver.m_interp.get_data(idx); + const auto coords = solver.m_interp.get_coords(idx); + const auto coords_cart = + solver.m_interp.get_cartesian_coords(idx); AHFunction func(data, coords, coords_cart); Tensor<2, double> g = func.get_metric(); @@ -1822,10 +1576,10 @@ double ApparentHorizon::calculate_area() #endif } - AHDeriv deriv = diff(in, u + AHDeriv deriv = solver.diff(in, u #if CH_SPACEDIM == 3 - , - v + , + v #endif ); @@ -1858,9 +1612,9 @@ double ApparentHorizon::calculate_area() double weight = m_integration_methods[0].weight( u, m_params.num_points_u, - m_interp.get_coord_system().is_u_periodic()); + solver.m_interp.get_coord_system().is_u_periodic()); - double element = sqrt(det) * weight * m_du; + double element = sqrt(det) * weight * solver.m_du; // assume a (GR_SPACEDIM - CH_SPACEDIM)-sphere leftover #if GR_SPACEDIM != CH_SPACEDIM @@ -1876,8 +1630,8 @@ double ApparentHorizon::calculate_area() #if CH_SPACEDIM == 3 double weight = m_integration_methods[1].weight( v, m_params.num_points_v, - m_interp.get_coord_system().is_v_periodic()); - integral += weight * m_dv * inner_integral; + solver.m_interp.get_coord_system().is_v_periodic()); + integral += weight * solver.m_dv * inner_integral; #elif CH_SPACEDIM == 2 integral += inner_integral; #endif @@ -1892,10 +1646,8 @@ double ApparentHorizon::calculate_area() integral *= n_sphere_coeff; #endif - DMDAVecRestoreArray(m_dmda, localF, &in); - DMRestoreLocalVector(m_dmda, &localF); - - m_interp.break_interpolation_loop(); + solver.restore_dmda_arr_t(localF, in); + solver.m_interp.break_interpolation_loop(); } // reduction across all Chombo processes (note that 'area' is 0 for non-PETSc @@ -1910,7 +1662,7 @@ double ApparentHorizon::calculate_area() area = integral; #endif - if (m_params.verbose > MIN) + if (m_params.verbose > AHParams::MIN) { pout() << "area = " << area << endl; } @@ -1952,20 +1704,21 @@ ApparentHorizon::calculate_center() // anyways { #if CH_SPACEDIM == 3 - for (int v = m_vmin; v < m_vmax; ++v) + for (int v = solver.m_vmin; v < solver.m_vmax; ++v) #endif { - for (int u = m_umin; u < m_umax; ++u) + for (int u = solver.m_umin; u < solver.m_umax; ++u) { for (unsigned i = 0; i < CH_SPACEDIM; ++i) { - double coord_i = m_interp.get_coord_system().get_grid_coord( - i, m_F[idx], m_u[idx] + double coord_i = + solver.m_interp.get_coord_system().get_grid_coord( + i, solver.m_F[idx], solver.m_u[idx] #if CH_SPACEDIM == 3 - , - m_v[idx] + , + solver.m_v[idx] #endif - ); + ); // temp[i] += point[i]; // old method if (coord_i > max_temp[i]) @@ -2006,7 +1759,7 @@ ApparentHorizon::calculate_center() center[i] = (max_point[i] + min_point[i]) / 2.; } - if (m_params.verbose > SOME) + if (m_params.verbose > AHParams::SOME) { pout() << "max_point: (" << max_point[0] << ", " << max_point[1] #if CH_SPACEDIM == 3 @@ -2022,7 +1775,7 @@ ApparentHorizon::calculate_center() update_old_centers(center); // set new - if (m_params.verbose > MIN) + if (m_params.verbose > AHParams::MIN) { pout() << "center: (" << center[0] << ", " << center[1] #if CH_SPACEDIM == 3 @@ -2044,7 +1797,8 @@ void ApparentHorizon::calculate_minmax_F() const double local_min = std::numeric_limits::max(); if (PETScCommunicator::is_rank_active()) { - auto local_minmax = (std::minmax_element(m_F.begin(), m_F.end())); + auto local_minmax = + (std::minmax_element(solver.m_F.begin(), solver.m_F.end())); local_min = *(local_minmax.first); local_max = *(local_minmax.second); } @@ -2080,10 +1834,10 @@ void ApparentHorizon::calculate_average_F() const return std::move(sums); }; - auto local_sums = - std::accumulate(m_F.begin(), m_F.end(), sums, lambda_sum); + auto local_sums = std::accumulate(solver.m_F.begin(), solver.m_F.end(), + sums, lambda_sum); - local_size = m_F.size(); + local_size = solver.m_F.size(); local_sum = local_sums.first; local_sum_sq = local_sums.second; } diff --git a/Source/ApparentHorizonFinder/PETScAHSolver.hpp b/Source/ApparentHorizonFinder/PETScAHSolver.hpp new file mode 100644 index 000000000..cfcaa0dee --- /dev/null +++ b/Source/ApparentHorizonFinder/PETScAHSolver.hpp @@ -0,0 +1,178 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef _PETSCAHSOLVER_HPP_ +#define _PETSCAHSOLVER_HPP_ + +#include +// #include + +#include "AHDeriv.hpp" +#include "AHInterpolation.hpp" +#include "AHParams.hpp" + +#if CH_SPACEDIM == 3 +typedef PetscScalar **dmda_arr_t; +#elif CH_SPACEDIM == 2 +typedef PetscScalar *dmda_arr_t; +#endif + +//! Helper class for ApparentHorizon class that manages all weird PETSc stuff +template class PETScAHSolver +{ + public: + using AHInterpolation = AHInterpolation_t; + using AHParams = AHParams_t; + + public: + //! AH that finds the zero of expansion + PETScAHSolver(const AHInterpolation &a_interp, double a_initial_guess, + const AHParams &a_params, + const typename AHFunction::params &a_func_params); + ~PETScAHSolver(); + + //! function to calculate 1st and 2nd derivatives of 'in' + //! (tipically corresponds to our 'f' function) + //! in the 'u' and 'v' directions + AHDeriv diff(const dmda_arr_t in, int u +#if CH_SPACEDIM == 3 + , + int v +#endif + ); + + //! interpolate (u,v) 2D grid points at restart if number of points in + //! either direction changed + //! This only interpolates the points that the PETSc rank that called it has + //! returns whether or not points were interpolated + bool interpolate_ah(const std::vector> &old_coords); + + void solve(); + SNESConvergedReason getConvergedReason() const; + + void get_dmda_arr_t(Vec &localF, dmda_arr_t &in); + // must be called at the end of the function that called 'get_dmda_arr_t' + void restore_dmda_arr_t(Vec &localF, dmda_arr_t &in); + + double get_initial_guess() const; + void set_initial_guess(double a_initial_guess); + void reset_initial_guess(); + + const std::array &get_origin() const; + void set_origin(const std::array &a_origin); + + void initialise(); //!< initialise automatically done in constructor + void finalise(); //!< finalise automatically done in destructor + + // variables + public: + //! Geometries of the AH + //! 'm_geom_plus' and 'm_geom_minus' are used to calculate the + //! jacobian of the expansion using a 'delta' numerical differentiation + AHInterpolation m_interp; + AHInterpolation m_interp_plus; + AHInterpolation m_interp_minus; + + const bool m_periodic_u; //!< is 'u' periodic? + PetscInt m_num_global_u; //!< total number of grid points in 'u' coordinate + double m_du; //!< physical 'delta' in 'u' coordinate + +#if CH_SPACEDIM == 3 + const bool m_periodic_v; //!< is 'v' periodic? + PetscInt m_num_global_v; //!< total number of grid points in 'u' coordinate + double m_dv; //!< physical 'delta' in 'v' coordinate +#endif + + //! vectors to store and manipulate 'F', u', and 'v' + //! internally, in interaction with the 'AHInterpolation's + std::vector m_F; + std::vector m_u; +#if CH_SPACEDIM == 3 + std::vector m_v; +#endif + + //! minimums and maximums of coordinates 'u' and 'v' + //! of the PETSc grid specific to the current rank + PetscInt m_umin; + PetscInt m_umax; + +#if CH_SPACEDIM == 3 + PetscInt m_vmin; + PetscInt m_vmax; +#endif + + //! number of points in 'u' and 'v' direction + //! (m_nu = m_umax - m_umin) + PetscInt m_nu; +#if CH_SPACEDIM == 3 + PetscInt m_nv; +#endif + + // any parameters that want to be saved to be passed to optimization + // function + typename AHFunction::params m_func_params; + + // variables + private: + double m_initial_guess; //!< initial guess for AH (saved so that it can be + //!< re-used when atempting to solve again) + + const AHParams &m_params; //!< set of AH parameters + + //!< used to compute jacobian of expansion (numerical differentiation) + static constexpr double eps = 1e-7; + + // PETSc main object + DM m_dmda; + //! Scalable Nonlinear Equations Solvers + SNES m_snes; + + Vec m_snes_soln; + Vec m_snes_rhs; + Mat m_snes_jac; + + private: + //! set the default stencils of AHDeriv at position {u,v} + void set_stencils(AHDeriv &out, int u +#if CH_SPACEDIM == 3 + , + int v +#endif + ); + + //! private functions used to compute the RHS (the expansion) and it's + //! jacobian + void form_function(Vec F, Vec Rhs); + void form_jacobian(Vec F, Mat J); + + //! helper for 'form_jacobian' + double point_jacobian(int u, int u_stencil, +#if CH_SPACEDIM == 3 + int v, int v_stencil, +#endif + dmda_arr_t in, int idx, + const AHInterpolation &interp_plus, + const AHInterpolation &interp_minus); + + //! functions used by PETSc based on 'form_function' and 'form_jacobian' + static PetscErrorCode Petsc_form_function(SNES snes, Vec F, Vec Rhs, + void *ptr); + + static PetscErrorCode +#if PETSC_VERSION_GE(3, 5, 0) + Petsc_form_jacobian(SNES snes, Vec F, Mat Amat, Mat Pmat, void *ptr); +#else + Petsc_form_jacobian(SNES snes, Vec F, Mat *Amat, Mat *Pmat, + MatStructure *flag, void *ptr); +#endif + + // monitor function required for SNES + static PetscErrorCode Petsc_SNES_monitor(SNES snes, PetscInt its, + PetscReal norm, void *ptr); +}; + +#include "PETScAHSolver.impl.hpp" + +#endif /* _PETSCAHSOLVER_HPP_ */ diff --git a/Source/ApparentHorizonFinder/ApparentHorizon_petsc.impl.hpp b/Source/ApparentHorizonFinder/PETScAHSolver.impl.hpp similarity index 73% rename from Source/ApparentHorizonFinder/ApparentHorizon_petsc.impl.hpp rename to Source/ApparentHorizonFinder/PETScAHSolver.impl.hpp index ef85723d2..a63c9c63b 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon_petsc.impl.hpp +++ b/Source/ApparentHorizonFinder/PETScAHSolver.impl.hpp @@ -3,12 +3,12 @@ * Please refer to LICENSE in GRChombo's root directory. */ -#ifndef _APPARENTHORIZON_HPP_ -#error "This file should only be included through ApparentHorizon.hpp" +#ifndef _PETSCAHSOLVER_HPP_ +#error "This file should only be included through PETScAHSolver.hpp" #endif -#ifndef _APPARENTHORIZON_PETSC_IMPL_HPP_ -#define _APPARENTHORIZON_PETSC_IMPL_HPP_ +#ifndef _PETSCAHSOLVER_IMPL_HPP_ +#define _PETSCAHSOLVER_IMPL_HPP_ // petsc libraries #include "petscsys.h" @@ -19,16 +19,38 @@ #include "SimpleArrayBox.hpp" #include "SimpleInterpSource.hpp" -///////////////////////////////////////////////////////////////////////// -///////////////////////////////////////////////////////////////////////// -/////////////////////////// PETSc stuff below /////////////////////////// -///////////////////////////////////////////////////////////////////////// -///////////////////////////////////////////////////////////////////////// +template +PETScAHSolver::PETScAHSolver( + const AHInterpolation &a_interp, double a_initial_guess, + const AHParams &a_params, const typename AHFunction::params &a_func_params) + : m_initial_guess(a_initial_guess), + + m_params(a_params), m_func_params(a_func_params), + + m_interp(a_interp), m_interp_plus(a_interp), m_interp_minus(a_interp), + + m_periodic_u(a_interp.get_coord_system().is_u_periodic()), + m_num_global_u(a_params.num_points_u) + +#if CH_SPACEDIM == 3 + , + m_periodic_v(a_interp.get_coord_system().is_v_periodic()), + m_num_global_v(a_params.num_points_v) +#endif +{ + initialise(); +} template -void ApparentHorizon::initialise_PETSc() +PETScAHSolver::~PETScAHSolver() { - CH_TIME("ApparentHorizon::initialise_PETSc"); + finalise(); +} + +template +void PETScAHSolver::initialise() +{ + CH_TIME("PETScAHSolver::initialise_PETSc"); if (!PETScCommunicator::is_rank_active()) return; @@ -134,7 +156,7 @@ void ApparentHorizon::initialise_PETSc() SNESSetFromOptions(m_snes); - if (m_params.verbose > MIN) + if (m_params.verbose > AHParams::MIN) { SNESType snes_type; SNESGetType(m_snes, &snes_type); @@ -156,7 +178,7 @@ void ApparentHorizon::initialise_PETSc() &ksp_maxits); pout() << "-------------------------------------\n"; - pout() << "ApparentHorizon Options:\n"; + pout() << "PETScAHSolver Options:\n"; pout() << "-------------------------------------\n"; pout() << "PETSc SNES Options:\n"; pout() << "Type: " << snes_type << "\n"; @@ -175,7 +197,7 @@ void ApparentHorizon::initialise_PETSc() } template -void ApparentHorizon::finalise_PETSc() +void PETScAHSolver::finalise() { if (!PETScCommunicator::is_rank_active()) return; @@ -188,9 +210,13 @@ void ApparentHorizon::finalise_PETSc() } template -bool ApparentHorizon::interpolate_ah( - dmda_arr_t f, const std::vector> &old_coords) +bool PETScAHSolver::interpolate_ah( + const std::vector> &old_coords) { + // read PETSc array to 'f' + dmda_arr_t f; + DMDAVecGetArray(m_dmda, m_snes_soln, &f); + // check if number of points changed bool points_changed = false; const double du = old_coords[0][1] - old_coords[0][0]; @@ -214,6 +240,8 @@ bool ApparentHorizon::interpolate_ah( #endif points_changed |= (old_number_of_u != m_num_global_u); + bool force_restart = false; + if (!points_changed) { // these should be equal if all is good @@ -261,107 +289,245 @@ bool ApparentHorizon::interpolate_ah( #endif } } - - return false; } - - if (m_params.verbose > NONE) + else { - pout() << "Number of AH points changed. Interpolating old ones." - << std::endl; - } + force_restart = true; - // start interpolating + if (m_params.verbose > AHParams::NONE) + { + pout() << "Number of AH points changed. Interpolating old ones." + << std::endl; + } -#if CH_SPACEDIM == 3 - // local, no derivative - std::array derivs = {0, 0}; - std::array dxs = {du, dv}; + // start interpolating - SimpleArrayBox box( - {old_number_of_u, old_number_of_v}, old_coords[CH_SPACEDIM - 1], - {m_interp.get_coord_system().is_u_periodic(), - m_interp.get_coord_system().is_v_periodic()}); - SimpleInterpSource source( - {old_number_of_u, old_number_of_v}, dxs, - {m_interp.get_coord_system().is_u_periodic(), - m_interp.get_coord_system().is_v_periodic()}); +#if CH_SPACEDIM == 3 + // local, no derivative + std::array derivs = {0, 0}; + std::array dxs = {du, dv}; + + SimpleArrayBox box( + {old_number_of_u, old_number_of_v}, old_coords[CH_SPACEDIM - 1], + {m_interp.get_coord_system().is_u_periodic(), + m_interp.get_coord_system().is_v_periodic()}); + SimpleInterpSource source( + {old_number_of_u, old_number_of_v}, dxs, + {m_interp.get_coord_system().is_u_periodic(), + m_interp.get_coord_system().is_v_periodic()}); #elif CH_SPACEDIM == 2 - std::array derivs = {0}; - std::array dxs = {du}; - - SimpleArrayBox box( - {old_number_of_u}, old_coords[CH_SPACEDIM - 1], - {m_interp.get_coord_system().is_u_periodic()}); - SimpleInterpSource source( - {old_number_of_u}, dxs, {m_interp.get_coord_system().is_u_periodic()}); + std::array derivs = {0}; + std::array dxs = {du}; + + SimpleArrayBox box( + {old_number_of_u}, old_coords[CH_SPACEDIM - 1], + {m_interp.get_coord_system().is_u_periodic()}); + SimpleInterpSource source( + {old_number_of_u}, dxs, + {m_interp.get_coord_system().is_u_periodic()}); #endif - const int Order = 4; - bool verbose = false; - Lagrange interpolator4(source, verbose); + const int Order = 4; + bool verbose = false; + Lagrange interpolator4(source, verbose); #if CH_SPACEDIM == 3 - for (int v = m_vmin; v < m_vmax; ++v) - { - double v_val = m_interp.get_coord_system().v(v, m_num_global_v); - double v_old_idx = v_val / dv; - // round such that we don't get AMRInterpolator errors - // (e.g. the last point idx=N-1 should be exact, and not being exact was - // generating this error in the Lagrange intrepolator trying to access - // cells beyond the last cell, and hence error... this fixed it) - if (v_old_idx - int(v_old_idx) < 1.e-7) - v_old_idx = int(v_old_idx); + for (int v = m_vmin; v < m_vmax; ++v) + { + double v_val = m_interp.get_coord_system().v(v, m_num_global_v); + double v_old_idx = v_val / dv; + // round such that we don't get AMRInterpolator errors + // (e.g. the last point idx=N-1 should be exact, and not being exact + // was generating this error in the Lagrange intrepolator trying to + // access cells beyond the last cell, and hence error... this fixed + // it) + if (v_old_idx - int(v_old_idx) < 1.e-7) + v_old_idx = int(v_old_idx); #else - { -#endif - for (int u = m_umin; u < m_umax; ++u) { - double u_val = m_interp.get_coord_system().u(u, m_num_global_u); - double u_old_idx = u_val / du; - // round such that we don't get AMRInterpolator errors (same as for - // 'v') - if (u_old_idx - int(u_old_idx) < 1.e-7) - u_old_idx = int(u_old_idx); +#endif + for (int u = m_umin; u < m_umax; ++u) + { + double u_val = m_interp.get_coord_system().u(u, m_num_global_u); + double u_old_idx = u_val / du; + // round such that we don't get AMRInterpolator errors (same as + // for 'v') + if (u_old_idx - int(u_old_idx) < 1.e-7) + u_old_idx = int(u_old_idx); #if CH_SPACEDIM == 3 - // int idx_global = v * m_num_global_u + u; - int idx_local = (v - m_vmin) * m_nu + (u - m_umin); + // int idx_global = v * m_num_global_u + u; + int idx_local = (v - m_vmin) * m_nu + (u - m_umin); - std::array evalCoord = {u_old_idx, - v_old_idx}; + std::array evalCoord = {u_old_idx, + v_old_idx}; #elif CH_SPACEDIM == 2 - // int idx_global = u; - int idx_local = (u - m_umin); - std::array evalCoord = {u_old_idx}; + // int idx_global = u; + int idx_local = (u - m_umin); + std::array evalCoord = {u_old_idx}; #endif - interpolator4.setup(derivs, evalCoord); - m_F[idx_local] = interpolator4.interpData(box); + interpolator4.setup(derivs, evalCoord); + m_F[idx_local] = interpolator4.interpData(box); #if CH_SPACEDIM == 3 - // as 'write_coords_file' preserves order - f[v][u] = m_F[idx_local]; + // as 'write_coords_file' preserves order + f[v][u] = m_F[idx_local]; #elif CH_SPACEDIM == 2 - f[u] = m_F[idx_local]; + f[u] = m_F[idx_local]; #endif + } + } + + if (m_params.verbose > AHParams::NONE) + { + pout() << "Interpolation Successfull." << std::endl; } } - if (m_params.verbose > NONE) + // write PETSc array back + DMDAVecRestoreArray(m_dmda, m_snes_soln, &f); + + return force_restart; +} + +template +const std::array & +PETScAHSolver::get_origin() const +{ + return m_interp.get_coord_system().get_origin(); +} + +template +void PETScAHSolver::set_origin( + const std::array &a_origin) +{ + m_interp.set_origin(a_origin); + m_interp_plus.set_origin(a_origin); + m_interp_minus.set_origin(a_origin); + + if (m_params.verbose > AHParams::SOME) { - pout() << "Interpolation Successfull." << std::endl; + pout() << "Setting origin to (" << a_origin[0] << "," << a_origin[1] +#if CH_SPACEDIM == 3 + << "," << a_origin[2] +#endif + << ")" << std::endl; } +} - return true; +template +double PETScAHSolver::get_initial_guess() const +{ + return m_initial_guess; } template -void ApparentHorizon::set_stencils(AHDeriv &out, - int u +void PETScAHSolver::set_initial_guess( + double a_initial_guess) +{ + m_initial_guess = a_initial_guess; +} + +template +void PETScAHSolver::reset_initial_guess() +{ + CH_TIME("ApparentHorizon::reset_initial_guess"); + + if (!PETScCommunicator::is_rank_active()) + return; + + auto origin = get_origin(); + + // verify origin +- initial guess is inside the grid + bool out_of_grid = m_interp.is_in_grid(origin, m_initial_guess); + CH_assert(!out_of_grid); + + if (m_params.verbose > AHParams::NONE) + { + pout() << "Setting Initial Guess to f=" << m_initial_guess + << " centered at (" << origin[0] << "," << origin[1] #if CH_SPACEDIM == 3 - , - int v + << "," << origin[2] +#endif + << ")" << std::endl; + } + + // read PETSc array to 'f' + dmda_arr_t f; + DMDAVecGetArray(m_dmda, m_snes_soln, &f); + +#if CH_SPACEDIM == 3 + for (int v = m_vmin; v < m_vmax; ++v) +#endif + { + for (int u = m_umin; u < m_umax; ++u) + { +#if CH_SPACEDIM == 3 + double &f_point = f[v][u]; +#else + double &f_point = f[u]; +#endif + + f_point = m_initial_guess; + } + } + + // write PETSc array back + DMDAVecRestoreArray(m_dmda, m_snes_soln, &f); +} + +template +void PETScAHSolver::solve() +{ + // actual solve happens here! + SNESSolve(m_snes, NULL, m_snes_soln); + + PetscInt its; + SNESGetIterationNumber(m_snes, &its); + if (m_params.verbose > AHParams::MIN) + { + pout() << "SNES Iteration number " << its << endl; + } + SNESGetLinearSolveIterations(m_snes, &its); + if (m_params.verbose > AHParams::MIN) + { + pout() << "KSP Iteration number " << its << endl; + } +} + +template +SNESConvergedReason +PETScAHSolver::getConvergedReason() const +{ + SNESConvergedReason reason; + SNESGetConvergedReason(m_snes, &reason); + return reason; +} + +template +void PETScAHSolver::get_dmda_arr_t(Vec &localF, + dmda_arr_t &in) +{ + DMGetLocalVector(m_dmda, &localF); + DMGlobalToLocalBegin(m_dmda, m_snes_soln, INSERT_VALUES, localF); + DMGlobalToLocalEnd(m_dmda, m_snes_soln, INSERT_VALUES, localF); + DMDAVecGetArray(m_dmda, localF, &in); +} +template +void PETScAHSolver::restore_dmda_arr_t( + Vec &localF, dmda_arr_t &in) +{ + DMDAVecRestoreArray(m_dmda, localF, &in); + DMRestoreLocalVector(m_dmda, &localF); +} + +template +void PETScAHSolver::set_stencils(AHDeriv &out, + int u +#if CH_SPACEDIM == 3 + , + int v #endif ) { @@ -526,15 +692,15 @@ void ApparentHorizon::set_stencils(AHDeriv &out, } template -AHDeriv ApparentHorizon::diff(const dmda_arr_t in, - int u +AHDeriv PETScAHSolver::diff(const dmda_arr_t in, + int u #if CH_SPACEDIM == 3 - , - int v + , + int v #endif ) { - CH_TIME("ApparentHorizon::diff"); + CH_TIME("PETScAHSolver::diff"); AHDeriv out; set_stencils(out, u @@ -603,9 +769,9 @@ AHDeriv ApparentHorizon::diff(const dmda_arr_t in, } template -void ApparentHorizon::form_function(Vec F, Vec Rhs) +void PETScAHSolver::form_function(Vec F, Vec Rhs) { - CH_TIME("ApparentHorizon::form_function"); + CH_TIME("PETScAHSolver::form_function"); // Scatter ghost cells Vec localF; @@ -699,9 +865,9 @@ void ApparentHorizon::form_function(Vec F, Vec Rhs) } template -void ApparentHorizon::form_jacobian(Vec F, Mat J) +void PETScAHSolver::form_jacobian(Vec F, Mat J) { - CH_TIME("ApparentHorizon::form_jacobian"); + CH_TIME("PETScAHSolver::form_jacobian"); // Scatter ghost cells Vec localF; @@ -878,14 +1044,13 @@ void ApparentHorizon::form_jacobian(Vec F, Mat J) } template -double ApparentHorizon::point_jacobian( +double PETScAHSolver::point_jacobian( int u, int u_stencil, #if CH_SPACEDIM == 3 int v, int v_stencil, #endif - dmda_arr_t in, int idx, - const AHInterpolation &interp_plus, - const AHInterpolation &interp_minus) + dmda_arr_t in, int idx, const AHInterpolation &interp_plus, + const AHInterpolation &interp_minus) { #if CH_SPACEDIM == 3 double &_in = in[v_stencil][u_stencil]; @@ -944,13 +1109,10 @@ double ApparentHorizon::point_jacobian( //! functions used by PETSc based on 'form_function' and 'form_jacobian' template -PetscErrorCode -ApparentHorizon::Petsc_form_function(SNES snes, - Vec F, - Vec Rhs, - void *ptr) +PetscErrorCode PETScAHSolver::Petsc_form_function( + SNES snes, Vec F, Vec Rhs, void *ptr) { - ApparentHorizon &ah = *reinterpret_cast(ptr); + PETScAHSolver &ah = *reinterpret_cast(ptr); CH_assert(ah.m_snes == snes); ah.form_function(F, Rhs); return 0; @@ -959,14 +1121,16 @@ ApparentHorizon::Petsc_form_function(SNES snes, template PetscErrorCode #if PETSC_VERSION_GE(3, 5, 0) -ApparentHorizon::Petsc_form_jacobian( - SNES snes, Vec F, Mat Amat, Mat Pmat, void *ptr) +PETScAHSolver::Petsc_form_jacobian(SNES snes, + Vec F, Mat Amat, + Mat Pmat, + void *ptr) #else -ApparentHorizon::Petsc_form_jacobian( +PETScAHSolver::Petsc_form_jacobian( SNES snes, Vec F, Mat *Amat, Mat *Pmat, MatStructure *flag, void *ptr) #endif { - ApparentHorizon &ah = *reinterpret_cast(ptr); + PETScAHSolver &ah = *reinterpret_cast(ptr); #if PETSC_VERSION_GE(3, 5, 0) CH_assert(ah.m_snes == snes && Amat == Pmat); @@ -980,12 +1144,12 @@ ApparentHorizon::Petsc_form_jacobian( } template -PetscErrorCode ApparentHorizon::Petsc_SNES_monitor( +PetscErrorCode PETScAHSolver::Petsc_SNES_monitor( SNES snes, PetscInt its, PetscReal norm, void *ptr) { - ApparentHorizon &ah = *reinterpret_cast(ptr); + PETScAHSolver &ah = *reinterpret_cast(ptr); CH_assert(ah.m_snes == snes); return 0; } -#endif // _APPARENTHORIZON_PETSC_IMPL_HPP_ +#endif // _PETSCAHSOLVER_IMPL_HPP_ diff --git a/Source/GRChomboCore/SimulationParametersBase.hpp b/Source/GRChomboCore/SimulationParametersBase.hpp index d50b72078..d238ee9b8 100644 --- a/Source/GRChomboCore/SimulationParametersBase.hpp +++ b/Source/GRChomboCore/SimulationParametersBase.hpp @@ -327,7 +327,7 @@ class SimulationParametersBase : public ChomboParameters #ifdef USE_AHFINDER bool AH_activate; - AHFinder::params AH_params; + AHParams_t AH_params; #endif }; diff --git a/Tests/ApparentHorizonFinderTest2D/SimulationParameters.hpp b/Tests/ApparentHorizonFinderTest2D/SimulationParameters.hpp index a4b7685b0..e38c55637 100755 --- a/Tests/ApparentHorizonFinderTest2D/SimulationParameters.hpp +++ b/Tests/ApparentHorizonFinderTest2D/SimulationParameters.hpp @@ -24,7 +24,7 @@ class SimulationParameters : public ChomboParameters } #ifdef USE_AHFINDER - AHFinder::params AH_params; + AHParams_t AH_params; #endif }; diff --git a/Tests/ApparentHorizonFinderTest3D/SimulationParameters.hpp b/Tests/ApparentHorizonFinderTest3D/SimulationParameters.hpp index e7f39c181..6953704d0 100755 --- a/Tests/ApparentHorizonFinderTest3D/SimulationParameters.hpp +++ b/Tests/ApparentHorizonFinderTest3D/SimulationParameters.hpp @@ -42,7 +42,7 @@ class SimulationParameters : public ChomboParameters #ifdef USE_AHFINDER double initial_guess; - AHFinder::params AH_params; + AHParams_t AH_params; #endif }; From db13f8480b2c7cc2d43c19254e36961a13b2b6ae Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Wed, 30 Jun 2021 00:43:33 +0100 Subject: [PATCH 27/92] Move AHFunction::params to inside AHParams --- Examples/BinaryBH/Main_BinaryBH.cpp | 6 ++--- Examples/BinaryBH/SimulationParameters.hpp | 3 --- Examples/KerrBH/Main_KerrBH.cpp | 11 +++++---- Examples/KerrBH/SimulationParameters.hpp | 9 -------- Examples/KerrBH/params.txt | 2 +- Examples/KerrBH/params_cheap.txt | 2 +- Source/ApparentHorizonFinder/AHFinder.hpp | 8 ------- .../ApparentHorizonFinder/AHFinder.impl.hpp | 18 ++------------- .../AHFunctionDefault.hpp | 2 ++ Source/ApparentHorizonFinder/AHFunctions.hpp | 23 +++++++++++++++---- Source/ApparentHorizonFinder/AHParams.hpp | 4 ++++ .../ApparentHorizonFinder/ApparentHorizon.hpp | 11 +-------- .../ApparentHorizon.impl.hpp | 13 +---------- .../ApparentHorizonFinder/PETScAHSolver.hpp | 7 +----- .../PETScAHSolver.impl.hpp | 10 ++++---- 15 files changed, 45 insertions(+), 84 deletions(-) diff --git a/Examples/BinaryBH/Main_BinaryBH.cpp b/Examples/BinaryBH/Main_BinaryBH.cpp index 676eedd63..898af893c 100644 --- a/Examples/BinaryBH/Main_BinaryBH.cpp +++ b/Examples/BinaryBH/Main_BinaryBH.cpp @@ -83,11 +83,9 @@ int runGRChombo(int argc, char *argv[]) AHSurfaceGeometry sph2(sim_params.bh2_params.center); bh_amr.m_ah_finder.add_ah(sph1, sim_params.AH_1_initial_guess, - sim_params.AH_params, - sim_params.expansion_params); + sim_params.AH_params); bh_amr.m_ah_finder.add_ah(sph2, sim_params.AH_2_initial_guess, - sim_params.AH_params, - sim_params.expansion_params); + sim_params.AH_params); bh_amr.m_ah_finder.add_ah_merger(0, 1, sim_params.AH_params); } #endif diff --git a/Examples/BinaryBH/SimulationParameters.hpp b/Examples/BinaryBH/SimulationParameters.hpp index d0adfc6db..c2358519f 100644 --- a/Examples/BinaryBH/SimulationParameters.hpp +++ b/Examples/BinaryBH/SimulationParameters.hpp @@ -47,8 +47,6 @@ class SimulationParameters : public SimulationParametersBase 0.5 * bh2_params.mass); pp.load("AH_set_origins_to_punctures", AH_set_origins_to_punctures, false); - pp.load("AH_expansion_radius_power", - expansion_params.expansion_radius_power, 1.); #endif } @@ -323,7 +321,6 @@ class SimulationParameters : public SimulationParametersBase double AH_1_initial_guess; double AH_2_initial_guess; bool AH_set_origins_to_punctures; - ExpansionFunction::params expansion_params; #endif }; diff --git a/Examples/KerrBH/Main_KerrBH.cpp b/Examples/KerrBH/Main_KerrBH.cpp index e6bc0fa09..d7f4543b8 100644 --- a/Examples/KerrBH/Main_KerrBH.cpp +++ b/Examples/KerrBH/Main_KerrBH.cpp @@ -56,11 +56,12 @@ int runGRChombo(int argc, char *argv[]) AHSurfaceGeometry sph(sim_params.kerr_params.center); #ifdef USE_CHI_CONTOURS // uncomment in UserVariables - std::string str_chi = std::to_string(sim_params.look_for_chi_contour); - bh_amr.m_ah_finder.add_ah( - sph, sim_params.AH_initial_guess, sim_params.AH_params, - sim_params.look_for_chi_contour, "stats_chi_" + str_chi + "_", - "coords_chi_" + str_chi + "_"); + std::string str_chi = std::to_string( + sim_params.AH_params.func_params.look_for_chi_contour); + sim_params.AH_params.stats_prefix = "stats_chi_" + str_chi + "_"; + sim_params.AH_params.coords_prefix = "coords_chi_" + str_chi + "_"; + bh_amr.m_ah_finder.add_ah(sph, sim_params.AH_initial_guess, + sim_params.AH_params); #else bh_amr.m_ah_finder.add_ah(sph, sim_params.AH_initial_guess, sim_params.AH_params); diff --git a/Examples/KerrBH/SimulationParameters.hpp b/Examples/KerrBH/SimulationParameters.hpp index f6bbb5088..8a8ec88a8 100644 --- a/Examples/KerrBH/SimulationParameters.hpp +++ b/Examples/KerrBH/SimulationParameters.hpp @@ -34,10 +34,6 @@ class SimulationParameters : public SimulationParametersBase #ifdef USE_AHFINDER pp.load("AH_initial_guess", AH_initial_guess, 0.5 * kerr_params.mass); -#ifdef USE_CHI_CONTOURS - pp.load("look_for_chi_contour", look_for_chi_contour); - CH_assert(look_for_chi_contour > 0.); -#endif #endif } @@ -64,11 +60,6 @@ class SimulationParameters : public SimulationParametersBase #ifdef USE_AHFINDER double AH_initial_guess; - // example of how to change the expansion function - double look_for_chi_contour; // look for a chi contour instead of the AH - // (negative number to look for AH) - // changes only the expansion function - // (ApparentHorizon::get_expansion) #endif }; diff --git a/Examples/KerrBH/params.txt b/Examples/KerrBH/params.txt index 00805d6b3..08be4b59c 100644 --- a/Examples/KerrBH/params.txt +++ b/Examples/KerrBH/params.txt @@ -166,7 +166,7 @@ AH_verbose = 2 # AH_initial_guess = 0.5 -# look_for_chi_contour = 0.2 +# AH_look_for_chi_contour = 0.2 AH_num_extra_vars = 1 AH_extra_vars = chi diff --git a/Examples/KerrBH/params_cheap.txt b/Examples/KerrBH/params_cheap.txt index 13c8c33f8..6836268ef 100644 --- a/Examples/KerrBH/params_cheap.txt +++ b/Examples/KerrBH/params_cheap.txt @@ -169,7 +169,7 @@ AH_verbose = 2 # AH_initial_guess = 0.5 -# look_for_chi_contour = 0.2 +# AH_look_for_chi_contour = 0.2 AH_num_extra_vars = 2 AH_extra_vars = chi d1_Ham diff --git a/Source/ApparentHorizonFinder/AHFinder.hpp b/Source/ApparentHorizonFinder/AHFinder.hpp index 029efbe07..c0cfe8ab5 100644 --- a/Source/ApparentHorizonFinder/AHFinder.hpp +++ b/Source/ApparentHorizonFinder/AHFinder.hpp @@ -102,14 +102,6 @@ template class AHFinder bool solve_first_step = true //!< whether or not to solve if t=0 ); - //! returns the index of the AH in m_apparent_horizons - //! allows for personalized optimizer that finds zero of function - //! 'AHFunction' (that can have some ::params) - int add_ah(const SurfaceGeometry &a_coord_system, double a_initial_guess, - const AHParams &a_params, - const typename AHFunction::params &a_func_params, - bool solve_first_step = true); - // returns the index of the AH in m_apparent_horizons int add_ah_merger(int ah1, int ah2, const AHParams &a_params); diff --git a/Source/ApparentHorizonFinder/AHFinder.impl.hpp b/Source/ApparentHorizonFinder/AHFinder.impl.hpp index 50e457e86..adfe4448d 100644 --- a/Source/ApparentHorizonFinder/AHFinder.impl.hpp +++ b/Source/ApparentHorizonFinder/AHFinder.impl.hpp @@ -25,16 +25,6 @@ template int AHFinder::add_ah( const SurfaceGeometry &a_coord_system, double a_initial_guess, const AHParams &a_params, bool solve_first_step) -{ - return add_ah(a_coord_system, a_initial_guess, a_params, - typename AHFunction::params(), solve_first_step); -} - -template -int AHFinder::add_ah( - const SurfaceGeometry &a_coord_system, double a_initial_guess, - const AHParams &a_params, const typename AHFunction::params &a_func_params, - bool solve_first_step) { PETScCommunicator::initialize(a_params.num_ranks); @@ -51,7 +41,7 @@ int AHFinder::add_ah( m_apparent_horizons.push_back( new ApparentHorizon( - interp, a_initial_guess, a_params, a_func_params, + interp, a_initial_guess, a_params, a_params.stats_prefix + std::to_string(num_ah + 1), a_params.coords_prefix + std::to_string(num_ah + 1) + "_", solve_first_step)); @@ -80,11 +70,7 @@ int AHFinder::add_ah_merger( m_apparent_horizons[ah1]->get_ah_interp().get_coord_system(); coord_system.set_origin(origin_merger); - auto &function_to_optimize_params = - m_apparent_horizons[ah1]->get_petsc_solver().m_func_params; - - int num = add_ah(coord_system, initial_guess_merger, a_params, - function_to_optimize_params, do_solve); + int num = add_ah(coord_system, initial_guess_merger, a_params, do_solve); m_merger_pairs[num] = {ah1, ah2}; return num; diff --git a/Source/ApparentHorizonFinder/AHFunctionDefault.hpp b/Source/ApparentHorizonFinder/AHFunctionDefault.hpp index cd7b599f2..1fd0608d4 100644 --- a/Source/ApparentHorizonFinder/AHFunctionDefault.hpp +++ b/Source/ApparentHorizonFinder/AHFunctionDefault.hpp @@ -10,6 +10,7 @@ #include "AHDeriv.hpp" #include "AHGeometryData.hpp" #include "AlwaysInline.hpp" +#include "GRParmParse.hpp" #include "Tensor.hpp" ///////////////////////////////////////////////////////// @@ -57,6 +58,7 @@ struct AHFunctionDefault struct params // no params needed { + void read_params(GRParmParse &pp) {} }; // not defined by default diff --git a/Source/ApparentHorizonFinder/AHFunctions.hpp b/Source/ApparentHorizonFinder/AHFunctions.hpp index 30339f5b8..b9b390ade 100644 --- a/Source/ApparentHorizonFinder/AHFunctions.hpp +++ b/Source/ApparentHorizonFinder/AHFunctions.hpp @@ -219,8 +219,14 @@ struct ExpansionFunction : AHFunctionDefault struct params { - double expansion_radius_power = 1.; + double expansion_radius_power; + + void read_params(GRParmParse &pp) + { + pp.load("AH_expansion_radius_power", expansion_radius_power, 1.); + } }; + double get(const AHGeometryData &geo_data, const AHDeriv &deriv, const params &a_params) const { @@ -362,12 +368,21 @@ struct ChiContourFunction : AHFunctionDefault chi = a_data.vars.at(c_chi); } - using params = double; // chi contour to look for + struct params + { + double look_for_chi_contour; + + void read_params(GRParmParse &pp) + { + pp.load("AH_look_for_chi_contour", look_for_chi_contour); + CH_assert(look_for_chi_contour > 0.); + } + }; + double get(const AHGeometryData &geo_data, const AHDeriv &deriv, const params &a_params) const { - double look_for_chi_contour = a_params; - return chi - look_for_chi_contour; + return chi - a_params.look_for_chi_contour; } }; diff --git a/Source/ApparentHorizonFinder/AHParams.hpp b/Source/ApparentHorizonFinder/AHParams.hpp index a262ede44..cd8fcf8ab 100644 --- a/Source/ApparentHorizonFinder/AHParams.hpp +++ b/Source/ApparentHorizonFinder/AHParams.hpp @@ -86,6 +86,8 @@ template struct AHParams_t //! set to somethig bigger to avoid finding the inner AH double merger_pre_factor; // see note above (default to 1.) + typename AHFunction::params func_params; + enum verbose_level { NONE, @@ -232,6 +234,8 @@ void AHParams_t::read_params( pp.load("AH_merger_search_factor", merger_search_factor, 1.); pp.load("AH_merger_pre_factor", merger_pre_factor, 1.); + + func_params.read_params(pp); } #endif /* _AHPARAMS_HPP_ */ diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.hpp index 231085021..dae308df8 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.hpp @@ -20,7 +20,7 @@ template class ApparentHorizon using AHParams = AHParams_t; public: - //! AH that finds the zero of expansion + //! AH that finds the zero of AHFunction ApparentHorizon( const AHInterpolation &a_interp, //!< Geometry class to exchange data double a_initial_guess, //!< Initial guess for radius (or whatever @@ -33,15 +33,6 @@ template class ApparentHorizon //!< time step bool solve_first_step = true //!< whether or not to solve if t=0 ); - //! personalized optimizer that finds zero of function - //! 'a_function_to_optimize' (a void* 'a_function_to_optimize_params' can be - //! passed for auxiliary parameters passed to 'a_function_to_optimize') - ApparentHorizon(const AHInterpolation &a_interp, double a_initial_guess, - const AHParams &a_params, - const typename AHFunction::params &a_func_params, - const std::string &a_stats = "stats", - const std::string &a_coords = "coords_", - bool solve_first_step = true); void solve(double a_dt, double a_time, double a_restart_time); diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp index 1a8196e67..32d6563a9 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp @@ -26,22 +26,11 @@ ApparentHorizon::ApparentHorizon( const AHInterpolation &a_interp, double a_initial_guess, const AHParams &a_params, const std::string &a_stats, const std::string &a_coords, bool solve_first_step) - : ApparentHorizon(a_interp, a_initial_guess, a_params, AHFunction::params(), - a_stats, a_coords, solve_first_step) -{ -} - -template -ApparentHorizon::ApparentHorizon( - const AHInterpolation &a_interp, double a_initial_guess, - const AHParams &a_params, const typename AHFunction::params &a_func_params, - const std::string &a_stats, const std::string &a_coords, - bool solve_first_step) : m_params(a_params), m_stats(a_stats), m_coords(a_coords), - solver(a_interp, a_initial_guess, a_params, a_func_params), + solver(a_interp, a_initial_guess, a_params), m_printed_once(false), diff --git a/Source/ApparentHorizonFinder/PETScAHSolver.hpp b/Source/ApparentHorizonFinder/PETScAHSolver.hpp index cfcaa0dee..2b293192d 100644 --- a/Source/ApparentHorizonFinder/PETScAHSolver.hpp +++ b/Source/ApparentHorizonFinder/PETScAHSolver.hpp @@ -29,8 +29,7 @@ template class PETScAHSolver public: //! AH that finds the zero of expansion PETScAHSolver(const AHInterpolation &a_interp, double a_initial_guess, - const AHParams &a_params, - const typename AHFunction::params &a_func_params); + const AHParams &a_params); ~PETScAHSolver(); //! function to calculate 1st and 2nd derivatives of 'in' @@ -110,10 +109,6 @@ template class PETScAHSolver PetscInt m_nv; #endif - // any parameters that want to be saved to be passed to optimization - // function - typename AHFunction::params m_func_params; - // variables private: double m_initial_guess; //!< initial guess for AH (saved so that it can be diff --git a/Source/ApparentHorizonFinder/PETScAHSolver.impl.hpp b/Source/ApparentHorizonFinder/PETScAHSolver.impl.hpp index a63c9c63b..29cc8c994 100644 --- a/Source/ApparentHorizonFinder/PETScAHSolver.impl.hpp +++ b/Source/ApparentHorizonFinder/PETScAHSolver.impl.hpp @@ -22,10 +22,10 @@ template PETScAHSolver::PETScAHSolver( const AHInterpolation &a_interp, double a_initial_guess, - const AHParams &a_params, const typename AHFunction::params &a_func_params) + const AHParams &a_params) : m_initial_guess(a_initial_guess), - m_params(a_params), m_func_params(a_func_params), + m_params(a_params), m_interp(a_interp), m_interp_plus(a_interp), m_interp_minus(a_interp), @@ -852,7 +852,7 @@ void PETScAHSolver::form_function(Vec F, Vec Rhs) const auto coords = m_interp.get_coords(idx); const auto coords_cart = m_interp.get_cartesian_coords(idx); AHFunction func(data, coords, coords_cart); - _out = func.get(geometry_data, deriv, m_func_params); + _out = func.get(geometry_data, deriv, m_params.func_params); } ++idx; @@ -1076,7 +1076,7 @@ double PETScAHSolver::point_jacobian( const auto coords = interp_plus.get_coords(idx); const auto coords_cart = interp_plus.get_cartesian_coords(idx); AHFunction func(data, coords, coords_cart); - expansionPlus = func.get(geometry_data, deriv, m_func_params); + expansionPlus = func.get(geometry_data, deriv, m_params.func_params); _in = in_old; } @@ -1099,7 +1099,7 @@ double PETScAHSolver::point_jacobian( const auto coords = interp_minus.get_coords(idx); const auto coords_cart = interp_minus.get_cartesian_coords(idx); AHFunction func(data, coords, coords_cart); - expansionMinus = func.get(geometry_data, deriv, m_func_params); + expansionMinus = func.get(geometry_data, deriv, m_params.func_params); _in = in_old; } From 16dfb519eb13b372dd4a20dab9c2b6bf059a5822 Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Wed, 30 Jun 2021 00:43:44 +0100 Subject: [PATCH 28/92] Improve naming of AH data structures --- .../{AHDeriv.hpp => AHDerivData.hpp} | 8 ++++---- .../AHFunctionDefault.hpp | 12 ++++++------ Source/ApparentHorizonFinder/AHFunctions.hpp | 12 ++++++------ .../ApparentHorizonFinder/AHInterpolation.hpp | 10 +++++----- .../AHInterpolation.impl.hpp | 8 ++++---- .../{AHData.hpp => AHVarsData.hpp} | 10 ++++------ .../ApparentHorizon.impl.hpp | 6 +++--- Source/ApparentHorizonFinder/PETScAHSolver.hpp | 8 ++++---- .../PETScAHSolver.impl.hpp | 18 +++++++++--------- 9 files changed, 45 insertions(+), 47 deletions(-) rename Source/ApparentHorizonFinder/{AHDeriv.hpp => AHDerivData.hpp} (84%) rename Source/ApparentHorizonFinder/{AHData.hpp => AHVarsData.hpp} (90%) diff --git a/Source/ApparentHorizonFinder/AHDeriv.hpp b/Source/ApparentHorizonFinder/AHDerivData.hpp similarity index 84% rename from Source/ApparentHorizonFinder/AHDeriv.hpp rename to Source/ApparentHorizonFinder/AHDerivData.hpp index 4c0fe5883..dc9abed26 100644 --- a/Source/ApparentHorizonFinder/AHDeriv.hpp +++ b/Source/ApparentHorizonFinder/AHDerivData.hpp @@ -14,7 +14,7 @@ #define DDWIDTH 6 //! class to store 1st and 2nd derivatives of 'F' -struct AHDeriv +struct AHDerivData { double duF; double duduF; @@ -38,10 +38,10 @@ struct AHDeriv double dvdv_weights[DDWIDTH]; #endif - AHDeriv() + AHDerivData() { - // force all (double) elements of AHDeriv to be 0 - memset(this, 0, sizeof(AHDeriv)); + // force all (double) elements of AHDerivData to be 0 + memset(this, 0, sizeof(AHDerivData)); } }; diff --git a/Source/ApparentHorizonFinder/AHFunctionDefault.hpp b/Source/ApparentHorizonFinder/AHFunctionDefault.hpp index 1fd0608d4..e99f20f8f 100644 --- a/Source/ApparentHorizonFinder/AHFunctionDefault.hpp +++ b/Source/ApparentHorizonFinder/AHFunctionDefault.hpp @@ -6,8 +6,8 @@ #ifndef _AHFUNCTIONDEFAULT_HPP_ #define _AHFUNCTIONDEFAULT_HPP_ -#include "AHData.hpp" -#include "AHDeriv.hpp" +#include "AHVarsData.hpp" +#include "AHDerivData.hpp" #include "AHGeometryData.hpp" #include "AlwaysInline.hpp" #include "GRParmParse.hpp" @@ -64,7 +64,7 @@ struct AHFunctionDefault // not defined by default Tensor<1, double> get_level_function_derivative(const AHGeometryData &geo_data, - const AHDeriv &deriv) const + const AHDerivData &deriv) const { return {0.}; } @@ -74,7 +74,7 @@ struct AHFunctionDefault } // not defined by default Tensor<2, double> get_level_function_2nd_covariant_derivative( - const AHGeometryData &geo_data, const AHDeriv &deriv, + const AHGeometryData &geo_data, const AHDerivData &deriv, const Tensor<1, double> &s_L) const { return {0.}; @@ -82,12 +82,12 @@ struct AHFunctionDefault // WHAT TO ADD TO YOUR OWN FUNCTIONS: // some constructor with these arguments: - // AHFunctionDefault(const AHData &a_data, + // AHFunctionDefault(const AHVarsData &a_data, // const Tensor<1, double> &a_coords, // const Tensor<1, double> &a_coords_cartesian); // and some 'get' - // double get(const AHGeometryData &geo_data, const AHDeriv &deriv, + // double get(const AHGeometryData &geo_data, const AHDerivData &deriv, // const params &a_params) const; }; diff --git a/Source/ApparentHorizonFinder/AHFunctions.hpp b/Source/ApparentHorizonFinder/AHFunctions.hpp index b9b390ade..90b2c5ee1 100644 --- a/Source/ApparentHorizonFinder/AHFunctions.hpp +++ b/Source/ApparentHorizonFinder/AHFunctions.hpp @@ -113,7 +113,7 @@ struct ExpansionFunction : AHFunctionDefault #endif } - ExpansionFunction(const AHData &a_data, + ExpansionFunction(const AHVarsData &a_data, const Tensor<1, double> &a_coords, const Tensor<1, double> &a_coords_cartesian) { @@ -227,7 +227,7 @@ struct ExpansionFunction : AHFunctionDefault } }; - double get(const AHGeometryData &geo_data, const AHDeriv &deriv, + double get(const AHGeometryData &geo_data, const AHDerivData &deriv, const params &a_params) const { Tensor<1, double> s_L = get_level_function_derivative(geo_data, deriv); @@ -270,7 +270,7 @@ struct ExpansionFunction : AHFunctionDefault // extra stuff: Tensor<1, double> get_level_function_derivative(const AHGeometryData &geo_data, - const AHDeriv &deriv) const + const AHDerivData &deriv) const { // calculate D_a L of 6.7.12 of Alcubierre for some level function // L picking L = f - F(u,v) @@ -313,7 +313,7 @@ struct ExpansionFunction : AHFunctionDefault } Tensor<2, double> get_level_function_2nd_covariant_derivative( - const AHGeometryData &geo_data, const AHDeriv &deriv, + const AHGeometryData &geo_data, const AHDerivData &deriv, const Tensor<1, double> &s_L) const { // calculates D_a D_b L, required for 6.7.13 of Alcubierre @@ -361,7 +361,7 @@ struct ChiContourFunction : AHFunctionDefault static int vars_min() { return c_chi; } static int vars_max() { return c_chi; } - ChiContourFunction(const AHData &a_data, + ChiContourFunction(const AHVarsData &a_data, const Tensor<1, double> &a_coords, const Tensor<1, double> &a_coords_cartesian) { @@ -379,7 +379,7 @@ struct ChiContourFunction : AHFunctionDefault } }; - double get(const AHGeometryData &geo_data, const AHDeriv &deriv, + double get(const AHGeometryData &geo_data, const AHDerivData &deriv, const params &a_params) const { return chi - a_params.look_for_chi_contour; diff --git a/Source/ApparentHorizonFinder/AHInterpolation.hpp b/Source/ApparentHorizonFinder/AHInterpolation.hpp index 5ba956993..58551cc0a 100644 --- a/Source/ApparentHorizonFinder/AHInterpolation.hpp +++ b/Source/ApparentHorizonFinder/AHInterpolation.hpp @@ -6,7 +6,7 @@ #ifndef _AHINTERPOLATION_HPP_ #define _AHINTERPOLATION_HPP_ -#include "AHData.hpp" +#include "AHVarsData.hpp" #include "AHGeometryData.hpp" #include "AMRInterpolator.hpp" #include "Lagrange.hpp" @@ -38,8 +38,8 @@ template class AHInterpolation_t std::vector m_z; #endif - AHData> m_data; - AHData> m_extra; + AHVarsData> m_data; + AHVarsData> m_extra; std::array m_coord_min, m_coord_max; //!< maximum and minimum of level 0 box, used in @@ -81,7 +81,7 @@ template class AHInterpolation_t const AHGeometryData get_geometry_data(int idx) const; const Tensor<1, double> get_cartesian_coords(int idx) const; const Tensor<1, double> get_coords(int idx) const; - const AHData get_data(int idx) const; + const AHVarsData get_data(int idx) const; //! verify origin +- initial guess is inside the grid bool is_in_grid(const std::array &a_origin, @@ -115,7 +115,7 @@ template class AHInterpolation_t void interpolate_extra_vars( const std::map> &extra_vars); - const AHData get_extra_data(int idx) const; + const AHVarsData get_extra_data(int idx) const; }; #include "AHInterpolation.impl.hpp" diff --git a/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp b/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp index f1ab2138b..0905cb0d5 100644 --- a/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp +++ b/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp @@ -473,10 +473,10 @@ AHInterpolation_t::get_coords(int idx) const } template -const AHData +const AHVarsData AHInterpolation_t::get_data(int idx) const { - return get_AHData_idx(idx, m_data); + return get_AHVarsData_idx(idx, m_data); } template @@ -516,10 +516,10 @@ void AHInterpolation_t::interpolate_extra_vars( } template -const AHData +const AHVarsData AHInterpolation_t::get_extra_data(int idx) const { - return get_AHData_idx(idx, m_extra); + return get_AHVarsData_idx(idx, m_extra); } #endif // _AHINTERPOLATION_IMPL_HPP_ \ No newline at end of file diff --git a/Source/ApparentHorizonFinder/AHData.hpp b/Source/ApparentHorizonFinder/AHVarsData.hpp similarity index 90% rename from Source/ApparentHorizonFinder/AHData.hpp rename to Source/ApparentHorizonFinder/AHVarsData.hpp index 8585bfa2e..2a4481b88 100644 --- a/Source/ApparentHorizonFinder/AHData.hpp +++ b/Source/ApparentHorizonFinder/AHVarsData.hpp @@ -13,9 +13,7 @@ #include // memset -// The d prefix refers to partial derivatives wrt Cartesian coordinates - -template struct AHData +template struct AHVarsData { // vars user might want to export std::map vars; @@ -60,10 +58,10 @@ template struct AHData }; template -AHData -get_AHData_idx(int idx, const AHData> &a_data) +AHVarsData +get_AHVarsData_idx(int idx, const AHVarsData> &a_data) { - AHData data; + AHVarsData data; for (auto &vec : a_data.vars) data.vars[vec.first] = (vec.second[idx]); diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp index 32d6563a9..52ea5619b 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp @@ -1296,7 +1296,7 @@ ApparentHorizon::calculate_spin_dimensionless( { if (u_equator == u) { - AHDeriv deriv = solver.diff(in, u, v); + AHDerivData deriv = solver.diff(in, u, v); const auto geometry_data = solver.m_interp.get_geometry_data(idx); const auto data = solver.m_interp.get_data(idx); @@ -1394,7 +1394,7 @@ ApparentHorizon::calculate_angular_momentum_J() Tensor<1, double> inner_integral = {0.}; for (int u = solver.m_umin; u < solver.m_umax; ++u) { - AHDeriv deriv = solver.diff(in, u, v); + AHDerivData deriv = solver.diff(in, u, v); const auto geometric_data = solver.m_interp.get_geometry_data(idx); const auto data = solver.m_interp.get_data(idx); @@ -1565,7 +1565,7 @@ double ApparentHorizon::calculate_area() #endif } - AHDeriv deriv = solver.diff(in, u + AHDerivData deriv = solver.diff(in, u #if CH_SPACEDIM == 3 , v diff --git a/Source/ApparentHorizonFinder/PETScAHSolver.hpp b/Source/ApparentHorizonFinder/PETScAHSolver.hpp index 2b293192d..9d6042259 100644 --- a/Source/ApparentHorizonFinder/PETScAHSolver.hpp +++ b/Source/ApparentHorizonFinder/PETScAHSolver.hpp @@ -9,7 +9,7 @@ #include // #include -#include "AHDeriv.hpp" +#include "AHDerivData.hpp" #include "AHInterpolation.hpp" #include "AHParams.hpp" @@ -35,7 +35,7 @@ template class PETScAHSolver //! function to calculate 1st and 2nd derivatives of 'in' //! (tipically corresponds to our 'f' function) //! in the 'u' and 'v' directions - AHDeriv diff(const dmda_arr_t in, int u + AHDerivData diff(const dmda_arr_t in, int u #if CH_SPACEDIM == 3 , int v @@ -129,8 +129,8 @@ template class PETScAHSolver Mat m_snes_jac; private: - //! set the default stencils of AHDeriv at position {u,v} - void set_stencils(AHDeriv &out, int u + //! set the default stencils of AHDerivData at position {u,v} + void set_stencils(AHDerivData &out, int u #if CH_SPACEDIM == 3 , int v diff --git a/Source/ApparentHorizonFinder/PETScAHSolver.impl.hpp b/Source/ApparentHorizonFinder/PETScAHSolver.impl.hpp index 29cc8c994..bc8ffc83c 100644 --- a/Source/ApparentHorizonFinder/PETScAHSolver.impl.hpp +++ b/Source/ApparentHorizonFinder/PETScAHSolver.impl.hpp @@ -523,7 +523,7 @@ void PETScAHSolver::restore_dmda_arr_t( } template -void PETScAHSolver::set_stencils(AHDeriv &out, +void PETScAHSolver::set_stencils(AHDerivData &out, int u #if CH_SPACEDIM == 3 , @@ -692,7 +692,7 @@ void PETScAHSolver::set_stencils(AHDeriv &out, } template -AHDeriv PETScAHSolver::diff(const dmda_arr_t in, +AHDerivData PETScAHSolver::diff(const dmda_arr_t in, int u #if CH_SPACEDIM == 3 , @@ -702,7 +702,7 @@ AHDeriv PETScAHSolver::diff(const dmda_arr_t in, { CH_TIME("PETScAHSolver::diff"); - AHDeriv out; + AHDerivData out; set_stencils(out, u #if CH_SPACEDIM == 3 , @@ -828,7 +828,7 @@ void PETScAHSolver::form_function(Vec F, Vec Rhs) double &_out = out[u]; #endif - AHDeriv deriv = diff(in, u + AHDerivData deriv = diff(in, u #if CH_SPACEDIM == 3 , v @@ -922,7 +922,7 @@ void PETScAHSolver::form_jacobian(Vec F, Mat J) MatStencil col[DWIDTH] = {0}; double val[DWIDTH] = {0}; - const AHDeriv deriv = diff(in, u + const AHDerivData deriv = diff(in, u #if CH_SPACEDIM == 3 , v @@ -947,7 +947,7 @@ void PETScAHSolver::form_jacobian(Vec F, Mat J) MatStencil col[DWIDTH] = {0}; double val[DWIDTH] = {0}; - const AHDeriv deriv = diff(in, u, v); + const AHDerivData deriv = diff(in, u, v); for (int b = 0; b < DWIDTH; ++b) { @@ -970,7 +970,7 @@ void PETScAHSolver::form_jacobian(Vec F, Mat J) MatStencil col[NVAL] = {0}; double val[NVAL] = {0}; - const AHDeriv deriv_default = diff(in, u + const AHDerivData deriv_default = diff(in, u #if CH_SPACEDIM == 3 , v @@ -1064,7 +1064,7 @@ double PETScAHSolver::point_jacobian( double in_old = _in; _in += eps; // perturb just the point {u,v} - const AHDeriv deriv = diff(in, u + const AHDerivData deriv = diff(in, u #if CH_SPACEDIM == 3 , v @@ -1087,7 +1087,7 @@ double PETScAHSolver::point_jacobian( double in_old = _in; _in -= eps; // perturb just the point {u,v} - const AHDeriv deriv = diff(in, u + const AHDerivData deriv = diff(in, u #if CH_SPACEDIM == 3 , v From 9ec2bc6efa866d2ad077d19fa9f0e55d9ea453a2 Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Wed, 30 Jun 2021 00:44:04 +0100 Subject: [PATCH 29/92] Added AHInitialGuess Initial guess can now be modified via a user-built class. Defaults to constant initial guess. Also designed initial guess for mergers and for ellipsoid AH (useful with high boosts). Added initial data class IsotropicBoostedBH that can go to high boosts and form highly deformed ellipsoidal AHs where we can test the code TODO: allow user to increase max SNES iterations --- Examples/BinaryBH/params_two_punctures.txt | 2 + Examples/KerrBH/params.txt | 3 +- Examples/KerrBH/params_cheap.txt | 3 +- Examples/SingleBH/Main_SingleBH.cpp | 5 + Examples/SingleBH/SimulationParameters.hpp | 27 +++ Examples/SingleBH/SingleBHLevel.cpp | 26 ++- Examples/SingleBH/params.txt | 32 +-- Source/ApparentHorizonFinder/AHFinder.hpp | 32 ++- .../ApparentHorizonFinder/AHFinder.impl.hpp | 76 ++++--- .../AHFunctionDefault.hpp | 7 +- Source/ApparentHorizonFinder/AHFunctions.hpp | 4 + .../ApparentHorizonFinder/AHInitialGuess.hpp | 186 ++++++++++++++++++ .../ApparentHorizonFinder/AHInterpolation.hpp | 23 +-- .../AHInterpolation.impl.hpp | 70 +------ Source/ApparentHorizonFinder/AHParams.hpp | 9 +- .../ApparentHorizonFinder/ApparentHorizon.hpp | 5 +- .../ApparentHorizon.impl.hpp | 6 +- .../ApparentHorizonFinder/PETScAHSolver.hpp | 17 +- .../PETScAHSolver.impl.hpp | 74 ++++--- Source/BlackHoles/BHAMR.hpp | 2 +- .../GRChomboCore/SimulationParametersBase.hpp | 2 +- .../BlackHoles/IsotropicBoostedBH.hpp | 37 ++++ .../BlackHoles/IsotropicBoostedBH.impl.hpp | 121 ++++++++++++ .../InitialConditions/BlackHoles/SingleBH.hpp | 2 + .../AHTest2DFunction.hpp | 8 +- .../SimulationParameters.hpp | 2 +- .../SimulationParameters.hpp | 2 +- 27 files changed, 578 insertions(+), 205 deletions(-) create mode 100644 Source/ApparentHorizonFinder/AHInitialGuess.hpp create mode 100644 Source/InitialConditions/BlackHoles/IsotropicBoostedBH.hpp create mode 100644 Source/InitialConditions/BlackHoles/IsotropicBoostedBH.impl.hpp diff --git a/Examples/BinaryBH/params_two_punctures.txt b/Examples/BinaryBH/params_two_punctures.txt index 94f82eead..075ec12c5 100644 --- a/Examples/BinaryBH/params_two_punctures.txt +++ b/Examples/BinaryBH/params_two_punctures.txt @@ -209,6 +209,8 @@ AH_num_points_v = 48 # AH_1_initial_guess = 0.3 # AH_2_initial_guess = 0.3 +# AH_expansion_radius_power = 1. + # AH_num_extra_vars = 2 # AH_extra_vars = chi d1_Ham d2_A11 diff --git a/Examples/KerrBH/params.txt b/Examples/KerrBH/params.txt index 08be4b59c..8200437e8 100644 --- a/Examples/KerrBH/params.txt +++ b/Examples/KerrBH/params.txt @@ -162,7 +162,8 @@ AH_stop_if_max_fails = 1 # AH_max_fails_after_lost = 0 # -1 to never # AH_print_geometry_data = 0 # AH_re_solve_at_restart = 0 -AH_verbose = 2 +# AH_verbose = 1 +# AH_expansion_radius_power = 1. # AH_initial_guess = 0.5 diff --git a/Examples/KerrBH/params_cheap.txt b/Examples/KerrBH/params_cheap.txt index 6836268ef..961bb26c4 100644 --- a/Examples/KerrBH/params_cheap.txt +++ b/Examples/KerrBH/params_cheap.txt @@ -165,7 +165,8 @@ AH_predict_origin = false # AH_max_fails_after_lost = 0 # -1 to never # AH_print_geometry_data = 0 # AH_re_solve_at_restart = 0 -AH_verbose = 2 +# AH_verbose = 1 +# AH_expansion_radius_power = 1. # AH_initial_guess = 0.5 diff --git a/Examples/SingleBH/Main_SingleBH.cpp b/Examples/SingleBH/Main_SingleBH.cpp index f36dcd2cd..5bb1e3acd 100644 --- a/Examples/SingleBH/Main_SingleBH.cpp +++ b/Examples/SingleBH/Main_SingleBH.cpp @@ -43,8 +43,13 @@ int runGRChombo(int argc, char *argv[]) if (sim_params.AH_activate) { AHSurfaceGeometry sph(sim_params.bh_params.center); +#ifndef USE_ISOTROPIC_BOOSTED_BH bh_amr.m_ah_finder.add_ah(sph, sim_params.AH_initial_guess, sim_params.AH_params); +#else + bh_amr.m_ah_finder.add_ah(sph, sim_params.AH_initial_guess_elipsoid, + sim_params.AH_params); +#endif } #endif diff --git a/Examples/SingleBH/SimulationParameters.hpp b/Examples/SingleBH/SimulationParameters.hpp index 07ea18a66..ad24645f5 100644 --- a/Examples/SingleBH/SimulationParameters.hpp +++ b/Examples/SingleBH/SimulationParameters.hpp @@ -13,6 +13,15 @@ // Problem specific includes: #include "BoostedBH.hpp" +// define to use IsotropicBoostedBH initial data instead of SingleBH ID +#define USE_ISOTROPIC_BOOSTED_BH + +#ifdef USE_ISOTROPIC_BOOSTED_BH +#ifdef USE_AHFINDER +#include "AHInitialGuess.hpp" +#endif +#endif + class SimulationParameters : public SimulationParametersBase { public: @@ -38,6 +47,21 @@ class SimulationParameters : public SimulationParametersBase #ifdef USE_AHFINDER pp.load("AH_initial_guess", AH_initial_guess, 0.5 * bh_params.mass); +#ifdef USE_ISOTROPIC_BOOSTED_BH + double r_x = AH_initial_guess, r_y = AH_initial_guess, + r_z = AH_initial_guess; + + bool ah_use_ellipsoid; + pp.load("AH_use_ellipsoid", ah_use_ellipsoid, true); + if (ah_use_ellipsoid) + { + double vel = bh_params.momentum[0]; + double contraction = sqrt(1. - vel * vel); + r_x *= contraction; + } + + AH_initial_guess_elipsoid.set_params(r_x, r_y, r_z); +#endif #endif } @@ -49,6 +73,9 @@ class SimulationParameters : public SimulationParametersBase #ifdef USE_AHFINDER double AH_initial_guess; +#ifdef USE_ISOTROPIC_BOOSTED_BH + AHInitialGuessElipsoid AH_initial_guess_elipsoid; +#endif #endif }; diff --git a/Examples/SingleBH/SingleBHLevel.cpp b/Examples/SingleBH/SingleBHLevel.cpp index 83abcf5a9..d7f512cc9 100644 --- a/Examples/SingleBH/SingleBHLevel.cpp +++ b/Examples/SingleBH/SingleBHLevel.cpp @@ -13,10 +13,19 @@ #include "NewConstraints.hpp" #include "PositiveChiAndAlpha.hpp" #include "SetValue.hpp" -#include "SingleBH.hpp" #include "SixthOrderDerivatives.hpp" #include "TraceARemoval.hpp" +// where 'USE_ISOTROPIC_BOOSTED_BH' may be defined +#include "SimulationParameters.hpp" + +#ifndef USE_ISOTROPIC_BOOSTED_BH +#include "SingleBH.hpp" +#else +#include "GammaCalculator.hpp" +#include "IsotropicBoostedBH.hpp" +#endif + void SingleBHLevel::specificAdvance() { // Enforce the trace free A_ij condition and positive chi and alpha @@ -36,13 +45,24 @@ void SingleBHLevel::initialData() if (m_verbosity) pout() << "SingleBHLevel::initialData " << m_level << endl; +#ifndef USE_ISOTROPIC_BOOSTED_BH // Set up the compute class for the SingleBH initial data - SingleBH single(m_p.bh_params, m_dx); + SingleBH bh(m_p.bh_params, m_dx); +#else + // Set up the compute class for the SingleBH initial data + IsotropicBoostedBH bh(m_p.bh_params, m_dx); +#endif // First set everything to zero (to avoid undefinded values in constraints) // then calculate initial data - BoxLoops::loop(make_compute_pack(SetValue(0.), single), m_state_new, + BoxLoops::loop(make_compute_pack(SetValue(0.), bh), m_state_new, m_state_new, INCLUDE_GHOST_CELLS); + +#ifdef USE_ISOTROPIC_BOOSTED_BH + fillAllGhosts(); + BoxLoops::loop(GammaCalculator(m_dx), m_state_new, m_state_new, + EXCLUDE_GHOST_CELLS); +#endif } void SingleBHLevel::prePlotLevel() diff --git a/Examples/SingleBH/params.txt b/Examples/SingleBH/params.txt index 3d84c0208..7dc781964 100644 --- a/Examples/SingleBH/params.txt +++ b/Examples/SingleBH/params.txt @@ -17,8 +17,8 @@ checkpoint_interval = 2 # set to 0 to turn off plot files (except at t=0 and t=stop_time) # set to -1 to never ever print plotfiles plot_interval = 1 -num_plot_vars = 1 -plot_vars = chi +num_plot_vars = 2 +plot_vars = chi Ham # subpaths - specific directories for hdf5, pout, extraction data # (these are created at runtime) @@ -36,9 +36,9 @@ print_progress_only_to_rank_0 = 1 ################################################# # Initial Data parameters -mass = 1.0 -offset = 0 0 0 -momentum = 0. -0.1 0.0 +mass = 1. +offset = 0. 0. 0. +momentum = 0.5 0. 0. # speed if using the IsotropicBoostBH ID ################################################# # Grid parameters @@ -50,8 +50,8 @@ momentum = 0. -0.1 0.0 # NB - if you have a non-cubic grid, you can specify 'N1' or 'N1_full', # 'N2' or 'N2_full' and 'N3' or 'N3_full' ( then dx_coarsest = L/N(max) ) # NB - the N values need to be multiples of the block_factor -N_full = 32 -L_full = 16 +N_full = 128 +L_full = 64 # Maximum number of times you can regrid above coarsest level max_level = 4 # There are (max_level+1) grids, so min is zero @@ -61,7 +61,7 @@ max_level = 4 # There are (max_level+1) grids, so min is zero # Generally you do not need to regrid frequently on every level # Level Regridding: 0 1 2 3 4 regrid_interval = 1 2 4 8 16 -regrid_threshold = 0.05 +regrid_threshold = 0.03 # Max and min box sizes max_box_size = 16 @@ -82,7 +82,7 @@ isPeriodic = 0 0 0 # 0 = static, 1 = sommerfeld, 2 = reflective # (see BoundaryConditions.hpp for details) hi_boundary = 1 1 1 -lo_boundary = 1 1 1 +lo_boundary = 1 2 2 # if reflective boundaries selected, must set # parity of all vars (in order given by UserVariables.hpp) @@ -148,13 +148,13 @@ sigma = 1.0 AH_activate = 1 AH_num_ranks = 4 -AH_num_points_u = 11 -AH_num_points_v = 20 +AH_num_points_u = 41 +AH_num_points_v = 40 AH_solve_interval = 1 AH_print_interval = 1 -# AH_track_center = true -# AH_predict_origin = true -# AH_level_to_run = 0 +# AH_track_center = 1 +# AH_predict_origin = 1 +AH_level_to_run = 0 # AH_allow_re_attempt = 0 # AH_stop_if_max_fails = 0 # AH_start_time = 0. @@ -162,9 +162,11 @@ AH_print_interval = 1 # AH_max_fails_after_lost = 0 # -1 to never # AH_print_geometry_data = 0 # AH_re_solve_at_restart = 0 -AH_verbose = 3 +# AH_verbose = 1 +# AH_expansion_radius_power = 1. # AH_initial_guess = 0.5 +# AH_use_ellipsoid = 1 # for IsotropicBoostedBH ID # look_for_chi_contour = 0.2 diff --git a/Source/ApparentHorizonFinder/AHFinder.hpp b/Source/ApparentHorizonFinder/AHFinder.hpp index c0cfe8ab5..6ea2d869d 100644 --- a/Source/ApparentHorizonFinder/AHFinder.hpp +++ b/Source/ApparentHorizonFinder/AHFinder.hpp @@ -42,6 +42,13 @@ #include "AHFunctions.hpp" #define AHFunction ExpansionFunction #endif + +#include "AHInitialGuess.hpp" +// // default to constant initial guess +// #ifndef AHInitialGuess +// #include "AHInitialGuess.hpp" +// #define AHInitialGuess AHInitialGuessConstant +// #endif ///////////////////////////////////////////////////////////////////////////// /* @@ -77,10 +84,12 @@ is more senstitive). */ //! Class to manage AHs and its mergers + control PETSc MPI sub-communicator -template class AHFinder +template +class AHFinder { using AHInterpolation = AHInterpolation_t; - using AHParams = AHParams_t; + using AHParams = AHParams_t; public: AHFinder(){}; @@ -94,6 +103,23 @@ template class AHFinder } //! returns the index of the AH in m_apparent_horizons + template + int add_ah(const SurfaceGeometry &a_coord_system, + AHInitialGuess + a_initial_guess, //!< Initial guess for radius (or whatever + //!< coordinate you're solving for) + const AHParams &a_params, //!< set of AH parameters + bool solve_first_step = true //!< whether or not to solve if t=0 + ); + //! returns the index of the AH in m_apparent_horizons + int add_ah(const SurfaceGeometry &a_coord_system, + AHInitialGuessPtr + a_initial_guess, //!< Initial guess for radius (or whatever + //!< coordinate you're solving for) + const AHParams &a_params, //!< set of AH parameters + bool solve_first_step = true //!< whether or not to solve if t=0 + ); + //! backward-compatibility: forces AHInitialGuess = AHInitialGuessConstant int add_ah(const SurfaceGeometry &a_coord_system, double a_initial_guess, //!< Initial guess for radius (or whatever @@ -125,7 +151,7 @@ template class AHFinder private: //! returns false if 'parent' AHs are too far //! sets the initial guess and the origin for the merger - bool solve_merger(int ah1, int ah2, double &initial_guess_merger, + bool solve_merger(int ah1, int ah2, AHInitialGuessPtr initial_guess_merger, std::array &origin_merged); private: diff --git a/Source/ApparentHorizonFinder/AHFinder.impl.hpp b/Source/ApparentHorizonFinder/AHFinder.impl.hpp index adfe4448d..ddb9cde78 100644 --- a/Source/ApparentHorizonFinder/AHFinder.impl.hpp +++ b/Source/ApparentHorizonFinder/AHFinder.impl.hpp @@ -22,8 +22,19 @@ AHFinder::~AHFinder() } template +template int AHFinder::add_ah( - const SurfaceGeometry &a_coord_system, double a_initial_guess, + const SurfaceGeometry &a_coord_system, AHInitialGuess a_initial_guess, + const AHParams &a_params, bool solve_first_step) +{ + return add_ah(a_coord_system, + AHInitialGuessPtr(new AHInitialGuess(a_initial_guess)), + a_params, solve_first_step); +} + +template +int AHFinder::add_ah( + const SurfaceGeometry &a_coord_system, AHInitialGuessPtr a_initial_guess, const AHParams &a_params, bool solve_first_step) { PETScCommunicator::initialize(a_params.num_ranks); @@ -50,6 +61,17 @@ int AHFinder::add_ah( return num_ah; } +template +int AHFinder::add_ah( + const SurfaceGeometry &a_coord_system, double a_initial_guess, + const AHParams &a_params, bool solve_first_step) +{ + return add_ah( + a_coord_system, + AHInitialGuessPtr(new AHInitialGuessConstant(a_initial_guess)), + a_params, solve_first_step); +} + template int AHFinder::add_ah_merger( int ah1, int ah2, const AHParams &a_params) @@ -61,7 +83,7 @@ int AHFinder::add_ah_merger( CH_assert(ah2 >= 0 && ah2 < num_ah); CH_assert(ah1 != ah2); - double initial_guess_merger; + AHInitialGuessPtr initial_guess_merger; std::array origin_merger; bool do_solve = solve_merger(ah1, ah2, initial_guess_merger, origin_merger); @@ -175,7 +197,7 @@ void AHFinder::solve(double a_dt, double a_time, continue; } - double initial_guess_merger; + AHInitialGuessPtr initial_guess_merger; std::array center_merger; bool do_solve = solve_merger(pair.first, pair.second, initial_guess_merger, @@ -200,8 +222,6 @@ void AHFinder::solve(double a_dt, double a_time, ah_solved[pair.second] == SOLVED) { m_apparent_horizons[i]->set_origin(center_merger); - // m_apparent_horizons[i]->set_initial_guess( - // initial_guess_merger); // should still be the same } } @@ -246,7 +266,7 @@ bool AHFinder::need_diagnostics( template bool AHFinder::solve_merger( - int ah1, int ah2, double &initial_guess_merger, + int ah1, int ah2, AHInitialGuessPtr initial_guess_merger, std::array ¢er_merger) { // SKIP if 'parents' not yet close enough @@ -258,36 +278,14 @@ bool AHFinder::solve_merger( auto initial_guess1 = AH1->get_petsc_solver().get_initial_guess(); auto initial_guess2 = AH2->get_petsc_solver().get_initial_guess(); - if (m_merger_pairs[ah1].first >= 0) - { - double merger_pre_factor = - std::max(m_apparent_horizons[m_merger_pairs[ah1].first] - ->m_params.merger_pre_factor, - m_apparent_horizons[m_merger_pairs[ah1].second] - ->m_params.merger_pre_factor); - if (merger_pre_factor > - 0.) // undo the factor is this a merger from a merger - initial_guess1 /= (merger_pre_factor * 4.); - } - if (m_merger_pairs[ah2].first >= 0) - { - double merger_pre_factor = - std::max(m_apparent_horizons[m_merger_pairs[ah2].first] - ->m_params.merger_pre_factor, - m_apparent_horizons[m_merger_pairs[ah2].second] - ->m_params.merger_pre_factor); - if (merger_pre_factor > - 0.) // undo the factor is this a merger from a merger - initial_guess2 /= (merger_pre_factor * 4.); - } - - double initial_guess_sum = (initial_guess1 + initial_guess2); - - // some tests revealed it was about 1.2 * initial_guess_sum, but 1.5 to - // ensure we don't catch the inner AH double merger_pre_factor = std::max(AH1->m_params.merger_pre_factor, AH2->m_params.merger_pre_factor); - initial_guess_merger = merger_pre_factor * 4. * initial_guess_sum; + double merger_search_factor = std::max(AH1->m_params.merger_search_factor, + AH2->m_params.merger_search_factor); + + initial_guess_merger = AHInitialGuessPtr( + new AHInitialGuessMerger(initial_guess1, initial_guess2, + merger_pre_factor, merger_search_factor)); // update center of merged, otherwise it does it by // itself in solve @@ -309,10 +307,7 @@ bool AHFinder::solve_merger( #endif ); - double merger_search_factor = std::max(AH1->m_params.merger_search_factor, - AH2->m_params.merger_search_factor); - - double min_distance = merger_search_factor * 4. * initial_guess_sum; + double min_distance = initial_guess_merger->get_merger_min_distance(); bool do_solve = false; @@ -322,11 +317,6 @@ bool AHFinder::solve_merger( { do_solve = merger_search_factor <= 0. || distance <= min_distance; - if (do_solve) - // make sure twice the radius of the guess is bigger than AH - // distance - CH_assert(min_distance <= 2. * initial_guess_merger); - if (AH1->m_params.verbose > AHParams::NONE) { pout() << "BHs #" << ah1 << " and #" << ah2 diff --git a/Source/ApparentHorizonFinder/AHFunctionDefault.hpp b/Source/ApparentHorizonFinder/AHFunctionDefault.hpp index e99f20f8f..d717ab8ac 100644 --- a/Source/ApparentHorizonFinder/AHFunctionDefault.hpp +++ b/Source/ApparentHorizonFinder/AHFunctionDefault.hpp @@ -6,9 +6,9 @@ #ifndef _AHFUNCTIONDEFAULT_HPP_ #define _AHFUNCTIONDEFAULT_HPP_ -#include "AHVarsData.hpp" #include "AHDerivData.hpp" #include "AHGeometryData.hpp" +#include "AHVarsData.hpp" #include "AlwaysInline.hpp" #include "GRParmParse.hpp" #include "Tensor.hpp" @@ -46,6 +46,11 @@ struct AHFunctionDefault FOR1(i) { g[i][i] = 1.; } return g; } + ALWAYS_INLINE const Tensor<2, double> get_inverse_metric() const + { + // cartesian flat metric + return get_metric(); + } ALWAYS_INLINE const Tensor<2, double> get_extrinsic_curvature() const { return {0.}; diff --git a/Source/ApparentHorizonFinder/AHFunctions.hpp b/Source/ApparentHorizonFinder/AHFunctions.hpp index 90b2c5ee1..b515486d7 100644 --- a/Source/ApparentHorizonFinder/AHFunctions.hpp +++ b/Source/ApparentHorizonFinder/AHFunctions.hpp @@ -66,6 +66,10 @@ struct ExpansionFunction : AHFunctionDefault static ALWAYS_INLINE int d1_vars_max() { return c_K - 1; } ALWAYS_INLINE const Tensor<2, double> get_metric() const { return g; } + ALWAYS_INLINE const Tensor<2, double> get_inverse_metric() const + { + return g_UU; + } ALWAYS_INLINE const Tensor<2, double> get_extrinsic_curvature() const { return K; diff --git a/Source/ApparentHorizonFinder/AHInitialGuess.hpp b/Source/ApparentHorizonFinder/AHInitialGuess.hpp new file mode 100644 index 000000000..ed2e6bc5a --- /dev/null +++ b/Source/ApparentHorizonFinder/AHInitialGuess.hpp @@ -0,0 +1,186 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef _AHINITIALGUESS_HPP_ +#define _AHINITIALGUESS_HPP_ + +#include "AlwaysInline.hpp" + +#include // sin, cos, sqrt +#include // std::shared_ptr + +class AHInitialGuessDefault +{ + public: +#if CH_SPACEDIM == 3 + virtual double get(double u, double v) const = 0; + inline virtual double get_merger_contribution(double u, double v) const + { + return get(u, v); + }; + inline virtual double get_merger_min_distance() const + { + return get(0., 0.); + }; +#elif CH_SPACEDIM == 2 + virtual double get(double u) const = 0; + inline virtual double get_merger_contribution(double u) const + { + return get(u); + }; + inline virtual double get_merger_min_distance() const { return get(0.); }; +#endif +}; + +using AHInitialGuessPtr = std::shared_ptr; + +///////////////////////////////////////////////////////// +// Predefined initial guess functions +///////////////////////////////////////////////////////// + +class AHInitialGuessConstant : public AHInitialGuessDefault +{ + public: + double m_initial_guess; + + AHInitialGuessConstant() {} + AHInitialGuessConstant(double a_initial_guess) + { + set_params(a_initial_guess); + } + + void set_params(double a_initial_guess) + { + m_initial_guess = a_initial_guess; + pout() << "Setting Initial Guess to f=" << m_initial_guess << std::endl; + } + +#if CH_SPACEDIM == 3 + ALWAYS_INLINE double get(double u, double v) const + { + return m_initial_guess; + } +#elif CH_SPACEDIM == 2 + ALWAYS_INLINE double get(double u) const { return m_initial_guess; } +#endif +}; + +class AHInitialGuessMerger : public AHInitialGuessDefault +{ + public: + const AHInitialGuessPtr m_ah1, m_ah2; + + double m_merger_search_factor; // see note above (default is 1) + double m_merger_pre_factor; // see notes in AHParams + + AHInitialGuessMerger(const AHInitialGuessPtr a_ah1, + const AHInitialGuessPtr a_ah2, + double a_merger_pre_factor, + double a_merger_search_factor) + : m_ah1(a_ah1), m_ah2(a_ah2), m_merger_pre_factor(a_merger_pre_factor), + m_merger_search_factor(a_merger_search_factor) + { + } + + inline virtual double get_merger_min_distance() const override + { + return m_merger_search_factor * 4. * + (m_ah1->get_merger_min_distance() + + m_ah2->get_merger_min_distance()); + }; + +#if CH_SPACEDIM == 3 + ALWAYS_INLINE double get(double u, double v) const + { + return m_merger_pre_factor * 4. * get_merger_contribution(u, v); + } + ALWAYS_INLINE double get_merger_contribution(double u, + double v) const override + { + return m_ah1->get_merger_contribution(u, v) + + m_ah2->get_merger_contribution(u, v); + } +#elif CH_SPACEDIM == 2 + ALWAYS_INLINE double get(double u) const + { + return m_merger_pre_factor * 4. * get_merger_contribution(u); + } + ALWAYS_INLINE double get_merger_contribution(double u) const override + { + return m_ah1->get_merger_contribution(u) + + m_ah2->get_merger_contribution(u); + } +#endif +}; + +// ellipsoid aligned with one of the axis +class AHInitialGuessElipsoid : public AHInitialGuessDefault +{ + public: + double m_radius_x, m_radius_y; +#if CH_SPACEDIM == 3 + double m_radius_z; +#endif + + AHInitialGuessElipsoid() {} + +#if CH_SPACEDIM == 3 + AHInitialGuessElipsoid(double a_radius_x, double a_radius_y, + double a_radius_z) + { + set_params(a_radius_x, a_radius_y, a_radius_z); + } + + void set_params(double a_radius_x, double a_radius_y, double a_radius_z) + { + m_radius_x = a_radius_x; + m_radius_y = a_radius_y; + m_radius_z = a_radius_z; + pout() << "Setting Initial Guess to ellipsoid=(" << m_radius_x << ", " + << m_radius_y << ", " << m_radius_z << ")" << std::endl; + } + + ALWAYS_INLINE double get(double u, double v) const + { + double sin_u = sin(u); + double cos_u = cos(u); + double sin_v = sin(v); + double cos_v = cos(v); + double radius = + 1. / + sqrt(sin_u * sin_u * cos_v * cos_v / (m_radius_x * m_radius_x) + + sin_u * sin_u * sin_v * sin_v / (m_radius_y * m_radius_y) + + cos_u * cos_u / (m_radius_z * m_radius_z)); + // std::cout << u << "|" << v << "|" << radius << std::endl; + return radius; + } +#elif CH_SPACEDIM == 2 + AHInitialGuessElipsoid(double a_radius_x, double a_radius_y) + { + set_params(a_radius_x, a_radius_y); + } + + void set_params(double a_radius_x, double a_radius_y) + { + m_radius_x = a_radius_x; + m_radius_y = a_radius_y; + pout() << "Setting Initial Guess to ellipsoid=(" << m_radius_x << ", " + << m_radius_y << ")" << std::endl; + } + + ALWAYS_INLINE double get(double u) const + { + double sin_u = sin(u); + double cos_u = cos(u); + // u = 0 corresponds to (x,y)=(-1,0) + // u = pi/2 corresponds to (x,y)=(0,1) + // u = pi corresponds to (x,y)=(1,0) + return 1. / sqrt(cos_u * cos_u / (m_radius_x * m_radius_x) + + sin_u * sin_u / (m_radius_y * m_radius_y)); + } +#endif +}; + +#endif /* _AHINITIALGUESS_HPP_ */ diff --git a/Source/ApparentHorizonFinder/AHInterpolation.hpp b/Source/ApparentHorizonFinder/AHInterpolation.hpp index 58551cc0a..1787ace59 100644 --- a/Source/ApparentHorizonFinder/AHInterpolation.hpp +++ b/Source/ApparentHorizonFinder/AHInterpolation.hpp @@ -6,8 +6,8 @@ #ifndef _AHINTERPOLATION_HPP_ #define _AHINTERPOLATION_HPP_ -#include "AHVarsData.hpp" #include "AHGeometryData.hpp" +#include "AHVarsData.hpp" #include "AMRInterpolator.hpp" #include "Lagrange.hpp" @@ -45,15 +45,6 @@ template class AHInterpolation_t m_coord_max; //!< maximum and minimum of level 0 box, used in //!< 'fit_in_grid' - //! when PETSc tried to diverge out of the grid, this doesn't let him do so - //! it forces him to stay on the grid, causing non-convergence - bool fit_in_grid(double &x, double &y -#if CH_SPACEDIM == 3 - , - double &z -#endif - ); - public: AHInterpolation_t(const SurfaceGeometry &a_coordSystem, AMRInterpolator> *a_interpolator); @@ -83,9 +74,15 @@ template class AHInterpolation_t const Tensor<1, double> get_coords(int idx) const; const AHVarsData get_data(int idx) const; - //! verify origin +- initial guess is inside the grid - bool is_in_grid(const std::array &a_origin, - double a_initial_guess); + //! verify point is inside the grid + //! when PETSc tried to diverge out of the grid, this doesn't let him do so + //! it forces him to stay on the grid, causing non-convergence + bool is_in_grid(double &x, double &y +#if CH_SPACEDIM == 3 + , + double &z +#endif + ); //! triplet of functions to be used together in blocks of code that require //! PETSc AND AMRInterpolator to interpolate diff --git a/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp b/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp index 0905cb0d5..9fa3092d7 100644 --- a/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp +++ b/Source/ApparentHorizonFinder/AHInterpolation.impl.hpp @@ -20,7 +20,7 @@ AHInterpolation_t::AHInterpolation_t( { // below: // determine maximum and minimum physical coordinate of the grid - // (so that 'fit_in_grid' knows when PETSc has diverged out of the grid and + // (so that 'is_in_grid' knows when PETSc has diverged out of the grid and // doesn't let him do so, // as it would cause an error in the AMRInterpolator) @@ -110,67 +110,15 @@ void AHInterpolation_t::set_origin( } template -bool AHInterpolation_t::is_in_grid( - const std::array &a_origin, double a_initial_guess) -{ - bool out_of_grid = false; - - double x = a_origin[0]; - double y = a_origin[1]; -#if CH_SPACEDIM == 3 - double z = a_origin[2]; -#endif - -#if CH_SPACEDIM == 3 - out_of_grid |= fit_in_grid(x, y, z); - - x = a_origin[0] - a_initial_guess; - out_of_grid |= fit_in_grid(x, y, z); - - x = a_origin[0] + a_initial_guess; - out_of_grid |= fit_in_grid(x, y, z); - - x = a_origin[0]; - y = a_origin[1] - a_initial_guess; - out_of_grid |= fit_in_grid(x, y, z); - - y = a_origin[1] + a_initial_guess; - out_of_grid |= fit_in_grid(x, y, z); - - y = a_origin[1]; - z = a_origin[2] - a_initial_guess; - out_of_grid |= fit_in_grid(x, y, z); - - z = a_origin[2] + a_initial_guess; - out_of_grid |= fit_in_grid(x, y, z); -#elif CH_SPACEDIM == 2 - out_of_grid |= fit_in_grid(x, y); - /* - x = a_origin[0] - a_initial_guess; - out_of_grid |= fit_in_grid(x, y); - - x = a_origin[0] + a_initial_guess; - out_of_grid |= fit_in_grid(x, y); - */ - y = a_origin[1] - a_initial_guess; - out_of_grid |= fit_in_grid(x, y); - - y = a_origin[1] + a_initial_guess; - out_of_grid |= fit_in_grid(x, y); -#endif - - return out_of_grid; -} -template -bool AHInterpolation_t::fit_in_grid(double &x, - double &y +bool AHInterpolation_t::is_in_grid(double &x, + double &y #if CH_SPACEDIM == 3 - , - double &z + , + double &z #endif ) { - CH_TIME("AHInterpolation::fit_in_grid"); + CH_TIME("AHInterpolation::is_in_grid"); bool out_of_grid = false; @@ -418,10 +366,10 @@ bool AHInterpolation_t::set_coordinates( // don't let PETSc diverge to outside of the grid (this can happen if // there is no BH) - out_of_grid |= fit_in_grid(m_x[i], m_y[i] + out_of_grid |= is_in_grid(m_x[i], m_y[i] #if CH_SPACEDIM == 3 - , - m_z[i] + , + m_z[i] #endif ); } diff --git a/Source/ApparentHorizonFinder/AHParams.hpp b/Source/ApparentHorizonFinder/AHParams.hpp index cd8fcf8ab..767ca0def 100644 --- a/Source/ApparentHorizonFinder/AHParams.hpp +++ b/Source/ApparentHorizonFinder/AHParams.hpp @@ -6,11 +6,12 @@ #ifndef _AHPARAMS_HPP_ #define _AHPARAMS_HPP_ +#include "AHInitialGuess.hpp" #include "ChomboParameters.hpp" #include "GRParmParse.hpp" // prepend with 'AH_' in params file -template struct AHParams_t +template struct AHParams_t { int num_ranks; //!< number of ranks for PETSc sub-communicator (default //!< 0, which is 'all') @@ -99,9 +100,9 @@ template struct AHParams_t void read_params(GRParmParse &pp, const ChomboParameters &a_p); }; -template -void AHParams_t::read_params( - GRParmParse &pp, const ChomboParameters &a_p) +template +void AHParams_t::read_params(GRParmParse &pp, + const ChomboParameters &a_p) { pp.load("AH_num_ranks", num_ranks, 0); // 0 means "all" diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.hpp index dae308df8..00e101176 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.hpp @@ -17,13 +17,14 @@ template class ApparentHorizon { using AHInterpolation = AHInterpolation_t; - using AHParams = AHParams_t; + using AHParams = AHParams_t; public: //! AH that finds the zero of AHFunction ApparentHorizon( const AHInterpolation &a_interp, //!< Geometry class to exchange data - double a_initial_guess, //!< Initial guess for radius (or whatever + const AHInitialGuessPtr + a_initial_guess, //!< Initial guess for radius (or whatever //!< coordinate you're solving for) const AHParams &a_params, //!< set of AH parameters const std::string &a_stats = diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp index 52ea5619b..b3eb08413 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp @@ -23,7 +23,7 @@ template ApparentHorizon::ApparentHorizon( - const AHInterpolation &a_interp, double a_initial_guess, + const AHInterpolation &a_interp, const AHInitialGuessPtr a_initial_guess, const AHParams &a_params, const std::string &a_stats, const std::string &a_coords, bool solve_first_step) : m_params(a_params), @@ -1567,8 +1567,8 @@ double ApparentHorizon::calculate_area() AHDerivData deriv = solver.diff(in, u #if CH_SPACEDIM == 3 - , - v + , + v #endif ); diff --git a/Source/ApparentHorizonFinder/PETScAHSolver.hpp b/Source/ApparentHorizonFinder/PETScAHSolver.hpp index 9d6042259..683955257 100644 --- a/Source/ApparentHorizonFinder/PETScAHSolver.hpp +++ b/Source/ApparentHorizonFinder/PETScAHSolver.hpp @@ -24,11 +24,12 @@ template class PETScAHSolver { public: using AHInterpolation = AHInterpolation_t; - using AHParams = AHParams_t; + using AHParams = AHParams_t; public: //! AH that finds the zero of expansion - PETScAHSolver(const AHInterpolation &a_interp, double a_initial_guess, + PETScAHSolver(const AHInterpolation &a_interp, + const AHInitialGuessPtr a_initial_guess, const AHParams &a_params); ~PETScAHSolver(); @@ -37,8 +38,8 @@ template class PETScAHSolver //! in the 'u' and 'v' directions AHDerivData diff(const dmda_arr_t in, int u #if CH_SPACEDIM == 3 - , - int v + , + int v #endif ); @@ -55,8 +56,7 @@ template class PETScAHSolver // must be called at the end of the function that called 'get_dmda_arr_t' void restore_dmda_arr_t(Vec &localF, dmda_arr_t &in); - double get_initial_guess() const; - void set_initial_guess(double a_initial_guess); + const AHInitialGuessPtr get_initial_guess() const; void reset_initial_guess(); const std::array &get_origin() const; @@ -111,8 +111,9 @@ template class PETScAHSolver // variables private: - double m_initial_guess; //!< initial guess for AH (saved so that it can be - //!< re-used when atempting to solve again) + const AHInitialGuessPtr + m_initial_guess; //!< initial guess for AH (saved so that it can be + //!< re-used when atempting to solve again) const AHParams &m_params; //!< set of AH parameters diff --git a/Source/ApparentHorizonFinder/PETScAHSolver.impl.hpp b/Source/ApparentHorizonFinder/PETScAHSolver.impl.hpp index bc8ffc83c..8ccf9f302 100644 --- a/Source/ApparentHorizonFinder/PETScAHSolver.impl.hpp +++ b/Source/ApparentHorizonFinder/PETScAHSolver.impl.hpp @@ -21,7 +21,7 @@ template PETScAHSolver::PETScAHSolver( - const AHInterpolation &a_interp, double a_initial_guess, + const AHInterpolation &a_interp, const AHInitialGuessPtr a_initial_guess, const AHParams &a_params) : m_initial_guess(a_initial_guess), @@ -417,18 +417,12 @@ void PETScAHSolver::set_origin( } template -double PETScAHSolver::get_initial_guess() const +const AHInitialGuessPtr +PETScAHSolver::get_initial_guess() const { return m_initial_guess; } -template -void PETScAHSolver::set_initial_guess( - double a_initial_guess) -{ - m_initial_guess = a_initial_guess; -} - template void PETScAHSolver::reset_initial_guess() { @@ -439,40 +433,42 @@ void PETScAHSolver::reset_initial_guess() auto origin = get_origin(); - // verify origin +- initial guess is inside the grid - bool out_of_grid = m_interp.is_in_grid(origin, m_initial_guess); - CH_assert(!out_of_grid); - - if (m_params.verbose > AHParams::NONE) - { - pout() << "Setting Initial Guess to f=" << m_initial_guess - << " centered at (" << origin[0] << "," << origin[1] -#if CH_SPACEDIM == 3 - << "," << origin[2] -#endif - << ")" << std::endl; - } - // read PETSc array to 'f' dmda_arr_t f; DMDAVecGetArray(m_dmda, m_snes_soln, &f); + bool out_of_grid = false; + const SurfaceGeometry &coord_system = m_interp.get_coord_system(); + #if CH_SPACEDIM == 3 for (int v = m_vmin; v < m_vmax; ++v) #endif { for (int u = m_umin; u < m_umax; ++u) { + double u_val = m_interp.get_coord_system().u(u, m_num_global_u); #if CH_SPACEDIM == 3 + double v_val = m_interp.get_coord_system().v(v, m_num_global_v); double &f_point = f[v][u]; + f_point = m_initial_guess->get(u_val, v_val); + + double x = coord_system.get_grid_coord(0, f_point, u_val, v_val); + double y = coord_system.get_grid_coord(1, f_point, u_val, v_val); + double z = coord_system.get_grid_coord(2, f_point, u_val, v_val); + out_of_grid |= m_interp.is_in_grid(x, y, z); #else double &f_point = f[u]; -#endif + f_point = m_initial_guess->get(u_val); - f_point = m_initial_guess; + double x = coord_system.get_grid_coord(0, f_point, u_val); + double y = coord_system.get_grid_coord(1, f_point, u_val); + out_of_grid |= m_interp.is_in_grid(x, y); +#endif } } + CH_assert(!out_of_grid); + // write PETSc array back DMDAVecRestoreArray(m_dmda, m_snes_soln, &f); } @@ -692,11 +688,11 @@ void PETScAHSolver::set_stencils(AHDerivData &out, } template -AHDerivData PETScAHSolver::diff(const dmda_arr_t in, - int u +AHDerivData +PETScAHSolver::diff(const dmda_arr_t in, int u #if CH_SPACEDIM == 3 - , - int v + , + int v #endif ) { @@ -830,8 +826,8 @@ void PETScAHSolver::form_function(Vec F, Vec Rhs) AHDerivData deriv = diff(in, u #if CH_SPACEDIM == 3 - , - v + , + v #endif ); @@ -924,8 +920,8 @@ void PETScAHSolver::form_jacobian(Vec F, Mat J) const AHDerivData deriv = diff(in, u #if CH_SPACEDIM == 3 - , - v + , + v #endif ); @@ -972,8 +968,8 @@ void PETScAHSolver::form_jacobian(Vec F, Mat J) const AHDerivData deriv_default = diff(in, u #if CH_SPACEDIM == 3 - , - v + , + v #endif ); @@ -1066,8 +1062,8 @@ double PETScAHSolver::point_jacobian( const AHDerivData deriv = diff(in, u #if CH_SPACEDIM == 3 - , - v + , + v #endif ); @@ -1089,8 +1085,8 @@ double PETScAHSolver::point_jacobian( const AHDerivData deriv = diff(in, u #if CH_SPACEDIM == 3 - , - v + , + v #endif ); diff --git a/Source/BlackHoles/BHAMR.hpp b/Source/BlackHoles/BHAMR.hpp index 43989941f..e19944bc3 100644 --- a/Source/BlackHoles/BHAMR.hpp +++ b/Source/BlackHoles/BHAMR.hpp @@ -28,7 +28,7 @@ class BHAMR : public GRAMR #endif #ifdef USE_AHFINDER - AHFinder m_ah_finder; + AHFinder<> m_ah_finder; #endif BHAMR() {} diff --git a/Source/GRChomboCore/SimulationParametersBase.hpp b/Source/GRChomboCore/SimulationParametersBase.hpp index d238ee9b8..19566db20 100644 --- a/Source/GRChomboCore/SimulationParametersBase.hpp +++ b/Source/GRChomboCore/SimulationParametersBase.hpp @@ -327,7 +327,7 @@ class SimulationParametersBase : public ChomboParameters #ifdef USE_AHFINDER bool AH_activate; - AHParams_t AH_params; + AHParams_t AH_params; #endif }; diff --git a/Source/InitialConditions/BlackHoles/IsotropicBoostedBH.hpp b/Source/InitialConditions/BlackHoles/IsotropicBoostedBH.hpp new file mode 100644 index 000000000..4a38ef7bd --- /dev/null +++ b/Source/InitialConditions/BlackHoles/IsotropicBoostedBH.hpp @@ -0,0 +1,37 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#ifndef ISOTROPICBOOSTEDBH_HPP_ +#define ISOTROPICBOOSTEDBH_HPP_ + +#include "Cell.hpp" + +// This is a simple class from the 2D code that was useful to test the AHFinder +// for high boosts, as it produces highly deformed (ellipsoidal) AHs +// For simplicity, it only boosts the BH in the 'x' direction of the momentum +// (so only bh_params.momentum[0] matters) +// Note: the momentum parameter of BoostedBH::params_t in this class is used as +// velocity, not momentum, and should be between ]-1,1[ +class IsotropicBoostedBH +{ + + public: + //! The constructor + IsotropicBoostedBH(BoostedBH::params_t a_bh_params, double a_dx) + : m_bh_params(a_bh_params), m_dx(a_dx) + { + } + + //! Function to compute the value of all the initial vars on the grid + template void compute(Cell current_cell) const; + + protected: + BoostedBH::params_t m_bh_params; + double m_dx; +}; + +#include "IsotropicBoostedBH.impl.hpp" + +#endif /* ISOTROPICBOOSTEDBH_HPP_ */ diff --git a/Source/InitialConditions/BlackHoles/IsotropicBoostedBH.impl.hpp b/Source/InitialConditions/BlackHoles/IsotropicBoostedBH.impl.hpp new file mode 100644 index 000000000..fb8a1e7e1 --- /dev/null +++ b/Source/InitialConditions/BlackHoles/IsotropicBoostedBH.impl.hpp @@ -0,0 +1,121 @@ +/* GRChombo + * Copyright 2012 The GRChombo collaboration. + * Please refer to LICENSE in GRChombo's root directory. + */ + +#if !defined(ISOTROPICBOOSTEDBH_HPP_) +#error "This file should only be included through IsotropicBoostedBH.hpp" +#endif + +#ifndef ISOTROPICBOOSTEDBH_IMPL_HPP_ +#define ISOTROPICBOOSTEDBH_IMPL_HPP_ + +#if CH_SPACEDIM == 3 +#include "CCZ4Vars.hpp" +#elif CH_SPACEDIM == 2 +#include "CCZ4CartoonVars.hpp" +#endif + +#include "Coordinates.hpp" +#include "Tensor.hpp" + +// Compute the value of the initial vars on the grid +template +void IsotropicBoostedBH::compute(Cell current_cell) const +{ + Coordinates coords(current_cell, m_dx, m_bh_params.center); + + const double mass = m_bh_params.mass; + const double vel = m_bh_params.momentum[0]; + const double gamma = 1. / sqrt(1. - vel * vel); + + // coords + data_t x = coords.x * gamma; + double y = coords.y; +#if CH_SPACEDIM == 3 + double z = coords.z; +#elif CH_SPACEDIM == 2 + double z = 0; +#endif + data_t r = sqrt(x * x + y * y + z * z); + + // auxiliary vars + data_t omega = 1. - mass / (2. * r); + data_t psi = 1. + mass / (2. * r); + + data_t omega2 = omega * omega; + data_t psi4 = psi * psi * psi * psi; + data_t psi6 = psi4 * psi * psi; + data_t dx_psi = -mass * x / (2. * pow(r, 3)); + data_t dy_psi = -mass * y / (2. * pow(r, 3)); + data_t dz_psi = -mass * z / (2. * pow(r, 3)); + + data_t psi_omega = sqrt(psi6 - vel * vel * omega2); + data_t beta_x = -vel * (psi6 - omega2) / (psi_omega * psi_omega); + data_t g_xx = gamma * gamma * (psi_omega * psi_omega) / (psi * psi); + data_t g_yy = psi4; + data_t g_zz = psi4; + + // build tensors + Tensor<2, data_t, 3> KLL = {0.}; + Tensor<2, data_t, 3> gammaLL = {0.}; + Tensor<2, data_t, 3> gammaUU = {0.}; + + KLL[0][0] = -gamma * gamma * vel * dx_psi * + (2. * psi6 * (psi + 2. * omega) - 2. * vel * vel * omega2) / + (psi4 * psi * psi_omega); + KLL[1][1] = 2. * vel * psi * omega * dx_psi / psi_omega; + KLL[2][2] = KLL[1][1]; + KLL[0][1] = -gamma * vel * psi * dy_psi * (3 * omega + psi) / psi_omega; + KLL[1][0] = KLL[0][1]; + KLL[2][0] = -gamma * vel * psi * dz_psi * (3 * omega + psi) / psi_omega; + KLL[0][2] = KLL[2][0]; + KLL[2][1] = 0.; + KLL[1][2] = KLL[1][2]; + + gammaLL[0][0] = g_xx; + gammaLL[1][1] = g_yy; + gammaLL[2][2] = g_zz; + gammaUU[0][0] = 1. / g_xx; + gammaUU[1][1] = 1. / g_yy; + gammaUU[2][2] = 1. / g_zz; + + data_t chi = pow(g_xx * g_yy * g_zz, -1. / 3.); + +#if CH_SPACEDIM == 3 + CCZ4Vars::VarsWithGauge vars; +#elif CH_SPACEDIM == 2 + CCZ4CartoonVars::VarsWithGauge vars; +#endif + + // Set only the non-zero components explicitly below + VarsTools::assign(vars, 0.); + + vars.chi = chi; + vars.shift[0] = beta_x; + vars.lapse = sqrt(vars.chi); + + for (int i = 0; i < 3; i++) + { + for (int j = 0; j < 3; j++) + { + vars.K += KLL[i][j] * gammaUU[i][j]; + } + } + + FOR2(i, j) { vars.h[i][j] = vars.chi * gammaLL[i][j]; } + FOR2(i, j) + { + vars.A[i][j] = vars.chi * (KLL[i][j] - vars.K * gammaLL[i][j] / 3.); + } + +#if CH_SPACEDIM == 2 + vars.hww = vars.chi * gammaLL[2][2]; + vars.Aww = vars.chi * (KLL[2][2] - one_third * vars.K * gammaLL[2][2]); +#endif + + // Store the initial values of the variables + current_cell.store_vars(vars); +} + +#endif /* ISOTROPICBOOSTEDBH _IMPL_HPP_ */ diff --git a/Source/InitialConditions/BlackHoles/SingleBH.hpp b/Source/InitialConditions/BlackHoles/SingleBH.hpp index 293fa6d2e..2e57c3840 100644 --- a/Source/InitialConditions/BlackHoles/SingleBH.hpp +++ b/Source/InitialConditions/BlackHoles/SingleBH.hpp @@ -21,6 +21,8 @@ enum Lapse CHI }; +// Just a wrapper of the BoostedBH class, to be used by itself (instead of a +// BinaryBH) class SingleBH { protected: diff --git a/Tests/ApparentHorizonFinderTest2D/AHTest2DFunction.hpp b/Tests/ApparentHorizonFinderTest2D/AHTest2DFunction.hpp index e98ae3721..1f2b264eb 100644 --- a/Tests/ApparentHorizonFinderTest2D/AHTest2DFunction.hpp +++ b/Tests/ApparentHorizonFinderTest2D/AHTest2DFunction.hpp @@ -8,9 +8,9 @@ #include "AHFunctionDefault.hpp" -#include "AHData.hpp" -#include "AHDeriv.hpp" +#include "AHDerivData.hpp" #include "AHGeometryData.hpp" +#include "AHVarsData.hpp" #include "UserVariables.hpp" @@ -22,14 +22,14 @@ struct AHTest2DFunction : AHFunctionDefault static int vars_min() { return c_V; } static int vars_max() { return c_V; } - AHTest2DFunction(const AHData &a_data, + AHTest2DFunction(const AHVarsData &a_data, const Tensor<1, double> &a_coords, const Tensor<1, double> &a_cart_coords) { v = a_data.vars.at(c_V); } - double get(const AHGeometryData &geo_data, const AHDeriv &deriv, + double get(const AHGeometryData &geo_data, const AHDerivData &deriv, const params &a_params) const { return v; diff --git a/Tests/ApparentHorizonFinderTest2D/SimulationParameters.hpp b/Tests/ApparentHorizonFinderTest2D/SimulationParameters.hpp index e38c55637..d11637d19 100755 --- a/Tests/ApparentHorizonFinderTest2D/SimulationParameters.hpp +++ b/Tests/ApparentHorizonFinderTest2D/SimulationParameters.hpp @@ -24,7 +24,7 @@ class SimulationParameters : public ChomboParameters } #ifdef USE_AHFINDER - AHParams_t AH_params; + AHParams_t AH_params; #endif }; diff --git a/Tests/ApparentHorizonFinderTest3D/SimulationParameters.hpp b/Tests/ApparentHorizonFinderTest3D/SimulationParameters.hpp index 6953704d0..f919af141 100755 --- a/Tests/ApparentHorizonFinderTest3D/SimulationParameters.hpp +++ b/Tests/ApparentHorizonFinderTest3D/SimulationParameters.hpp @@ -42,7 +42,7 @@ class SimulationParameters : public ChomboParameters #ifdef USE_AHFINDER double initial_guess; - AHParams_t AH_params; + AHParams_t AH_params; #endif }; From 07774ad2cd34b70b189c5c1fa8a40347f661f594 Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Wed, 30 Jun 2021 00:44:19 +0100 Subject: [PATCH 30/92] Allow to change SNES/KSP PETSc parameters This is useful for the slow convergence of highly boosted BHs with highly ellipsoidal AHs --- Examples/SingleBH/params.txt | 2 +- Source/ApparentHorizonFinder/AHParams.hpp | 61 ++++++++++ .../ApparentHorizon.impl.hpp | 7 +- .../PETScAHSolver.impl.hpp | 105 +++++++++++++++--- 4 files changed, 155 insertions(+), 20 deletions(-) diff --git a/Examples/SingleBH/params.txt b/Examples/SingleBH/params.txt index 7dc781964..ae68c5e28 100644 --- a/Examples/SingleBH/params.txt +++ b/Examples/SingleBH/params.txt @@ -154,7 +154,7 @@ AH_solve_interval = 1 AH_print_interval = 1 # AH_track_center = 1 # AH_predict_origin = 1 -AH_level_to_run = 0 +# AH_level_to_run = 0 # AH_allow_re_attempt = 0 # AH_stop_if_max_fails = 0 # AH_start_time = 0. diff --git a/Source/ApparentHorizonFinder/AHParams.hpp b/Source/ApparentHorizonFinder/AHParams.hpp index 767ca0def..b0215d7f6 100644 --- a/Source/ApparentHorizonFinder/AHParams.hpp +++ b/Source/ApparentHorizonFinder/AHParams.hpp @@ -10,6 +10,64 @@ #include "ChomboParameters.hpp" #include "GRParmParse.hpp" +// PETSc Solver related params +// not very relevant to change, but this allows us to in case we want +// possible the only relevant one is 'snes_maxit' ('AH_SNES_max_iterations') +// which for highly ellipsoidal AHs sometimes needs to be increased as +// convergence is rather slow +// (as of 06/2021, the code outputs this as a recommendation when that happens) +struct PETSc_params +{ + /* + (defaults show according to a certain PETSc version, + can't assure it is the same for all) + atol - absolute convergence tolerance (default 1e-50) + rtol - relative convergence tolerance (default 1e-08) + stol - convergence tolerance in terms of the norm of the change in the + solution between steps (default 1e-08) + maxit - maximum number of iterations (default 50) + maxf - maximum number of function evaluations (default 10000) + divtol - divergence tolerance (default 10000) + */ + PetscReal snes_atol = -1., snes_rtol = -1., snes_stol = -1.; + PetscInt snes_maxit = -1, snes_maxf = -1; + PetscReal snes_divtol = -1.; + + /* + rtol - the relative convergence tolerance (default 1e-05) + abstol - the absolute convergence tolerance (default 1e-50) + dtol - the divergence tolerance (default 10000) + maxits - maximum number of iterations (default 10000) + */ + PetscReal ksp_rtol = -1., ksp_abstol = -1., ksp_dtol = -1.; + PetscInt ksp_maxits = -1; + + void read_params(GRParmParse &pp) + { + if (pp.contains("AH_SNES_absolute_tolerance")) + pp.load("AH_SNES_absolute_tolerance", snes_atol); + if (pp.contains("AH_SNES_relative_tolerance")) + pp.load("AH_SNES_relative_tolerance", snes_rtol); + if (pp.contains("AH_SNES_step_change_tolerance")) + pp.load("AH_SNES_step_change_tolerance", snes_stol); + if (pp.contains("AH_SNES_max_iterations")) + pp.load("AH_SNES_max_iterations", snes_maxit); + if (pp.contains("AH_SNES_max_evaluations")) + pp.load("AH_SNES_max_evaluations", snes_maxf); + if (pp.contains("AH_SNES_divergence_tolerance")) + pp.load("AH_SNES_divergence_tolerance", snes_divtol); + + if (pp.contains("AH_KSP_relative_tolerance")) + pp.load("AH_KSP_relative_tolerance", ksp_rtol); + if (pp.contains("AH_KSP_absolute_tolerance")) + pp.load("AH_KSP_absolute_tolerance", ksp_abstol); + if (pp.contains("AH_KSP_divergence_tolerance")) + pp.load("AH_KSP_divergence_tolerance", ksp_dtol); + if (pp.contains("AH_KSP_max_evaluations")) + pp.load("AH_KSP_max_evaluations", ksp_maxits); + } +}; + // prepend with 'AH_' in params file template struct AHParams_t { @@ -89,6 +147,8 @@ template struct AHParams_t typename AHFunction::params func_params; + PETSc_params petsc_params; + enum verbose_level { NONE, @@ -237,6 +297,7 @@ void AHParams_t::read_params(GRParmParse &pp, pp.load("AH_merger_pre_factor", merger_pre_factor, 1.); func_params.read_params(pp); + petsc_params.read_params(pp); } #endif /* _AHPARAMS_HPP_ */ diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp index b3eb08413..787549821 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp @@ -571,9 +571,12 @@ void ApparentHorizon::check_convergence() << std::endl; if (m_params.verbose > AHParams::MIN) - { pout() << "SNESConvergedReason = " << result << std::endl; - } + + if (result == SNES_DIVERGED_MAX_IT) + pout() << "(maximum iterations reached, try increasing " + "AH_SNES_max_iterations)" + << std::endl; } if (!m_has_been_found && m_converged) diff --git a/Source/ApparentHorizonFinder/PETScAHSolver.impl.hpp b/Source/ApparentHorizonFinder/PETScAHSolver.impl.hpp index 8ccf9f302..222eb0a8b 100644 --- a/Source/ApparentHorizonFinder/PETScAHSolver.impl.hpp +++ b/Source/ApparentHorizonFinder/PETScAHSolver.impl.hpp @@ -156,8 +156,10 @@ void PETScAHSolver::initialise() SNESSetFromOptions(m_snes); - if (m_params.verbose > AHParams::MIN) + // setup options { + const PETSc_params &pars = m_params.petsc_params; + SNESType snes_type; SNESGetType(m_snes, &snes_type); PetscReal snes_atol, snes_rtol, snes_stol; @@ -177,22 +179,91 @@ void PETScAHSolver::initialise() KSPGetTolerances(snes_ksp, &ksp_rtol, &ksp_abstol, &ksp_dtol, &ksp_maxits); - pout() << "-------------------------------------\n"; - pout() << "PETScAHSolver Options:\n"; - pout() << "-------------------------------------\n"; - pout() << "PETSc SNES Options:\n"; - pout() << "Type: " << snes_type << "\n"; - pout() << "atol = " << snes_atol << ", rtol = " << snes_rtol - << ", stol = " << snes_stol << ",\n"; - pout() << "maxit = " << snes_maxit << ", maxf = " << snes_maxf << "\n"; - pout() << "divtol = " << snes_divtol << "\n"; - pout() << "-------------------------------------\n"; - pout() << "PETSc KSP Options:\n"; - pout() << "Type: " << ksp_type << "\n"; - pout() << "rtol = " << ksp_rtol << ", abstol = " << ksp_abstol - << ", dtol = " << ksp_dtol << ", maxits = " << ksp_maxits - << "\n"; - pout() << "-------------------------------------" << std::endl; + bool any_changed = false; // SNES params + if (pars.snes_atol > 0 && pars.snes_atol != snes_atol) + { + any_changed = true; + snes_atol = pars.snes_atol; + } + if (pars.snes_rtol > 0 && pars.snes_rtol != snes_rtol) + { + any_changed = true; + snes_rtol = pars.snes_rtol; + } + if (pars.snes_stol > 0 && pars.snes_stol != snes_stol) + { + any_changed = true; + snes_stol = pars.snes_stol; + } + if (pars.snes_maxit > 0 && pars.snes_maxit != snes_maxit) + { + any_changed = true; + snes_maxit = pars.snes_maxit; + } + if (pars.snes_maxf > 0 && pars.snes_maxf != snes_maxf) + { + any_changed = true; + snes_maxf = pars.snes_maxf; + } + if (any_changed) + { + SNESSetTolerances(m_snes, snes_atol, snes_rtol, snes_stol, + snes_maxit, snes_maxf); + } + + if (pars.snes_divtol > 0 && pars.snes_divtol != snes_divtol) + { + snes_divtol = pars.snes_divtol; + SNESSetDivergenceTolerance(m_snes, snes_divtol); + } + + any_changed = false; // KSP params + if (pars.ksp_rtol > 0 && pars.ksp_rtol != ksp_rtol) + { + any_changed = true; + ksp_rtol = pars.ksp_rtol; + } + if (pars.ksp_abstol > 0 && pars.ksp_abstol != ksp_abstol) + { + any_changed = true; + ksp_abstol = pars.ksp_abstol; + } + if (pars.ksp_dtol > 0 && pars.ksp_dtol != ksp_dtol) + { + any_changed = true; + ksp_dtol = pars.ksp_dtol; + } + if (pars.ksp_maxits > 0 && pars.ksp_maxits != ksp_maxits) + { + any_changed = true; + ksp_maxits = pars.ksp_maxits; + } + if (any_changed) + { + KSPSetTolerances(snes_ksp, ksp_rtol, ksp_abstol, ksp_dtol, + ksp_maxits); + } + + if (m_params.verbose > AHParams::MIN) + { + pout() << "-------------------------------------\n"; + pout() << "PETScAHSolver Options:\n"; + pout() << "-------------------------------------\n"; + pout() << "PETSc SNES Options:\n"; + pout() << "Type: " << snes_type << "\n"; + pout() << "atol = " << snes_atol << ", rtol = " << snes_rtol + << ", stol = " << snes_stol << ",\n"; + pout() << "maxit = " << snes_maxit << ", maxf = " << snes_maxf + << "\n"; + pout() << "divtol = " << snes_divtol << "\n"; + pout() << "-------------------------------------\n"; + pout() << "PETSc KSP Options:\n"; + pout() << "Type: " << ksp_type << "\n"; + pout() << "rtol = " << ksp_rtol << ", abstol = " << ksp_abstol + << ", dtol = " << ksp_dtol << ", maxits = " << ksp_maxits + << "\n"; + pout() << "-------------------------------------" << std::endl; + } } } From 19755fa2f1c91b217f0d2c3100519945e6a65619 Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Wed, 30 Jun 2021 00:48:20 +0100 Subject: [PATCH 31/92] Post merge: FOR? to FOR in AHFinder code --- Examples/SingleBH/SimulationParameters.hpp | 2 +- .../SurfaceExtraction.impl.hpp | 2 +- .../ApparentHorizonFinder/AHFinder.impl.hpp | 4 +- .../AHFunctionDefault.hpp | 2 +- Source/ApparentHorizonFinder/AHFunctions.hpp | 28 +++++++------- .../ApparentHorizon.impl.hpp | 38 ++++++++----------- .../BlackHoles/IsotropicBoostedBH.impl.hpp | 4 +- .../BlackHoles/SingleBH.impl.hpp | 4 +- Source/utils/CoordinateTransformations.hpp | 16 ++++---- Source/utils/VarsTools.hpp | 2 +- 10 files changed, 48 insertions(+), 54 deletions(-) diff --git a/Examples/SingleBH/SimulationParameters.hpp b/Examples/SingleBH/SimulationParameters.hpp index ad24645f5..dca6088b6 100644 --- a/Examples/SingleBH/SimulationParameters.hpp +++ b/Examples/SingleBH/SimulationParameters.hpp @@ -43,7 +43,7 @@ class SimulationParameters : public SimulationParametersBase // provided) std::array offset; pp.load("offset", offset, {0.0, 0.0, 0.0}); - FOR1(idir) { bh_params.center[idir] = center[idir] + offset[idir]; } + FOR(idir) { bh_params.center[idir] = center[idir] + offset[idir]; } #ifdef USE_AHFINDER pp.load("AH_initial_guess", AH_initial_guess, 0.5 * bh_params.mass); diff --git a/Source/AMRInterpolator/SurfaceExtraction.impl.hpp b/Source/AMRInterpolator/SurfaceExtraction.impl.hpp index 0fc276ae0..2632c37fa 100644 --- a/Source/AMRInterpolator/SurfaceExtraction.impl.hpp +++ b/Source/AMRInterpolator/SurfaceExtraction.impl.hpp @@ -63,7 +63,7 @@ SurfaceExtraction::SurfaceExtraction( } } #elif CH_SPACEDIM == 2 - FOR1(idir) + FOR(idir) { int idx = index(isurface, iu); m_interp_coords[idir][idx] = diff --git a/Source/ApparentHorizonFinder/AHFinder.impl.hpp b/Source/ApparentHorizonFinder/AHFinder.impl.hpp index ddb9cde78..ae4e21b7a 100644 --- a/Source/ApparentHorizonFinder/AHFinder.impl.hpp +++ b/Source/ApparentHorizonFinder/AHFinder.impl.hpp @@ -289,7 +289,7 @@ bool AHFinder::solve_merger( // update center of merged, otherwise it does it by // itself in solve - FOR1(a) + FOR(a) { if (!AH1->get_ah_interp().get_interpolator()->get_boundary_reflective( Side::Lo, a)) @@ -373,7 +373,7 @@ void AHFinder::set_origins( auto origin2 = m_apparent_horizons[m_merger_pairs[i].second] ->get_origin(); std::array origin; - FOR1(a) { origin[a] = (origin1[a] + origin2[a]) / 2.; } + FOR(a) { origin[a] = (origin1[a] + origin2[a]) / 2.; } m_apparent_horizons[i]->set_origin(origin); } } diff --git a/Source/ApparentHorizonFinder/AHFunctionDefault.hpp b/Source/ApparentHorizonFinder/AHFunctionDefault.hpp index d717ab8ac..f330119bb 100644 --- a/Source/ApparentHorizonFinder/AHFunctionDefault.hpp +++ b/Source/ApparentHorizonFinder/AHFunctionDefault.hpp @@ -43,7 +43,7 @@ struct AHFunctionDefault { // cartesian flat metric Tensor<2, double> g = {0.}; - FOR1(i) { g[i][i] = 1.; } + FOR(i) { g[i][i] = 1.; } return g; } ALWAYS_INLINE const Tensor<2, double> get_inverse_metric() const diff --git a/Source/ApparentHorizonFinder/AHFunctions.hpp b/Source/ApparentHorizonFinder/AHFunctions.hpp index b515486d7..a65062272 100644 --- a/Source/ApparentHorizonFinder/AHFunctions.hpp +++ b/Source/ApparentHorizonFinder/AHFunctions.hpp @@ -155,7 +155,7 @@ struct ExpansionFunction : AHFunctionDefault // INVERSE METRIC Tensor<2, double, CH_SPACEDIM> h_DD; - FOR2(i, j) { h_DD[i][j] = a_data.vars.at(h[i][j]); } + FOR(i, j) { h_DD[i][j] = a_data.vars.at(h[i][j]); } Tensor<2, double, CH_SPACEDIM> h_UU = TensorAlgebra::compute_inverse_sym(h_DD); @@ -163,7 +163,7 @@ struct ExpansionFunction : AHFunctionDefault // Reconstructing ADM variables Tensor<1, double, CH_SPACEDIM> dchi; - FOR1(i) { dchi[i] = a_data.d1.at(c_chi)[i]; } + FOR(i) { dchi[i] = a_data.d1.at(c_chi)[i]; } for (int i = 0; i < CH_SPACEDIM; ++i) { @@ -211,11 +211,11 @@ struct ExpansionFunction : AHFunctionDefault coords = a_coords_cartesian; double hww = a_data.vars.at(comp_hww); - FOR1(a) { dhww[a] = a_data.d1.at(comp_hww)[a]; } + FOR(a) { dhww[a] = a_data.d1.at(comp_hww)[a]; } double Aww = a_data.vars.at(comp_Aww); g_hd = hww / chi; - FOR1(a) { dg_hd[a] = (dhww[a] - (hww * dchi[a]) / chi) / chi; } + FOR(a) { dg_hd[a] = (dhww[a] - (hww * dchi[a]) / chi) / chi; } Kww = Aww / chi + trK * g_hd / GR_SPACEDIM; #endif @@ -244,7 +244,7 @@ struct ExpansionFunction : AHFunctionDefault // calculate D_i S^i and S^i S^j K_ij double DiSi = 0.; double Kij_dot_Si_Sj = 0.; - FOR2(a, b) + FOR(a, b) { DiSi += (g_UU[a][b] - S_U[a] * S_U[b]) * Ds[a][b] / norm_s; Kij_dot_Si_Sj += S_U[a] * S_U[b] * K[a][b]; @@ -257,7 +257,7 @@ struct ExpansionFunction : AHFunctionDefault #if GR_SPACEDIM != CH_SPACEDIM expansion += (GR_SPACEDIM - CH_SPACEDIM) * S_U[CH_SPACEDIM - 1] / coords[CH_SPACEDIM - 1]; - FOR1(a) + FOR(a) { expansion += (GR_SPACEDIM - CH_SPACEDIM) * 0.5 * dg_hd[a] / g_hd * S_U[a]; @@ -281,7 +281,7 @@ struct ExpansionFunction : AHFunctionDefault // D_a L = d_a f - dF/du * du/dx^a - dF/dv * dv/dx^a Tensor<1, double> s_L = {0.}; // not normalized, just D_a L - FOR1(a) + FOR(a) { s_L[a] = geo_data.df[a] - (deriv.duF * geo_data.du[a]) #if CH_SPACEDIM == 3 @@ -306,14 +306,14 @@ struct ExpansionFunction : AHFunctionDefault // norm of s_L = | D_a L| (the 'u' in 6.7.12 of Alcubierre) norm_s = 0.0; - FOR2(a, b) { norm_s += g_UU[a][b] * s_L[a] * s_L[b]; } + FOR(a, b) { norm_s += g_UU[a][b] * s_L[a] * s_L[b]; } norm_s = sqrt(norm_s); // raise s_L Tensor<1, double> s_U = {0.}; - FOR2(a, b) { s_U[a] += g_UU[a][b] * s_L[b]; } + FOR(a, b) { s_U[a] += g_UU[a][b] * s_L[b]; } - FOR1(a) { S_U[a] = s_U[a] / norm_s; } + FOR(a) { S_U[a] = s_U[a] / norm_s; } } Tensor<2, double> get_level_function_2nd_covariant_derivative( @@ -324,7 +324,7 @@ struct ExpansionFunction : AHFunctionDefault // for the level function L = f - F(u,v) Tensor<2, double> ds = {0.}; - FOR2(a, b) + FOR(a, b) { ds[a][b] = geo_data.ddf[a][b] - (deriv.duF * geo_data.ddu[a][b]) - (deriv.duduF * geo_data.du[a] * geo_data.du[b]) @@ -339,7 +339,7 @@ struct ExpansionFunction : AHFunctionDefault // calculate Christoffels on this point (u,v,f) Tensor<3, double> chris = {0.}; - FOR4(a, b, c, d) + FOR(a, b, c, d) { chris[a][b][c] += 0.5 * g_UU[a][d] * (dg[b][d][c] + dg[c][d][b] - dg[b][c][d]); @@ -347,10 +347,10 @@ struct ExpansionFunction : AHFunctionDefault // covariant derivatrive of s_a to use for DS Tensor<2, double> Ds = {0.}; - FOR2(a, b) + FOR(a, b) { Ds[a][b] = ds[a][b]; - FOR1(c) { Ds[a][b] -= chris[c][a][b] * s_L[c]; } + FOR(c) { Ds[a][b] -= chris[c][a][b] * s_L[c]; } } return Ds; diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp index 787549821..4fdee1c5c 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp @@ -188,7 +188,7 @@ void ApparentHorizon::predict_next_origin() std::array new_center = m_old_centers[0]; if (m_converged >= 3) // add 2nd derivative { - FOR1(a) + FOR(a) { if (!solver.m_interp.get_interpolator()->get_boundary_reflective( Side::Lo, a) && @@ -211,7 +211,7 @@ void ApparentHorizon::predict_next_origin() } if (m_converged >= 2) // add 1st derivative { - FOR1(a) + FOR(a) { if (!solver.m_interp.get_interpolator()->get_boundary_reflective( Side::Lo, a) && @@ -255,7 +255,7 @@ template void ApparentHorizon::update_old_centers( std::array new_center) { - FOR1(a) + FOR(a) { if (!solver.m_interp.get_interpolator()->get_boundary_reflective( Side::Lo, a) && @@ -355,7 +355,7 @@ void ApparentHorizon::solve(double a_dt, #if CH_SPACEDIM == 3 m_spin = J_norm / m_mass; - FOR1(a) { m_dimensionless_spin_vector[a] = J[a] / (m_mass * m_mass); } + FOR(a) { m_dimensionless_spin_vector[a] = J[a] / (m_mass * m_mass); } m_spin_z_alt = calculate_spin_dimensionless(m_area); #endif @@ -461,7 +461,7 @@ void ApparentHorizon::write_outputs( values[idx++] = m_irreducible_mass; #if CH_SPACEDIM == 3 values[idx++] = m_spin; - FOR1(a) { values[idx++] = m_dimensionless_spin_vector[a]; } + FOR(a) { values[idx++] = m_dimensionless_spin_vector[a]; } values[idx++] = m_spin_z_alt; #endif #endif @@ -767,7 +767,7 @@ void ApparentHorizon::restart( #endif int offset = CH_SPACEDIM + was_center_tracked * CH_SPACEDIM; - FOR1(a) { origin[a] = stats[cols - offset + a][idx]; } + FOR(a) { origin[a] = stats[cols - offset + a][idx]; } if (m_params.verbose > AHParams::NONE) { @@ -1318,7 +1318,7 @@ ApparentHorizon::calculate_spin_dimensionless( deriv.dvF * geometry_data.dxdf[2]; double norm2 = 0.; - FOR2(i, j) norm2 += g[i][j] * dxdv[i] * dxdv[j]; + FOR(i, j) norm2 += g[i][j] * dxdv[i] * dxdv[j]; CH_assert(norm2 >= 0.); double weight = m_integration_methods[1].weight( @@ -1412,17 +1412,14 @@ ApparentHorizon::calculate_angular_momentum_J() Tensor<1, double> S_U = func.get_spatial_normal_U(s_L); Tensor<1, double> coords_cart_centered; - FOR1(i) - { - coords_cart_centered[i] = coords_cart[i] - center[i]; - } + FOR(i) { coords_cart_centered[i] = coords_cart[i] - center[i]; } Tensor<1, double> spin_integrand = {0.}; double directions[3][3] = { {0, -coords_cart_centered[2], coords_cart_centered[1]}, {coords_cart_centered[2], 0., -coords_cart_centered[0]}, {-coords_cart_centered[1], coords_cart_centered[0], 0.}}; - FOR3(a, b, c) + FOR(a, b, c) { // as in arXiv:gr-qc/0206008 eq. 25 // but computing with 'killing vector' in directions 'x,y,z' @@ -1436,7 +1433,7 @@ ApparentHorizon::calculate_angular_momentum_J() // Calculate Jacobian matrix for transformation from Cartesian // to (f,u,v) coords Tensor<2, double> Jac; - FOR1(k) + FOR(k) { Jac[0][k] = geometric_data.dxdf[k]; Jac[1][k] = geometric_data.dxdu[k]; @@ -1445,7 +1442,7 @@ ApparentHorizon::calculate_angular_momentum_J() // Now do the coordinate transformation Tensor<2, double> g_spherical = {0.}; - FOR4(i, j, k, l) + FOR(i, j, k, l) { g_spherical[i][j] += Jac[i][k] * Jac[j][l] * g[k][l]; } @@ -1472,7 +1469,7 @@ ApparentHorizon::calculate_angular_momentum_J() u, m_params.num_points_u, solver.m_interp.get_coord_system().is_u_periodic()); - FOR1(a) + FOR(a) { double element = spin_integrand[a] / (8. * M_PI) * sqrt(det) * weight * solver.m_du; @@ -1490,10 +1487,7 @@ ApparentHorizon::calculate_angular_momentum_J() double weight = m_integration_methods[1].weight( v, m_params.num_points_v, solver.m_interp.get_coord_system().is_v_periodic()); - FOR1(a) - { - integrals[a] += weight * solver.m_dv * inner_integral[a]; - } + FOR(a) { integrals[a] += weight * solver.m_dv * inner_integral[a]; } } solver.restore_dmda_arr_t(localF, in); @@ -1508,7 +1502,7 @@ ApparentHorizon::calculate_angular_momentum_J() MPI_Allreduce(&integrals, &J, GR_SPACEDIM, MPI_DOUBLE, MPI_SUM, Chombo_MPI::comm); #else // serial - FOR1(a) { J[a] = integrals[a]; } + FOR(a) { J[a] = integrals[a]; } #endif return J; @@ -1559,7 +1553,7 @@ double ApparentHorizon::calculate_area() // Calculate Jacobian matrix for transformation from Cartesian // to (f,u,v) coords Tensor<2, double> Jac; - FOR1(k) + FOR(k) { Jac[0][k] = geometric_data.dxdf[k]; Jac[1][k] = geometric_data.dxdu[k]; @@ -1577,7 +1571,7 @@ double ApparentHorizon::calculate_area() // Now do the coordinate transformation Tensor<2, double> g_spherical = {0.}; - FOR4(i, j, k, l) + FOR(i, j, k, l) { g_spherical[i][j] += Jac[i][k] * Jac[j][l] * g[k][l]; } diff --git a/Source/InitialConditions/BlackHoles/IsotropicBoostedBH.impl.hpp b/Source/InitialConditions/BlackHoles/IsotropicBoostedBH.impl.hpp index fb8a1e7e1..8bae6b2dc 100644 --- a/Source/InitialConditions/BlackHoles/IsotropicBoostedBH.impl.hpp +++ b/Source/InitialConditions/BlackHoles/IsotropicBoostedBH.impl.hpp @@ -103,8 +103,8 @@ void IsotropicBoostedBH::compute(Cell current_cell) const } } - FOR2(i, j) { vars.h[i][j] = vars.chi * gammaLL[i][j]; } - FOR2(i, j) + FOR(i, j) { vars.h[i][j] = vars.chi * gammaLL[i][j]; } + FOR(i, j) { vars.A[i][j] = vars.chi * (KLL[i][j] - vars.K * gammaLL[i][j] / 3.); } diff --git a/Source/InitialConditions/BlackHoles/SingleBH.impl.hpp b/Source/InitialConditions/BlackHoles/SingleBH.impl.hpp index 9a59cf777..8d2991394 100644 --- a/Source/InitialConditions/BlackHoles/SingleBH.impl.hpp +++ b/Source/InitialConditions/BlackHoles/SingleBH.impl.hpp @@ -25,7 +25,7 @@ template void SingleBH::compute(Cell current_cell) const vars.chi = compute_chi(coords); // Conformal metric is flat - FOR1(i) vars.h[i][i] = 1.; + FOR(i) vars.h[i][i] = 1.; vars.A = compute_A(vars.chi, coords); @@ -63,7 +63,7 @@ Tensor<2, data_t> SingleBH::compute_A(data_t chi, Tensor<2, data_t> out; // Aij(CCZ4) = psi^(-6) * Aij(Baumgarte&Shapiro book) - FOR2(i, j) out[i][j] = pow(chi, 3 / 2.) * Aij[i][j]; + FOR(i, j) out[i][j] = pow(chi, 3 / 2.) * Aij[i][j]; return out; } diff --git a/Source/utils/CoordinateTransformations.hpp b/Source/utils/CoordinateTransformations.hpp index 96574a88b..9d4175bdc 100644 --- a/Source/utils/CoordinateTransformations.hpp +++ b/Source/utils/CoordinateTransformations.hpp @@ -80,10 +80,10 @@ Tensor<1, data_t, 3> transform_vector(const Tensor<1, data_t, 3> &vec_U, const Tensor<2, data2_t, 3> &jacobian) { Tensor<1, data_t, 3> transformed_U; - FOR1(i) + FOR(i) { transformed_U[i] = 0.0; - FOR1(j) { transformed_U[i] += jacobian[i][j] * vec_U[j]; } + FOR(j) { transformed_U[i] += jacobian[i][j] * vec_U[j]; } } return transformed_U; } @@ -95,10 +95,10 @@ Tensor<1, data_t, 3> transform_covector(const Tensor<1, data_t, 3> &vec_L, const Tensor<2, data2_t, 3> &jacobian) { Tensor<1, data_t, 3> transformed_L; - FOR1(i) + FOR(i) { transformed_L[i] = 0.0; - FOR1(j) { transformed_L[i] += vec_L[j] * jacobian[j][i]; } + FOR(j) { transformed_L[i] += vec_L[j] * jacobian[j][i]; } } return transformed_L; } @@ -110,10 +110,10 @@ Tensor<2, data_t, 3> transform_tensor_UU(const Tensor<2, data_t, 3> &tensor_UU, const Tensor<2, data2_t, 3> &jacobian) { Tensor<2, data_t, 3> transformed_UU; - FOR2(i, j) + FOR(i, j) { transformed_UU[i][j] = 0.; - FOR2(k, l) + FOR(k, l) { transformed_UU[i][j] += jacobian[i][k] * jacobian[j][l] * tensor_UU[k][l]; @@ -129,10 +129,10 @@ Tensor<2, data_t, 3> transform_tensor_LL(const Tensor<2, data_t, 3> &tensor_LL, const Tensor<2, data2_t, 3> &jacobian) { Tensor<2, data_t, 3> transformed_LL; - FOR2(i, j) + FOR(i, j) { transformed_LL[i][j] = 0.; - FOR2(k, l) + FOR(k, l) { transformed_LL[i][j] += tensor_LL[k][l] * jacobian[k][i] * jacobian[l][j]; diff --git a/Source/utils/VarsTools.hpp b/Source/utils/VarsTools.hpp index 75e0a4893..1df28c345 100644 --- a/Source/utils/VarsTools.hpp +++ b/Source/utils/VarsTools.hpp @@ -46,7 +46,7 @@ void define_symmetric_enum_mapping( DEFAULT_TENSOR_DIM * (DEFAULT_TENSOR_DIM + 1) / 2, "Interval has wrong size"); int idx = 0; - FOR1(idir1) + FOR(idir1) { for (int idir2 = idir1; idir2 < DEFAULT_TENSOR_DIM; ++idir2, ++idx) { From 2e6f80d685a4a561132ec1c0841545c33a18dc16 Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Wed, 30 Jun 2021 18:51:28 +0100 Subject: [PATCH 32/92] Fix typo and debug mode warnings --- Examples/SingleBH/Main_SingleBH.cpp | 2 +- Examples/SingleBH/SimulationParameters.hpp | 4 ++-- Source/ApparentHorizonFinder/AHInitialGuess.hpp | 14 +++++++------- .../ApparentHorizonFinder/ApparentHorizon.impl.hpp | 6 +++--- .../ApparentHorizonFinder/PETScAHSolver.impl.hpp | 14 +++++--------- 5 files changed, 18 insertions(+), 22 deletions(-) diff --git a/Examples/SingleBH/Main_SingleBH.cpp b/Examples/SingleBH/Main_SingleBH.cpp index 5bb1e3acd..6ae9196cf 100644 --- a/Examples/SingleBH/Main_SingleBH.cpp +++ b/Examples/SingleBH/Main_SingleBH.cpp @@ -47,7 +47,7 @@ int runGRChombo(int argc, char *argv[]) bh_amr.m_ah_finder.add_ah(sph, sim_params.AH_initial_guess, sim_params.AH_params); #else - bh_amr.m_ah_finder.add_ah(sph, sim_params.AH_initial_guess_elipsoid, + bh_amr.m_ah_finder.add_ah(sph, sim_params.AH_initial_guess_ellipsoid, sim_params.AH_params); #endif } diff --git a/Examples/SingleBH/SimulationParameters.hpp b/Examples/SingleBH/SimulationParameters.hpp index dca6088b6..c6391ad6a 100644 --- a/Examples/SingleBH/SimulationParameters.hpp +++ b/Examples/SingleBH/SimulationParameters.hpp @@ -60,7 +60,7 @@ class SimulationParameters : public SimulationParametersBase r_x *= contraction; } - AH_initial_guess_elipsoid.set_params(r_x, r_y, r_z); + AH_initial_guess_ellipsoid.set_params(r_x, r_y, r_z); #endif #endif } @@ -74,7 +74,7 @@ class SimulationParameters : public SimulationParametersBase #ifdef USE_AHFINDER double AH_initial_guess; #ifdef USE_ISOTROPIC_BOOSTED_BH - AHInitialGuessElipsoid AH_initial_guess_elipsoid; + AHInitialGuessEllipsoid AH_initial_guess_ellipsoid; #endif #endif }; diff --git a/Source/ApparentHorizonFinder/AHInitialGuess.hpp b/Source/ApparentHorizonFinder/AHInitialGuess.hpp index ed2e6bc5a..3d9f15ff7 100644 --- a/Source/ApparentHorizonFinder/AHInitialGuess.hpp +++ b/Source/ApparentHorizonFinder/AHInitialGuess.hpp @@ -72,8 +72,8 @@ class AHInitialGuessMerger : public AHInitialGuessDefault public: const AHInitialGuessPtr m_ah1, m_ah2; - double m_merger_search_factor; // see note above (default is 1) - double m_merger_pre_factor; // see notes in AHParams + double m_merger_pre_factor; // see notes in AHParams + double m_merger_search_factor; AHInitialGuessMerger(const AHInitialGuessPtr a_ah1, const AHInitialGuessPtr a_ah2, @@ -116,7 +116,7 @@ class AHInitialGuessMerger : public AHInitialGuessDefault }; // ellipsoid aligned with one of the axis -class AHInitialGuessElipsoid : public AHInitialGuessDefault +class AHInitialGuessEllipsoid : public AHInitialGuessDefault { public: double m_radius_x, m_radius_y; @@ -124,11 +124,11 @@ class AHInitialGuessElipsoid : public AHInitialGuessDefault double m_radius_z; #endif - AHInitialGuessElipsoid() {} + AHInitialGuessEllipsoid() {} #if CH_SPACEDIM == 3 - AHInitialGuessElipsoid(double a_radius_x, double a_radius_y, - double a_radius_z) + AHInitialGuessEllipsoid(double a_radius_x, double a_radius_y, + double a_radius_z) { set_params(a_radius_x, a_radius_y, a_radius_z); } @@ -157,7 +157,7 @@ class AHInitialGuessElipsoid : public AHInitialGuessDefault return radius; } #elif CH_SPACEDIM == 2 - AHInitialGuessElipsoid(double a_radius_x, double a_radius_y) + AHInitialGuessEllipsoid(double a_radius_x, double a_radius_y) { set_params(a_radius_x, a_radius_y); } diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp index 4fdee1c5c..a2215a2e6 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp @@ -30,8 +30,6 @@ ApparentHorizon::ApparentHorizon( m_stats(a_stats), m_coords(a_coords), - solver(a_interp, a_initial_guess, a_params), - m_printed_once(false), m_converged(0), @@ -57,7 +55,9 @@ ApparentHorizon::ApparentHorizon( m_area(NAN), m_spin(NAN), m_mass(NAN), m_irreducible_mass(NAN), m_spin_z_alt(NAN), m_dimensionless_spin_vector({NAN}), - origin_already_updated(false) + origin_already_updated(false), + + solver(a_interp, a_initial_guess, a_params) { set_origin(a_interp.get_coord_system().get_origin()); check_integration_methods(); diff --git a/Source/ApparentHorizonFinder/PETScAHSolver.impl.hpp b/Source/ApparentHorizonFinder/PETScAHSolver.impl.hpp index 222eb0a8b..096784a64 100644 --- a/Source/ApparentHorizonFinder/PETScAHSolver.impl.hpp +++ b/Source/ApparentHorizonFinder/PETScAHSolver.impl.hpp @@ -23,20 +23,16 @@ template PETScAHSolver::PETScAHSolver( const AHInterpolation &a_interp, const AHInitialGuessPtr a_initial_guess, const AHParams &a_params) - : m_initial_guess(a_initial_guess), - - m_params(a_params), - - m_interp(a_interp), m_interp_plus(a_interp), m_interp_minus(a_interp), + : m_interp(a_interp), m_interp_plus(a_interp), m_interp_minus(a_interp), m_periodic_u(a_interp.get_coord_system().is_u_periodic()), - m_num_global_u(a_params.num_points_u) - + m_num_global_u(a_params.num_points_u), #if CH_SPACEDIM == 3 - , m_periodic_v(a_interp.get_coord_system().is_v_periodic()), - m_num_global_v(a_params.num_points_v) + m_num_global_v(a_params.num_points_v), #endif + + m_initial_guess(a_initial_guess), m_params(a_params) { initialise(); } From 0020ed4e14073b35ca936d6d9cfab528a6b474ef Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Wed, 30 Jun 2021 19:16:45 +0100 Subject: [PATCH 33/92] Fix SphericalExtractionUniformTest (make it more robust by decreasing the number of points, otherwise getting too close to numerical precision) --- .../SphericalExtractionUniformTest.cpp | 141 ++++++++++-------- .../SphericalExtractionUniformTest.inputs | 14 +- 2 files changed, 88 insertions(+), 67 deletions(-) diff --git a/Tests/SphericalExtractionUniformTest/SphericalExtractionUniformTest.cpp b/Tests/SphericalExtractionUniformTest/SphericalExtractionUniformTest.cpp index 877a84af4..7df6159aa 100644 --- a/Tests/SphericalExtractionUniformTest/SphericalExtractionUniformTest.cpp +++ b/Tests/SphericalExtractionUniformTest/SphericalExtractionUniformTest.cpp @@ -45,38 +45,25 @@ using std::endl; // Chombo namespace #include "UsingNamespace.H" -int runSphericalExtractionUniformTest(int argc, char *argv[]) +void surface_integration( + const spherical_extraction_params_t &extraction_params_lo, + const SimulationParameters &sim_params, + AMRInterpolator> &interpolator, + std::pair, std::vector> &integral_lo, + std::pair, std::vector> &integral_hi, + const IntegrationMethod &method) { - // Load the parameter file and construct the SimulationParameter class - // To add more parameters edit the SimulationParameters file. - std::string in_string = argv[argc - 1]; - pout() << in_string << std::endl; - char const *in_file = argv[argc - 1]; - GRParmParse pp(0, argv + argc, NULL, in_file); - SimulationParameters sim_params(pp); - - GRAMR gr_amr; - DefaultLevelFactory - surface_extraction_test_level_fact(gr_amr, sim_params); - // the initial data for the two variables is the spherical harmonic - // specified by params - setupAMRObject(gr_amr, surface_extraction_test_level_fact); - - AMRInterpolator> interpolator( - gr_amr, sim_params.origin, sim_params.dx, sim_params.boundary_params); - // low resolution spherical extraction SphericalExtractionUniform spherical_extraction_lo( - sim_params.extraction_params_lo, - sim_params.coarsest_dx * sim_params.dt_multiplier, 0.0, true, 0.0); + extraction_params_lo, sim_params.coarsest_dx * sim_params.dt_multiplier, + 0.0, true, 0.0); spherical_extraction_lo.add_var(c_phi_Re); spherical_extraction_lo.add_var(c_phi_Im); spherical_extraction_lo.extract(&interpolator); spherical_extraction_lo.write_extraction("ExtractionOutLo_"); // high resolution spherical extraction - spherical_extraction_params_t extraction_params_hi = - sim_params.extraction_params_lo; + spherical_extraction_params_t extraction_params_hi = extraction_params_lo; // we are only checking the converence in theta integration // extraction_params_hi.num_points_phi *= 2; extraction_params_hi.num_points_theta *= 2; @@ -94,55 +81,89 @@ int runSphericalExtractionUniformTest(int argc, char *argv[]) return std::make_pair(data[0], data[1]); }; - // add the spherical harmonic mode integrands for each resolution and for - // the midpoint rule and Milne's regularized rule, Open 3rd and Open 4th - // order rules + // add the spherical harmonic mode integrands for each resolution // Always use trapezium rule in phi as this is periodic bool broadcast_integral = true; - std::pair, std::vector> integral_lo_midpoint, - integral_hi_midpoint; spherical_extraction_lo.add_mode_integrand( sim_params.es, sim_params.el, sim_params.em, extracted_harmonic, - integral_lo_midpoint, IntegrationMethod::midpoint, - IntegrationMethod::trapezium, broadcast_integral); + integral_lo, method, IntegrationMethod::trapezium, broadcast_integral); spherical_extraction_hi.add_mode_integrand( sim_params.es, sim_params.el, sim_params.em, extracted_harmonic, - integral_hi_midpoint, IntegrationMethod::midpoint, - IntegrationMethod::trapezium, broadcast_integral); + integral_hi, method, IntegrationMethod::trapezium, broadcast_integral); + + // do the surface integration + spherical_extraction_lo.integrate(); + spherical_extraction_hi.integrate(); +} + +int runSphericalExtractionUniformTest(int argc, char *argv[]) +{ + // Load the parameter file and construct the SimulationParameter class + // To add more parameters edit the SimulationParameters file. + std::string in_string = argv[argc - 1]; + pout() << in_string << std::endl; + char const *in_file = argv[argc - 1]; + GRParmParse pp(0, argv + argc, NULL, in_file); + SimulationParameters sim_params(pp); + + GRAMR gr_amr; + DefaultLevelFactory + surface_extraction_test_level_fact(gr_amr, sim_params); + // the initial data for the two variables is the spherical harmonic + // specified by params + setupAMRObject(gr_amr, surface_extraction_test_level_fact); + + AMRInterpolator> interpolator( + gr_amr, sim_params.origin, sim_params.dx, sim_params.boundary_params); + + std::pair, std::vector> integral_lo_midpoint, + integral_hi_midpoint; + spherical_extraction_params_t extraction_params_lo_midpoint = + sim_params.extraction_params_lo; + // must be multiple of 2 + extraction_params_lo_midpoint.num_points_theta = + std::ceil(extraction_params_lo_midpoint.num_points_theta / 2) * 2; + surface_integration(extraction_params_lo_midpoint, sim_params, interpolator, + integral_lo_midpoint, integral_hi_midpoint, + IntegrationMethod::midpoint); + std::pair, std::vector> integral_lo_milne_regularized, integral_hi_milne_regularized; - spherical_extraction_lo.add_mode_integrand( - sim_params.es, sim_params.el, sim_params.em, extracted_harmonic, - integral_lo_milne_regularized, IntegrationMethod::milne_regularized, - IntegrationMethod::trapezium, broadcast_integral); - spherical_extraction_hi.add_mode_integrand( - sim_params.es, sim_params.el, sim_params.em, extracted_harmonic, - integral_hi_milne_regularized, IntegrationMethod::milne_regularized, - IntegrationMethod::trapezium, broadcast_integral); + spherical_extraction_params_t extraction_params_lo_milne_regularized = + sim_params.extraction_params_lo; + // must be multiple of 3 + extraction_params_lo_milne_regularized.num_points_theta = + std::ceil(extraction_params_lo_milne_regularized.num_points_theta / 3) * + 3; + surface_integration(extraction_params_lo_milne_regularized, sim_params, + interpolator, integral_lo_milne_regularized, + integral_hi_milne_regularized, + IntegrationMethod::milne_regularized); + std::pair, std::vector> integral_lo_open_3rd_order, integral_hi_open_3rd_order; - spherical_extraction_lo.add_mode_integrand( - sim_params.es, sim_params.el, sim_params.em, extracted_harmonic, - integral_lo_open_3rd_order, IntegrationMethod::open_3rd_order, - IntegrationMethod::trapezium, broadcast_integral); - spherical_extraction_hi.add_mode_integrand( - sim_params.es, sim_params.el, sim_params.em, extracted_harmonic, - integral_hi_open_3rd_order, IntegrationMethod::open_3rd_order, - IntegrationMethod::trapezium, broadcast_integral); + spherical_extraction_params_t extraction_params_lo_open_3rd_order = + sim_params.extraction_params_lo; + // must be multiple of 4 + extraction_params_lo_open_3rd_order.num_points_theta = + std::ceil(extraction_params_lo_open_3rd_order.num_points_theta / 4) * 4; + surface_integration(extraction_params_lo_open_3rd_order, sim_params, + interpolator, integral_lo_open_3rd_order, + integral_hi_open_3rd_order, + IntegrationMethod::open_3rd_order); + std::pair, std::vector> integral_lo_open_4th_order, integral_hi_open_4th_order; - spherical_extraction_lo.add_mode_integrand( - sim_params.es, sim_params.el, sim_params.em, extracted_harmonic, - integral_lo_open_4th_order, IntegrationMethod::open_4th_order, - IntegrationMethod::trapezium, broadcast_integral); - spherical_extraction_hi.add_mode_integrand( - sim_params.es, sim_params.el, sim_params.em, extracted_harmonic, - integral_hi_open_4th_order, IntegrationMethod::open_4th_order, - IntegrationMethod::trapezium, broadcast_integral); - - // do the surface integration - spherical_extraction_lo.integrate(); - spherical_extraction_hi.integrate(); + spherical_extraction_params_t extraction_params_lo_open_4th_order = + sim_params.extraction_params_lo; + // must be multiple of 5 + extraction_params_lo_open_4th_order.num_points_theta = + std::floor(extraction_params_lo_open_4th_order.num_points_theta / 5) * + 5; + surface_integration(extraction_params_lo_open_4th_order, sim_params, + interpolator, integral_lo_open_4th_order, + integral_hi_open_4th_order, + IntegrationMethod::open_4th_order); int status = 0; pout() << std::setprecision(10); diff --git a/Tests/SphericalExtractionUniformTest/SphericalExtractionUniformTest.inputs b/Tests/SphericalExtractionUniformTest/SphericalExtractionUniformTest.inputs index cd11bf9f8..a270a2b49 100644 --- a/Tests/SphericalExtractionUniformTest/SphericalExtractionUniformTest.inputs +++ b/Tests/SphericalExtractionUniformTest/SphericalExtractionUniformTest.inputs @@ -18,12 +18,12 @@ checkpoint_interval = 1 # extraction params num_extraction_radii = 2 extraction_radii = 4. 6. -num_points_phi_lo = 60 -num_points_theta_lo = 60 # multiple of 3,4,5 +num_points_phi_lo = 12 +num_points_theta_lo = 12 # multiple of 3 for Milne's method, + # must be multiple of 4 for 3rd order + # and of 5 for the 4th order open method # Spherical Harmonic to set and extract -# needed to choose a complicated mode, otherwise computation -# is too precise (since we use 60 points) and doesn't converge -es = -1 -el = 6 -em = 1 +es = 0 +el = 3 # 2 was too easy for the 4th order method and not give convergence +em = 0 From 634ba0a6eab74c6f8f11227ed930916c56653d81 Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Wed, 7 Jul 2021 15:26:50 +0100 Subject: [PATCH 34/92] Fix Initial Guess for mergers Need to pass AHInitialGuessPtr by reference Also change the Ellipsoidal InitialGuess to give a circular guess for the merger instead of an ellipsoidal one --- Source/ApparentHorizonFinder/AHFinder.hpp | 8 +-- .../ApparentHorizonFinder/AHFinder.impl.hpp | 7 ++- .../ApparentHorizonFinder/AHInitialGuess.hpp | 61 ++++++++++++++----- Source/ApparentHorizonFinder/AHParams.hpp | 2 +- .../ApparentHorizonFinder/ApparentHorizon.hpp | 2 +- .../ApparentHorizon.impl.hpp | 2 +- .../ApparentHorizonFinder/PETScAHSolver.hpp | 4 +- .../PETScAHSolver.impl.hpp | 4 +- 8 files changed, 61 insertions(+), 29 deletions(-) diff --git a/Source/ApparentHorizonFinder/AHFinder.hpp b/Source/ApparentHorizonFinder/AHFinder.hpp index 6ea2d869d..32b9fa001 100644 --- a/Source/ApparentHorizonFinder/AHFinder.hpp +++ b/Source/ApparentHorizonFinder/AHFinder.hpp @@ -113,9 +113,9 @@ class AHFinder ); //! returns the index of the AH in m_apparent_horizons int add_ah(const SurfaceGeometry &a_coord_system, - AHInitialGuessPtr - a_initial_guess, //!< Initial guess for radius (or whatever - //!< coordinate you're solving for) + const AHInitialGuessPtr + &a_initial_guess, //!< Initial guess for radius (or whatever + //!< coordinate you're solving for) const AHParams &a_params, //!< set of AH parameters bool solve_first_step = true //!< whether or not to solve if t=0 ); @@ -151,7 +151,7 @@ class AHFinder private: //! returns false if 'parent' AHs are too far //! sets the initial guess and the origin for the merger - bool solve_merger(int ah1, int ah2, AHInitialGuessPtr initial_guess_merger, + bool solve_merger(int ah1, int ah2, AHInitialGuessPtr &initial_guess_merger, std::array &origin_merged); private: diff --git a/Source/ApparentHorizonFinder/AHFinder.impl.hpp b/Source/ApparentHorizonFinder/AHFinder.impl.hpp index ae4e21b7a..1009c3a21 100644 --- a/Source/ApparentHorizonFinder/AHFinder.impl.hpp +++ b/Source/ApparentHorizonFinder/AHFinder.impl.hpp @@ -34,8 +34,9 @@ int AHFinder::add_ah( template int AHFinder::add_ah( - const SurfaceGeometry &a_coord_system, AHInitialGuessPtr a_initial_guess, - const AHParams &a_params, bool solve_first_step) + const SurfaceGeometry &a_coord_system, + const AHInitialGuessPtr &a_initial_guess, const AHParams &a_params, + bool solve_first_step) { PETScCommunicator::initialize(a_params.num_ranks); @@ -266,7 +267,7 @@ bool AHFinder::need_diagnostics( template bool AHFinder::solve_merger( - int ah1, int ah2, AHInitialGuessPtr initial_guess_merger, + int ah1, int ah2, AHInitialGuessPtr &initial_guess_merger, std::array ¢er_merger) { // SKIP if 'parents' not yet close enough diff --git a/Source/ApparentHorizonFinder/AHInitialGuess.hpp b/Source/ApparentHorizonFinder/AHInitialGuess.hpp index 3d9f15ff7..5dc35a037 100644 --- a/Source/ApparentHorizonFinder/AHInitialGuess.hpp +++ b/Source/ApparentHorizonFinder/AHInitialGuess.hpp @@ -7,6 +7,7 @@ #define _AHINITIALGUESS_HPP_ #include "AlwaysInline.hpp" +#include "DimensionDefinitions.hpp" #include // sin, cos, sqrt #include // std::shared_ptr @@ -16,21 +17,25 @@ class AHInitialGuessDefault public: #if CH_SPACEDIM == 3 virtual double get(double u, double v) const = 0; - inline virtual double get_merger_contribution(double u, double v) const + ALWAYS_INLINE virtual double get_merger_contribution(double u, + double v) const { return get(u, v); }; - inline virtual double get_merger_min_distance() const + ALWAYS_INLINE virtual double get_merger_min_distance() const { return get(0., 0.); }; #elif CH_SPACEDIM == 2 virtual double get(double u) const = 0; - inline virtual double get_merger_contribution(double u) const + ALWAYS_INLINE virtual double get_merger_contribution(double u) const { return get(u); }; - inline virtual double get_merger_min_distance() const { return get(0.); }; + ALWAYS_INLINE virtual double get_merger_min_distance() const + { + return get(0.); + }; #endif }; @@ -58,12 +63,15 @@ class AHInitialGuessConstant : public AHInitialGuessDefault } #if CH_SPACEDIM == 3 - ALWAYS_INLINE double get(double u, double v) const + ALWAYS_INLINE double get(double u, double v) const override { return m_initial_guess; } #elif CH_SPACEDIM == 2 - ALWAYS_INLINE double get(double u) const { return m_initial_guess; } + ALWAYS_INLINE double get(double u) const override + { + return m_initial_guess; + } #endif }; @@ -84,7 +92,7 @@ class AHInitialGuessMerger : public AHInitialGuessDefault { } - inline virtual double get_merger_min_distance() const override + ALWAYS_INLINE double get_merger_min_distance() const override { return m_merger_search_factor * 4. * (m_ah1->get_merger_min_distance() + @@ -92,7 +100,7 @@ class AHInitialGuessMerger : public AHInitialGuessDefault }; #if CH_SPACEDIM == 3 - ALWAYS_INLINE double get(double u, double v) const + ALWAYS_INLINE double get(double u, double v) const override { return m_merger_pre_factor * 4. * get_merger_contribution(u, v); } @@ -103,7 +111,7 @@ class AHInitialGuessMerger : public AHInitialGuessDefault m_ah2->get_merger_contribution(u, v); } #elif CH_SPACEDIM == 2 - ALWAYS_INLINE double get(double u) const + ALWAYS_INLINE double get(double u) const override { return m_merger_pre_factor * 4. * get_merger_contribution(u); } @@ -142,7 +150,12 @@ class AHInitialGuessEllipsoid : public AHInitialGuessDefault << m_radius_y << ", " << m_radius_z << ")" << std::endl; } - ALWAYS_INLINE double get(double u, double v) const + ALWAYS_INLINE double get_merger_min_distance() const override + { + return std::max(m_radius_x, std::max(m_radius_y, m_radius_z)); + }; + + ALWAYS_INLINE double get(double u, double v) const override { double sin_u = sin(u); double cos_u = cos(u); @@ -153,9 +166,15 @@ class AHInitialGuessEllipsoid : public AHInitialGuessDefault sqrt(sin_u * sin_u * cos_v * cos_v / (m_radius_x * m_radius_x) + sin_u * sin_u * sin_v * sin_v / (m_radius_y * m_radius_y) + cos_u * cos_u / (m_radius_z * m_radius_z)); - // std::cout << u << "|" << v << "|" << radius << std::endl; + // pout() << u << "|" << v << "|" << radius << std::endl; return radius; } + ALWAYS_INLINE double get_merger_contribution(double u, + double v) const override + { + return get_merger_min_distance(); // use constant + // return get(u, v); // use ellipsoid + } #elif CH_SPACEDIM == 2 AHInitialGuessEllipsoid(double a_radius_x, double a_radius_y) { @@ -166,19 +185,31 @@ class AHInitialGuessEllipsoid : public AHInitialGuessDefault { m_radius_x = a_radius_x; m_radius_y = a_radius_y; - pout() << "Setting Initial Guess to ellipsoid=(" << m_radius_x << ", " + pout() << "Setting Initial Guess to ellipsoid = (" << m_radius_x << ", " << m_radius_y << ")" << std::endl; } - ALWAYS_INLINE double get(double u) const + ALWAYS_INLINE double get_merger_min_distance() const override + { + return std::max(m_radius_x, m_radius_y); + }; + + ALWAYS_INLINE double get(double u) const override { double sin_u = sin(u); double cos_u = cos(u); // u = 0 corresponds to (x,y)=(-1,0) // u = pi/2 corresponds to (x,y)=(0,1) // u = pi corresponds to (x,y)=(1,0) - return 1. / sqrt(cos_u * cos_u / (m_radius_x * m_radius_x) + - sin_u * sin_u / (m_radius_y * m_radius_y)); + double radius = 1. / sqrt(cos_u * cos_u / (m_radius_x * m_radius_x) + + sin_u * sin_u / (m_radius_y * m_radius_y)); + // pout() << u << "|" << radius << std::endl; + return radius; + } + ALWAYS_INLINE double get_merger_contribution(double u) const override + { + return get_merger_min_distance(); // use constant + // return get(u); // use ellipsoid } #endif }; diff --git a/Source/ApparentHorizonFinder/AHParams.hpp b/Source/ApparentHorizonFinder/AHParams.hpp index b0215d7f6..e30bb64de 100644 --- a/Source/ApparentHorizonFinder/AHParams.hpp +++ b/Source/ApparentHorizonFinder/AHParams.hpp @@ -142,7 +142,7 @@ template struct AHParams_t double merger_search_factor; // see note above (default is 1) //! initial guess for merger is 'merger_pre_factor * 4. * //! (AH_initial_guess_1 + AH_initial_guess_2)' - //! set to somethig bigger to avoid finding the inner AH + //! set to something bigger to avoid finding the inner AH double merger_pre_factor; // see note above (default to 1.) typename AHFunction::params func_params; diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.hpp index 00e101176..fca6d588b 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.hpp @@ -24,7 +24,7 @@ template class ApparentHorizon ApparentHorizon( const AHInterpolation &a_interp, //!< Geometry class to exchange data const AHInitialGuessPtr - a_initial_guess, //!< Initial guess for radius (or whatever + &a_initial_guess, //!< Initial guess for radius (or whatever //!< coordinate you're solving for) const AHParams &a_params, //!< set of AH parameters const std::string &a_stats = diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp index a2215a2e6..93c37f952 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp @@ -23,7 +23,7 @@ template ApparentHorizon::ApparentHorizon( - const AHInterpolation &a_interp, const AHInitialGuessPtr a_initial_guess, + const AHInterpolation &a_interp, const AHInitialGuessPtr &a_initial_guess, const AHParams &a_params, const std::string &a_stats, const std::string &a_coords, bool solve_first_step) : m_params(a_params), diff --git a/Source/ApparentHorizonFinder/PETScAHSolver.hpp b/Source/ApparentHorizonFinder/PETScAHSolver.hpp index 683955257..902185204 100644 --- a/Source/ApparentHorizonFinder/PETScAHSolver.hpp +++ b/Source/ApparentHorizonFinder/PETScAHSolver.hpp @@ -29,7 +29,7 @@ template class PETScAHSolver public: //! AH that finds the zero of expansion PETScAHSolver(const AHInterpolation &a_interp, - const AHInitialGuessPtr a_initial_guess, + const AHInitialGuessPtr &a_initial_guess, const AHParams &a_params); ~PETScAHSolver(); @@ -56,7 +56,7 @@ template class PETScAHSolver // must be called at the end of the function that called 'get_dmda_arr_t' void restore_dmda_arr_t(Vec &localF, dmda_arr_t &in); - const AHInitialGuessPtr get_initial_guess() const; + const AHInitialGuessPtr &get_initial_guess() const; void reset_initial_guess(); const std::array &get_origin() const; diff --git a/Source/ApparentHorizonFinder/PETScAHSolver.impl.hpp b/Source/ApparentHorizonFinder/PETScAHSolver.impl.hpp index 096784a64..ad82297ff 100644 --- a/Source/ApparentHorizonFinder/PETScAHSolver.impl.hpp +++ b/Source/ApparentHorizonFinder/PETScAHSolver.impl.hpp @@ -21,7 +21,7 @@ template PETScAHSolver::PETScAHSolver( - const AHInterpolation &a_interp, const AHInitialGuessPtr a_initial_guess, + const AHInterpolation &a_interp, const AHInitialGuessPtr &a_initial_guess, const AHParams &a_params) : m_interp(a_interp), m_interp_plus(a_interp), m_interp_minus(a_interp), @@ -484,7 +484,7 @@ void PETScAHSolver::set_origin( } template -const AHInitialGuessPtr +const AHInitialGuessPtr & PETScAHSolver::get_initial_guess() const { return m_initial_guess; From cb7863a77f218e7a849e8e5264fc32956531e173 Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Thu, 5 Aug 2021 17:40:13 +0100 Subject: [PATCH 35/92] Remove irreducible mass from 2D runs and fix restart bug for merger AHs Bug: when restarting from an early checkpoint file, SmallDataIO deletes extra lines in the stats file via 'remove_duplicate_time_data'. For mergers that do not necessarily start right after the restart, calling this does not work and the lines were never deleted. Solution: force SmallDataIO to think 'restart_time' is the current time when mergers solve for the first time, and hence allowing to remove duplicate lines. --- .../ApparentHorizonFinder/AHInitialGuess.hpp | 9 ++-- .../ApparentHorizonFinder/ApparentHorizon.hpp | 20 ++++---- .../ApparentHorizon.impl.hpp | 47 ++++++++++++------- 3 files changed, 47 insertions(+), 29 deletions(-) diff --git a/Source/ApparentHorizonFinder/AHInitialGuess.hpp b/Source/ApparentHorizonFinder/AHInitialGuess.hpp index 5dc35a037..109e6c59b 100644 --- a/Source/ApparentHorizonFinder/AHInitialGuess.hpp +++ b/Source/ApparentHorizonFinder/AHInitialGuess.hpp @@ -59,7 +59,8 @@ class AHInitialGuessConstant : public AHInitialGuessDefault void set_params(double a_initial_guess) { m_initial_guess = a_initial_guess; - pout() << "Setting Initial Guess to f=" << m_initial_guess << std::endl; + pout() << "Setting Original Guess to f=" << m_initial_guess + << std::endl; } #if CH_SPACEDIM == 3 @@ -146,7 +147,7 @@ class AHInitialGuessEllipsoid : public AHInitialGuessDefault m_radius_x = a_radius_x; m_radius_y = a_radius_y; m_radius_z = a_radius_z; - pout() << "Setting Initial Guess to ellipsoid=(" << m_radius_x << ", " + pout() << "Setting Original Guess to ellipsoid=(" << m_radius_x << ", " << m_radius_y << ", " << m_radius_z << ")" << std::endl; } @@ -185,8 +186,8 @@ class AHInitialGuessEllipsoid : public AHInitialGuessDefault { m_radius_x = a_radius_x; m_radius_y = a_radius_y; - pout() << "Setting Initial Guess to ellipsoid = (" << m_radius_x << ", " - << m_radius_y << ")" << std::endl; + pout() << "Setting Original Guess to ellipsoid = (" << m_radius_x + << ", " << m_radius_y << ")" << std::endl; } ALWAYS_INLINE double get_merger_min_distance() const override diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.hpp index fca6d588b..9cb89fc7d 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.hpp @@ -83,21 +83,22 @@ template class ApparentHorizon double calculate_area(); //!< calculate AH area -#if CH_SPACEDIM == 3 - Tensor<1, double> calculate_angular_momentum_J(); //!< calculate spin, ONLY - //!< FOR 3D - double calculate_spin_dimensionless( - double a_area); //!< calculate spin with 'z' direction, ONLY FOR 3D -#endif // estimate based on area and angular momentum J ALWAYS_INLINE double calculate_mass(double area, double J_norm) { return sqrt(area / (16. * M_PI) + J_norm * J_norm * 4. * M_PI / area); } +#if CH_SPACEDIM == 3 + Tensor<1, double> calculate_angular_momentum_J(); //!< calculate spin, ONLY + //!< FOR 3D + double calculate_spin_dimensionless( + double a_area); //!< calculate spin with 'z' direction, ONLY FOR 3D + ALWAYS_INLINE double calculate_irreducible_mass(double area) { return calculate_mass(area, 0.); } +#endif void calculate_minmax_F() const; void calculate_average_F() const; @@ -125,7 +126,7 @@ template class ApparentHorizon // variables private: - bool m_printed_once; + bool m_printed_once, m_printed_after_restart; int m_converged; //!< flag saying if PETSc has converged the last 'N' times //!<(read using 'get_converged()') @@ -149,8 +150,11 @@ template class ApparentHorizon std::array m_integration_methods; // just to save the result temporarily at each iteration - double m_area, m_spin, m_mass, m_irreducible_mass, m_spin_z_alt; + double m_area, m_mass; +#if CH_SPACEDIM == 3 + double m_spin, m_irreducible_mass, m_spin_z_alt; Tensor<1, double> m_dimensionless_spin_vector; +#endif // prevents resetting the origin when the user externally did 'set_origin' // before 'solve' diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp index 93c37f952..b762b5195 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp @@ -32,6 +32,8 @@ ApparentHorizon::ApparentHorizon( m_printed_once(false), + m_printed_after_restart(false), + m_converged(0), m_has_been_found(false), @@ -52,8 +54,11 @@ ApparentHorizon::ApparentHorizon( #endif }), - m_area(NAN), m_spin(NAN), m_mass(NAN), m_irreducible_mass(NAN), - m_spin_z_alt(NAN), m_dimensionless_spin_vector({NAN}), + m_area(NAN), m_mass(NAN), +#if CH_SPACEDIM == 3 + m_spin(NAN), m_irreducible_mass(NAN), m_spin_z_alt(NAN), + m_dimensionless_spin_vector({NAN}), +#endif origin_already_updated(false), @@ -347,11 +352,11 @@ void ApparentHorizon::solve(double a_dt, #if CH_SPACEDIM == 3 Tensor<1, double> J = calculate_angular_momentum_J(); double J_norm = sqrt(J[0] * J[0] + J[1] * J[1] + J[2] * J[2]); -#elif CH_SPACEDIM == 2 - double J_norm = 0.; -#endif m_mass = calculate_mass(m_area, J_norm); m_irreducible_mass = calculate_irreducible_mass(m_area); +#elif CH_SPACEDIM == 2 + m_mass = calculate_mass(m_area, 0.); +#endif #if CH_SPACEDIM == 3 m_spin = J_norm / m_mass; @@ -365,11 +370,10 @@ void ApparentHorizon::solve(double a_dt, pout() << "mass = " << m_mass << endl; #if CH_SPACEDIM == 3 pout() << "spin = " << m_spin << endl; -#endif + if (m_params.verbose > AHParams::MIN) { pout() << "irreducible mass = " << m_irreducible_mass << endl; -#if CH_SPACEDIM == 3 pout() << "dimensionless spin vector = (" << m_dimensionless_spin_vector[0] << ", " << m_dimensionless_spin_vector[1] << ", " @@ -430,6 +434,15 @@ void ApparentHorizon::write_outputs( pout() << "Printing statistics and coordinates." << std::endl; } + if (!m_printed_after_restart && m_printed_once) + { + // this forces 'remove_duplicate_time_data' to work (necessary for + // example with mergers, when the merger doesn't start right at + // restart but we still need to do 'remove_duplicate_time_data') + a_restart_time = a_time; + m_printed_after_restart = true; + } + // write stats double fake_dt = a_dt * m_params.solve_interval * m_params.print_interval; @@ -443,9 +456,9 @@ void ApparentHorizon::write_outputs( CH_assert(CH_SPACEDIM == 3 || CH_SPACEDIM == 2); // first '1' corresponds to 'step' - // area+mass+irred.mass+spin+spin_vec+spin_alt OR area+mass+irred.mass + // area+mass+irred.mass+spin+spin_vec+spin_alt OR area+mass // in 2D Cartoon OR only area for other cases (e.g. 4D -> 2D cartoon) - int stats = (GR_SPACEDIM == 3 ? (CH_SPACEDIM == 3 ? 8 : 3) : 1); + int stats = (GR_SPACEDIM == 3 ? (CH_SPACEDIM == 3 ? 8 : 2) : 1); std::vector values(1 + stats + CH_SPACEDIM * (1 + m_params.track_center)); @@ -458,8 +471,8 @@ void ApparentHorizon::write_outputs( values[idx++] = m_area; #if GR_SPACEDIM == 3 // GR_SPACEDIM, not CH_SPACEDIM !!! values[idx++] = m_mass; - values[idx++] = m_irreducible_mass; #if CH_SPACEDIM == 3 + values[idx++] = m_irreducible_mass; values[idx++] = m_spin; FOR(a) { values[idx++] = m_dimensionless_spin_vector[a]; } values[idx++] = m_spin_z_alt; @@ -482,8 +495,8 @@ void ApparentHorizon::write_outputs( headers[idx++] = "area"; #if GR_SPACEDIM == 3 // GR_SPACEDIM, not CH_SPACEDIM !!! headers[idx++] = "mass"; - headers[idx++] = "irreducible mass"; #if CH_SPACEDIM == 3 + headers[idx++] = "irreducible mass"; headers[idx++] = "spin"; headers[idx++] = "dimless spin-x"; headers[idx++] = "dimless spin-y"; @@ -758,8 +771,7 @@ void ApparentHorizon::restart( #else // 3D -> 2D Cartoon method bool was_center_tracked = - (cols > 5 + CH_SPACEDIM); // 5 for time + file + area + mass + - // irreducible mass + (cols > 4 + CH_SPACEDIM); // 4 for time + file + area + mass #endif #else bool was_center_tracked = @@ -829,8 +841,7 @@ void ApparentHorizon::restart( // skip i=0, this one is directly in the file and is done after for (int i = 1; i < m_old_centers.size(); ++i) { - double time = current_time - current_solve_dt * i; - double index = idx - (current_time - time) / old_print_dt; + double index = idx - (current_solve_dt * i) / old_print_dt; if (index >= last_nan_idx) old_centers_time_index.push_back(index); @@ -846,8 +857,10 @@ void ApparentHorizon::restart( << std::endl; else pout() - << "Old AH time step is different than current one " - "(this might happen if 'AH_print_interval != " + << "Old AH time step, " << old_print_dt + << ", is different than current one, " + << current_solve_dt + << " (this might happen if 'AH_print_interval != " "1').\nRecovering " << old_centers_time_index.size() << " old centers from interpolation, for accurate " From 170af900023b5862cb4ce1f00b71cb8ebec3699b Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Thu, 5 Aug 2021 18:16:06 +0100 Subject: [PATCH 36/92] Add AH Linear Momentum calculation --- Examples/SingleBH/Main_SingleBH.cpp | 3 + Examples/SingleBH/params.txt | 12 +- .../ApparentHorizonFinder/ApparentHorizon.hpp | 39 +- .../ApparentHorizon.impl.hpp | 387 ++++++++++-------- 4 files changed, 243 insertions(+), 198 deletions(-) diff --git a/Examples/SingleBH/Main_SingleBH.cpp b/Examples/SingleBH/Main_SingleBH.cpp index 6ae9196cf..ce3c7592c 100644 --- a/Examples/SingleBH/Main_SingleBH.cpp +++ b/Examples/SingleBH/Main_SingleBH.cpp @@ -25,6 +25,9 @@ int runGRChombo(int argc, char *argv[]) GRParmParse pp(argc - 2, argv + 2, NULL, in_file); SimulationParameters sim_params(pp); + if (sim_params.just_check_params) + return 0; + // The line below selects the problem that is simulated // (To simulate a different problem, define a new child of AMRLevel // and an associated LevelFactory) diff --git a/Examples/SingleBH/params.txt b/Examples/SingleBH/params.txt index ae68c5e28..b340baa22 100644 --- a/Examples/SingleBH/params.txt +++ b/Examples/SingleBH/params.txt @@ -13,10 +13,10 @@ plot_prefix = SingleBHPlot_ # restart_file = SingleBH_000004.3d.hdf5 # HDF5files are written every dt = L/N*dt_multiplier*checkpoint_interval -checkpoint_interval = 2 +checkpoint_interval = 20 # set to 0 to turn off plot files (except at t=0 and t=stop_time) # set to -1 to never ever print plotfiles -plot_interval = 1 +plot_interval = 10 num_plot_vars = 2 plot_vars = chi Ham @@ -38,7 +38,7 @@ print_progress_only_to_rank_0 = 1 mass = 1. offset = 0. 0. 0. -momentum = 0.5 0. 0. # speed if using the IsotropicBoostBH ID +momentum = 0.2 0.0 0.0 # speed if using the IsotropicBoostBH ID ################################################# # Grid parameters @@ -112,8 +112,8 @@ nonzero_asymptotic_values = 1.0 1.0 1.0 1.0 1.0 # dt will be dx*dt_multiplier on each grid level dt_multiplier = 0.25 -# stop_time = 10.0 -max_steps = 10 +stop_time = 0.0 +# max_steps = 10 # Spatial derivative order (only affects CCZ4 RHS) max_spatial_derivative_order = 4 # can be 4 or 6 @@ -165,6 +165,8 @@ AH_print_interval = 1 # AH_verbose = 1 # AH_expansion_radius_power = 1. +AH_SNES_max_iterations = 500 + # AH_initial_guess = 0.5 # AH_use_ellipsoid = 1 # for IsotropicBoostedBH ID diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.hpp index 9cb89fc7d..9a18095d2 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.hpp @@ -12,6 +12,8 @@ #include "PETScAHSolver.hpp" // Class to manage ApparentHorizon for 2+1D and 3+1D simulations +// (has only been implemented for 3+1 spacetimes, 3+1 cartoon-reduced to 2+1 and +// 4+1 cartoon-reduced to 2+1) //! AHFunction defines the optimizing function (see AHFunction.hpp for //! expansion example calculation) template class ApparentHorizon @@ -81,23 +83,35 @@ template class ApparentHorizon //!< based on the last output file and the //!< origin based on the stats file - double calculate_area(); //!< calculate AH area + // compute area, linear momentum and angular momentum of BH (P only for 3D, + // J only for 3D without cartoon) + void calculate_ah_quantities(double &area +#if GR_SPACEDIM == 3 // GR_SPACEDIM, not CH_SPACEDIM !!! + , + Tensor<1, double> &P +#if CH_SPACEDIM == 3 + , + Tensor<1, double> &J +#endif +#endif + ); +#if GR_SPACEDIM == 3 // GR_SPACEDIM, not CH_SPACEDIM !!! // estimate based on area and angular momentum J - ALWAYS_INLINE double calculate_mass(double area, double J_norm) + ALWAYS_INLINE static double calculate_mass(double area, double J_norm) { return sqrt(area / (16. * M_PI) + J_norm * J_norm * 4. * M_PI / area); } -#if CH_SPACEDIM == 3 - Tensor<1, double> calculate_angular_momentum_J(); //!< calculate spin, ONLY - //!< FOR 3D - double calculate_spin_dimensionless( - double a_area); //!< calculate spin with 'z' direction, ONLY FOR 3D - ALWAYS_INLINE double calculate_irreducible_mass(double area) +#if CH_SPACEDIM == 3 + ALWAYS_INLINE static double calculate_irreducible_mass(double area) { return calculate_mass(area, 0.); } + + double calculate_spin_dimensionless( + double a_area); //!< calculate spin with 'z' direction, ONLY FOR 3D +#endif #endif void calculate_minmax_F() const; @@ -150,10 +164,13 @@ template class ApparentHorizon std::array m_integration_methods; // just to save the result temporarily at each iteration - double m_area, m_mass; + double m_area; +#if GR_SPACEDIM == 3 + double m_mass, m_linear_momentum_P_norm; #if CH_SPACEDIM == 3 - double m_spin, m_irreducible_mass, m_spin_z_alt; - Tensor<1, double> m_dimensionless_spin_vector; + double m_irreducible_mass, m_spin, m_spin_z_alt; + Tensor<1, double> m_dimensionless_spin_vector, m_linear_momentum_P; +#endif #endif // prevents resetting the origin when the user externally did 'set_origin' diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp index b762b5195..171361934 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp @@ -54,10 +54,13 @@ ApparentHorizon::ApparentHorizon( #endif }), - m_area(NAN), m_mass(NAN), + m_area(NAN), +#if GR_SPACEDIM == 3 + m_mass(NAN), m_linear_momentum_P_norm(NAN), #if CH_SPACEDIM == 3 - m_spin(NAN), m_irreducible_mass(NAN), m_spin_z_alt(NAN), - m_dimensionless_spin_vector({NAN}), + m_irreducible_mass(NAN), m_spin(NAN), m_spin_z_alt(NAN), + m_dimensionless_spin_vector({NAN}), m_linear_momentum_P({NAN}), +#endif #endif origin_already_updated(false), @@ -313,7 +316,7 @@ void ApparentHorizon::solve(double a_dt, { if (m_params.verbose > AHParams::SOME) { - pout() << "In [ApparentHorizon::solve::post-solving]" << endl; + pout() << "In [ApparentHorizon::solve::post-solving]" << std::endl; } CH_TIME("ApparentHorizon::solve::post-solving"); @@ -347,15 +350,27 @@ void ApparentHorizon::solve(double a_dt, origin_already_updated = false; - m_area = calculate_area(); #if GR_SPACEDIM == 3 // GR_SPACEDIM, not CH_SPACEDIM !!! #if CH_SPACEDIM == 3 - Tensor<1, double> J = calculate_angular_momentum_J(); + Tensor<1, double> J; + calculate_ah_quantities(m_area, m_linear_momentum_P, J); + m_linear_momentum_P_norm = + sqrt(m_linear_momentum_P[0] * m_linear_momentum_P[0] + + m_linear_momentum_P[1] * m_linear_momentum_P[1] + + m_linear_momentum_P[2] * m_linear_momentum_P[2]); double J_norm = sqrt(J[0] * J[0] + J[1] * J[1] + J[2] * J[2]); m_mass = calculate_mass(m_area, J_norm); m_irreducible_mass = calculate_irreducible_mass(m_area); #elif CH_SPACEDIM == 2 - m_mass = calculate_mass(m_area, 0.); + Tensor<1, double> P; + calculate_ah_quantities(m_area, P); + // assume that P_y = 0 in Cartoon code (if we calculate it it + // will not be 0, but that is because in the Cartoon code the + // AH is only half an horizon, so the P_y gets wrongly + // calculated) + m_linear_momentum_P_norm = P[0]; + double J_norm = 0.; + m_mass = calculate_mass(m_area, J_norm); #endif #if CH_SPACEDIM == 3 @@ -365,26 +380,42 @@ void ApparentHorizon::solve(double a_dt, m_spin_z_alt = calculate_spin_dimensionless(m_area); #endif +#else // GR_SPACEDIM != 3 + calculate_ah_quantities(m_area); +#endif + if (m_params.verbose > AHParams::NONE) { - pout() << "mass = " << m_mass << endl; +#if GR_SPACEDIM == 3 // GR_SPACEDIM, not CH_SPACEDIM !!! + pout() << "mass = " << m_mass << std::endl; #if CH_SPACEDIM == 3 - pout() << "spin = " << m_spin << endl; - + pout() << "spin = " << m_spin << std::endl; +#endif if (m_params.verbose > AHParams::MIN) { - pout() << "irreducible mass = " << m_irreducible_mass << endl; +#if CH_SPACEDIM == 3 + pout() << "irreducible mass = " << m_irreducible_mass + << std::endl; pout() << "dimensionless spin vector = (" << m_dimensionless_spin_vector[0] << ", " << m_dimensionless_spin_vector[1] << ", " - << m_dimensionless_spin_vector[2] << ")" << endl; + << m_dimensionless_spin_vector[2] << ")" << std::endl; pout() << "dimensionless spin in z (from equator-length " "integral) = " - << m_spin_z_alt << endl; + << m_spin_z_alt << std::endl; +#endif + pout() << "linear momentum norm |P| = " + << m_linear_momentum_P_norm << std::endl; +#if CH_SPACEDIM == 3 + pout() << "linear momentum P = (" << m_linear_momentum_P[0] + << ", " << m_linear_momentum_P[1] << ", " + << m_linear_momentum_P[2] << ")" << std::endl; #endif } - } +#else // GR_SPACEDIM != 3 + pout() << "area = " << m_area << std::endl; #endif + } // reset min and max F, to force re-calculation m_max_F = 0.; @@ -408,7 +439,7 @@ void ApparentHorizon::solve(double a_dt, if (m_params.verbose > AHParams::SOME) { - pout() << "ApparentHorizon::solve finished successfully!" << endl; + pout() << "ApparentHorizon::solve finished successfully!" << std::endl; } } @@ -456,9 +487,10 @@ void ApparentHorizon::write_outputs( CH_assert(CH_SPACEDIM == 3 || CH_SPACEDIM == 2); // first '1' corresponds to 'step' - // area+mass+irred.mass+spin+spin_vec+spin_alt OR area+mass - // in 2D Cartoon OR only area for other cases (e.g. 4D -> 2D cartoon) - int stats = (GR_SPACEDIM == 3 ? (CH_SPACEDIM == 3 ? 8 : 2) : 1); + // area+mass+irred.mass+spin+spin_vec+spin_alt+|P|+ P vec OR + // area+mass+|P| in 2D Cartoon OR only area for other cases (e.g. + // 4D -> 2D cartoon) + int stats = (GR_SPACEDIM == 3 ? (CH_SPACEDIM == 3 ? 12 : 3) : 1); std::vector values(1 + stats + CH_SPACEDIM * (1 + m_params.track_center)); @@ -476,8 +508,13 @@ void ApparentHorizon::write_outputs( values[idx++] = m_spin; FOR(a) { values[idx++] = m_dimensionless_spin_vector[a]; } values[idx++] = m_spin_z_alt; +#endif + values[idx++] = m_linear_momentum_P_norm; +#if CH_SPACEDIM == 3 + FOR(a) { values[idx++] = m_linear_momentum_P[a]; } #endif #endif + for (int i = 0; i < CH_SPACEDIM; ++i) values[idx++] = origin[i]; if (m_params.track_center) @@ -503,6 +540,13 @@ void ApparentHorizon::write_outputs( headers[idx++] = "dimless spin-z"; headers[idx++] = "dimless spin-z-alt"; #endif + headers[idx++] = "linear mom. |P|"; +#if CH_SPACEDIM == 3 + headers[idx++] = "linear mom. Px"; + headers[idx++] = "linear mom. Py"; + headers[idx++] = "linear mom. Pz"; +#endif + #endif headers[idx++] = "origin_x"; headers[idx++] = "origin_y"; @@ -765,13 +809,14 @@ void ApparentHorizon::restart( #if CH_SPACEDIM == 3 bool was_center_tracked = (cols > - 10 + - CH_SPACEDIM); // 10 for time + file + area + mass + irreducible - // + spin + spin_vector + spin_z_alt + 14 + + CH_SPACEDIM); // 14 for time + file + area + mass + irreducible + // + spin + spin_vector + spin_z_alt + |P| + P + // vec #else // 3D -> 2D Cartoon method bool was_center_tracked = - (cols > 4 + CH_SPACEDIM); // 4 for time + file + area + mass + (cols > 5 + CH_SPACEDIM); // 5 for time + file + area + mass + |P| #endif #else bool was_center_tracked = @@ -1295,7 +1340,6 @@ ApparentHorizon::calculate_spin_dimensionless( // 'solver.m_interp.break_interpolation_loop()' if (solver.m_interp.keep_interpolating_if_inactive()) { - int idx = 0; Vec localF; @@ -1374,27 +1418,47 @@ ApparentHorizon::calculate_spin_dimensionless( } #endif -#if CH_SPACEDIM == 3 -// ONLY FOR 3D template -Tensor<1, double> -ApparentHorizon::calculate_angular_momentum_J() +void ApparentHorizon::calculate_ah_quantities( + double &area +#if GR_SPACEDIM == 3 + , + Tensor<1, double> &P +#if CH_SPACEDIM == 3 + , + Tensor<1, double> &J +#endif +#endif +) { - CH_assert(CH_SPACEDIM == 3); - CH_TIME("ApparentHorizon::calculate_angular_momentum_J"); + CH_TIME("ApparentHorizon::calculate_ah_quantities"); if (!get_converged()) - return {NAN}; + { + area = NAN; +#if GR_SPACEDIM == 3 + P = {NAN}; +#if CH_SPACEDIM == 3 + J = {NAN}; +#endif +#endif + return; + } - Tensor<1, double> J; - Tensor<1, double> integrals = {0.}; // temporary, but defined outside to be - // in scope for non-PETSc processes + // temporary, but defined outside to be + // in scope for non-PETSc processes + double integral_area = 0.; +#if GR_SPACEDIM == 3 + Tensor<1, double> integrals_P = {0.}; +#if CH_SPACEDIM == 3 + Tensor<1, double> integrals_J = {0.}; +#endif +#endif // PETSc processes go inside 'if', others "wait" until 'if' gets to // 'solver.m_interp.break_interpolation_loop()' if (solver.m_interp.keep_interpolating_if_inactive()) { - int idx = 0; Vec localF; @@ -1405,12 +1469,27 @@ ApparentHorizon::calculate_angular_momentum_J() const std::array ¢er = get_center(); +#if CH_SPACEDIM == 3 for (int v = solver.m_vmin; v < solver.m_vmax; ++v) +#endif { - Tensor<1, double> inner_integral = {0.}; + + double inner_integral_area = 0.; +#if GR_SPACEDIM == 3 + Tensor<1, double> inner_integral_P = {0.}; +#if CH_SPACEDIM == 3 + Tensor<1, double> inner_integral_J = {0.}; +#endif +#endif + for (int u = solver.m_umin; u < solver.m_umax; ++u) { - AHDerivData deriv = solver.diff(in, u, v); + AHDerivData deriv = solver.diff(in, u +#if CH_SPACEDIM == 3 + , + v +#endif + ); const auto geometric_data = solver.m_interp.get_geometry_data(idx); const auto data = solver.m_interp.get_data(idx); @@ -1419,6 +1498,7 @@ ApparentHorizon::calculate_angular_momentum_J() solver.m_interp.get_cartesian_coords(idx); AHFunction func(data, coords, coords_cart); Tensor<2, double> g = func.get_metric(); +#if GR_SPACEDIM == 3 Tensor<2, double> K = func.get_extrinsic_curvature(); Tensor<1, double> s_L = func.get_level_function_derivative(geometric_data, deriv); @@ -1427,9 +1507,22 @@ ApparentHorizon::calculate_angular_momentum_J() Tensor<1, double> coords_cart_centered; FOR(i) { coords_cart_centered[i] = coords_cart[i] - center[i]; } + // Linear Momentum P + Tensor<1, double> p_integrand = {0.}; + // double directions[3][3] = { + // {1., 0., 0.}, {0., 1., 0.}, {0., 0., 1.}}; + // FOR(a, b, c) + // { + // p_integrand[c] += directions[c][a] * S_U[b] * K[a][b]; + // } + // simplify the commented above to: + FOR(a, b) { p_integrand[a] += S_U[b] * K[a][b]; } + +#if CH_SPACEDIM == 3 + // Angular Momentum J Tensor<1, double> spin_integrand = {0.}; double directions[3][3] = { - {0, -coords_cart_centered[2], coords_cart_centered[1]}, + {0., -coords_cart_centered[2], coords_cart_centered[1]}, {coords_cart_centered[2], 0., -coords_cart_centered[0]}, {-coords_cart_centered[1], coords_cart_centered[0], 0.}}; FOR(a, b, c) @@ -1440,7 +1533,8 @@ ApparentHorizon::calculate_angular_momentum_J() // epsilon[c][j][a] * x[j], just like for ADM Momentum) spin_integrand[c] += directions[c][a] * S_U[b] * K[a][b]; } - +#endif +#endif // now calculate sqrt(-g) area element // Calculate Jacobian matrix for transformation from Cartesian @@ -1450,7 +1544,9 @@ ApparentHorizon::calculate_angular_momentum_J() { Jac[0][k] = geometric_data.dxdf[k]; Jac[1][k] = geometric_data.dxdu[k]; +#if CH_SPACEDIM == 3 Jac[2][k] = geometric_data.dxdv[k]; +#endif } // Now do the coordinate transformation @@ -1467,6 +1563,7 @@ ApparentHorizon::calculate_angular_momentum_J() g_horizon[0][0] = g_spherical[1][1] + g_spherical[0][0] * deriv.duF * deriv.duF + 2.0 * g_spherical[0][1] * deriv.duF; +#if CH_SPACEDIM == 3 g_horizon[1][1] = g_spherical[2][2] + g_spherical[0][0] * deriv.dvF * deriv.dvF + 2.0 * g_spherical[0][2] * deriv.dvF; @@ -1475,6 +1572,7 @@ ApparentHorizon::calculate_angular_momentum_J() g_spherical[0][0] * deriv.duF * deriv.dvF + g_spherical[0][1] * deriv.dvF + g_spherical[0][2] * deriv.duF; +#endif double det = TensorAlgebra::compute_determinant(g_horizon); @@ -1482,157 +1580,73 @@ ApparentHorizon::calculate_angular_momentum_J() u, m_params.num_points_u, solver.m_interp.get_coord_system().is_u_periodic()); - FOR(a) - { - double element = spin_integrand[a] / (8. * M_PI) * - sqrt(det) * weight * solver.m_du; - - // hack for the poles (theta=0,\pi) - // where nans appear in 'spin_integrand', but - // 'det' should be 0 - if (std::isnan(element)) - element = 0.; - - inner_integral[a] += element; - } - idx++; - } - double weight = m_integration_methods[1].weight( - v, m_params.num_points_v, - solver.m_interp.get_coord_system().is_v_periodic()); - FOR(a) { integrals[a] += weight * solver.m_dv * inner_integral[a]; } - } - - solver.restore_dmda_arr_t(localF, in); - solver.m_interp.break_interpolation_loop(); - } - - // reduction across all Chombo processes (note that 'integral' is 0 for - // non-PETSc processes) because SmallDataIO uses rank 0 to write this - // ensures rank 0 will have 'integral' (even though for now the PETSc - // processes always include rank 0) -#ifdef CH_MPI - MPI_Allreduce(&integrals, &J, GR_SPACEDIM, MPI_DOUBLE, MPI_SUM, - Chombo_MPI::comm); -#else // serial - FOR(a) { J[a] = integrals[a]; } -#endif + double element_area = sqrt(det) * weight * solver.m_du; - return J; -} +// assume a (GR_SPACEDIM - CH_SPACEDIM)-sphere leftover +#if GR_SPACEDIM != CH_SPACEDIM + double n_sphere = (GR_SPACEDIM - CH_SPACEDIM); + double element_hd = pow(sqrt(func.get_metric_hd()) * + coords_cart[CH_SPACEDIM - 1], + n_sphere); + element_area *= element_hd; #endif -template -double ApparentHorizon::calculate_area() -{ - CH_TIME("ApparentHorizon::calculate_area"); - - if (!get_converged()) - return NAN; - - double area; - double integral = - 0.; // temporary, but defined outside to exist for non-PETSc processes + inner_integral_area += element_area; - // PETSc processes go inside 'if', others "wait" until 'if' gets to - // 'solver.m_interp.break_interpolation_loop()' - if (solver.m_interp.keep_interpolating_if_inactive()) - { +#if GR_SPACEDIM == 3 + // Linear Momentum P + FOR(a) + { + double element_P = p_integrand[a] / (8. * M_PI) * + sqrt(det) * weight * solver.m_du; - int idx = 0; +// assume a (GR_SPACEDIM - CH_SPACEDIM)-sphere leftover +#if GR_SPACEDIM != CH_SPACEDIM + element_P *= element_hd; +#endif - Vec localF; - dmda_arr_t in; - solver.get_dmda_arr_t(localF, in); + // hack for the poles (theta=0,\pi) + // where nans appear in 'p_integrand', but + // 'det' should be 0 + if (std::isnan(element_P)) + element_P = 0.; - // solver.m_F is already set from solve + inner_integral_P[a] += element_P; + } #if CH_SPACEDIM == 3 - for (int v = solver.m_vmin; v < solver.m_vmax; ++v) -#endif - { - double inner_integral = 0.; - for (int u = solver.m_umin; u < solver.m_umax; ++u) - { - const auto geometric_data = - solver.m_interp.get_geometry_data(idx); - const auto data = solver.m_interp.get_data(idx); - const auto coords = solver.m_interp.get_coords(idx); - const auto coords_cart = - solver.m_interp.get_cartesian_coords(idx); - AHFunction func(data, coords, coords_cart); - Tensor<2, double> g = func.get_metric(); - - // Calculate Jacobian matrix for transformation from Cartesian - // to (f,u,v) coords - Tensor<2, double> Jac; - FOR(k) + // Angular Momentum J + FOR(a) { - Jac[0][k] = geometric_data.dxdf[k]; - Jac[1][k] = geometric_data.dxdu[k]; -#if CH_SPACEDIM == 3 - Jac[2][k] = geometric_data.dxdv[k]; -#endif - } + double element_J = spin_integrand[a] / (8. * M_PI) * + sqrt(det) * weight * solver.m_du; - AHDerivData deriv = solver.diff(in, u -#if CH_SPACEDIM == 3 - , - v -#endif - ); + // hack for the poles (theta=0,\pi) + // where nans appear in 'spin_integrand', but + // 'det' should be 0 + if (std::isnan(element_J)) + element_J = 0.; - // Now do the coordinate transformation - Tensor<2, double> g_spherical = {0.}; - FOR(i, j, k, l) - { - g_spherical[i][j] += Jac[i][k] * Jac[j][l] * g[k][l]; + inner_integral_J[a] += element_J; } - - // Construct the 2-metric on the horizon in (u,v) coords - // i.e. substitute df = (df/du)du + (df/dv)dv - // into the spherical metric - Tensor<2, double, CH_SPACEDIM - 1> g_horizon = {0.}; - g_horizon[0][0] = g_spherical[1][1] + - g_spherical[0][0] * deriv.duF * deriv.duF + - 2.0 * g_spherical[0][1] * deriv.duF; -#if CH_SPACEDIM == 3 - g_horizon[1][1] = g_spherical[2][2] + - g_spherical[0][0] * deriv.dvF * deriv.dvF + - 2.0 * g_spherical[0][2] * deriv.dvF; - g_horizon[0][1] = g_horizon[1][0] = - g_spherical[1][2] + - g_spherical[0][0] * deriv.duF * deriv.dvF + - g_spherical[0][1] * deriv.dvF + - g_spherical[0][2] * deriv.duF; #endif - - double det = TensorAlgebra::compute_determinant(g_horizon); - - double weight = m_integration_methods[0].weight( - u, m_params.num_points_u, - solver.m_interp.get_coord_system().is_u_periodic()); - - double element = sqrt(det) * weight * solver.m_du; - -// assume a (GR_SPACEDIM - CH_SPACEDIM)-sphere leftover -#if GR_SPACEDIM != CH_SPACEDIM - double n_sphere = (GR_SPACEDIM - CH_SPACEDIM); - element *= pow(sqrt(func.get_metric_hd()) * - coords_cart[CH_SPACEDIM - 1], - n_sphere); #endif - inner_integral += element; idx++; } #if CH_SPACEDIM == 3 double weight = m_integration_methods[1].weight( v, m_params.num_points_v, solver.m_interp.get_coord_system().is_v_periodic()); - integral += weight * solver.m_dv * inner_integral; + double weight_dv = weight * solver.m_dv; + integral_area += weight_dv * inner_integral_area; + FOR(a) { integrals_P[a] += weight_dv * inner_integral_P[a]; } + FOR(a) { integrals_J[a] += weight_dv * inner_integral_J[a]; } #elif CH_SPACEDIM == 2 - integral += inner_integral; + integral_area += inner_integral_area; +#if GR_SPACEDIM == 3 + FOR(a) { integrals_P[a] += inner_integral_P[a]; } +#endif #endif } @@ -1642,31 +1656,40 @@ double ApparentHorizon::calculate_area() // this is 2pi for n=1, 4pi for n=2, 2pi^2 for n=3, ... double n_sphere_coeff = 2. * std::pow(M_PI, (n_sphere + 1.) / 2.) / std::tgamma((n_sphere + 1.) / 2.); - integral *= n_sphere_coeff; + integral_area *= n_sphere_coeff; +#if GR_SPACEDIM == 3 + FOR(a) { integrals_P[a] *= n_sphere_coeff; } +#endif #endif solver.restore_dmda_arr_t(localF, in); solver.m_interp.break_interpolation_loop(); } -// reduction across all Chombo processes (note that 'area' is 0 for non-PETSc -// processes) because SmallDataIO uses rank 0 to write this ensures rank 0 will -// have the area (even though for now the PETSc processes always include rank 0) + // reduction across all Chombo processes (note that 'integral' is 0 for + // non-PETSc processes) because SmallDataIO uses rank 0 to write this + // ensures rank 0 will have 'integral' (even though for now the PETSc + // processes always include rank 0) #ifdef CH_MPI - // we want all the processes to have the area so that all use it in the - // spin calculation (which in reality is needed not for the output files, - // but only for the pout() prints) - MPI_Allreduce(&integral, &area, 1, MPI_DOUBLE, MPI_SUM, Chombo_MPI::comm); + MPI_Allreduce(&integral_area, &area, 1, MPI_DOUBLE, MPI_SUM, + Chombo_MPI::comm); +#if GR_SPACEDIM == 3 + MPI_Allreduce(&integrals_P, &P, GR_SPACEDIM, MPI_DOUBLE, MPI_SUM, + Chombo_MPI::comm); +#if CH_SPACEDIM == 3 + MPI_Allreduce(&integrals_J, &J, GR_SPACEDIM, MPI_DOUBLE, MPI_SUM, + Chombo_MPI::comm); +#endif +#endif #else // serial - area = integral; + area = integral_area; +#if GR_SPACEDIM == 3 + FOR(a) { P[a] = integrals_P[a]; } +#if CH_SPACEDIM == 3 + FOR(a) { J[a] = integrals_J[a]; } +#endif +#endif #endif - - if (m_params.verbose > AHParams::MIN) - { - pout() << "area = " << area << endl; - } - - return area; } template From 0b748689fd7f4050d1f637a775052a57acb36d06 Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Thu, 4 Nov 2021 20:53:25 +0000 Subject: [PATCH 37/92] Fix Weyl Imaginary part extraction bug --- Source/AMRInterpolator/SphericalExtraction.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/AMRInterpolator/SphericalExtraction.hpp b/Source/AMRInterpolator/SphericalExtraction.hpp index d9659eccf..5708cebd7 100644 --- a/Source/AMRInterpolator/SphericalExtraction.hpp +++ b/Source/AMRInterpolator/SphericalExtraction.hpp @@ -121,8 +121,8 @@ class ModeExtraction : public SurfaceExtraction // note that spin_Y_lm requires the coordinates with the center // at the origin double x = geom.get_grid_coord(dirs[0], r, theta, phi) - center[0]; - double y = geom.get_grid_coord(dirs[0], r, theta, phi) - center[1]; - double z = geom.get_grid_coord(dirs[0], r, theta, phi) - center[2]; + double y = geom.get_grid_coord(dirs[1], r, theta, phi) - center[1]; + double z = geom.get_grid_coord(dirs[2], r, theta, phi) - center[2]; SphericalHarmonics::Y_lm_t Y_lm = SphericalHarmonics::spin_Y_lm(x, y, z, es, el, em); auto function_here = a_function(a_data_here, r, theta, phi); From 403975d3fcb1179d51074b4e1e401fc4b5b16a0b Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Thu, 27 Jan 2022 10:25:00 +0000 Subject: [PATCH 38/92] Fix PetscInt and PetscReal types errors in ParmParse (only for some compilers - happened in MN4) --- Source/ApparentHorizonFinder/AHParams.hpp | 54 ++++++++++++++++++----- 1 file changed, 44 insertions(+), 10 deletions(-) diff --git a/Source/ApparentHorizonFinder/AHParams.hpp b/Source/ApparentHorizonFinder/AHParams.hpp index e30bb64de..bc20ad88c 100644 --- a/Source/ApparentHorizonFinder/AHParams.hpp +++ b/Source/ApparentHorizonFinder/AHParams.hpp @@ -44,27 +44,61 @@ struct PETSc_params void read_params(GRParmParse &pp) { + // variables to prevent ParmParse to complain about types (only happens + // in some clusters) + int tmp_int; + float tmp_float; if (pp.contains("AH_SNES_absolute_tolerance")) - pp.load("AH_SNES_absolute_tolerance", snes_atol); + { + pp.load("AH_SNES_absolute_tolerance", tmp_float); + snes_atol = tmp_float; + } if (pp.contains("AH_SNES_relative_tolerance")) - pp.load("AH_SNES_relative_tolerance", snes_rtol); + { + pp.load("AH_SNES_relative_tolerance", tmp_float); + snes_rtol = tmp_float; + } if (pp.contains("AH_SNES_step_change_tolerance")) - pp.load("AH_SNES_step_change_tolerance", snes_stol); + { + pp.load("AH_SNES_step_change_tolerance", tmp_float); + snes_stol = tmp_float; + } if (pp.contains("AH_SNES_max_iterations")) - pp.load("AH_SNES_max_iterations", snes_maxit); + { + pp.load("AH_SNES_max_iterations", tmp_int); + snes_maxit = tmp_int; + } if (pp.contains("AH_SNES_max_evaluations")) - pp.load("AH_SNES_max_evaluations", snes_maxf); + { + pp.load("AH_SNES_max_evaluations", tmp_int); + snes_maxf = tmp_int; + } if (pp.contains("AH_SNES_divergence_tolerance")) - pp.load("AH_SNES_divergence_tolerance", snes_divtol); + { + pp.load("AH_SNES_divergence_tolerance", tmp_float); + snes_divtol = tmp_float; + } if (pp.contains("AH_KSP_relative_tolerance")) - pp.load("AH_KSP_relative_tolerance", ksp_rtol); + { + pp.load("AH_KSP_relative_tolerance", tmp_float); + ksp_rtol = tmp_float; + } if (pp.contains("AH_KSP_absolute_tolerance")) - pp.load("AH_KSP_absolute_tolerance", ksp_abstol); + { + pp.load("AH_KSP_absolute_tolerance", tmp_float); + ksp_abstol = tmp_float; + } if (pp.contains("AH_KSP_divergence_tolerance")) - pp.load("AH_KSP_divergence_tolerance", ksp_dtol); + { + pp.load("AH_KSP_divergence_tolerance", tmp_float); + ksp_dtol = tmp_float; + } if (pp.contains("AH_KSP_max_evaluations")) - pp.load("AH_KSP_max_evaluations", ksp_maxits); + { + pp.load("AH_KSP_max_evaluations", tmp_int); + ksp_maxits = tmp_int; + } } }; From d25c16a69a440e141c000dd75daa33c0b5b8bc24 Mon Sep 17 00:00:00 2001 From: De Jong Date: Thu, 27 Jan 2022 12:27:30 +0000 Subject: [PATCH 39/92] MPI_Allreduce argument fix in calculate_average_F() --- Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp index 171361934..7f0f351d1 100644 --- a/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp +++ b/Source/ApparentHorizonFinder/ApparentHorizon.impl.hpp @@ -1872,7 +1872,7 @@ void ApparentHorizon::calculate_average_F() const Chombo_MPI::comm); MPI_Allreduce(&local_sum, &global_sum, 1, MPI_DOUBLE, MPI_SUM, Chombo_MPI::comm); - MPI_Allreduce(&global_sum_sq, &global_sum_sq, 1, MPI_DOUBLE, MPI_SUM, + MPI_Allreduce(&local_sum_sq, &global_sum_sq, 1, MPI_DOUBLE, MPI_SUM, Chombo_MPI::comm); #endif From 6526559fe4dcc8504a2d382d24fb0b97c962f51a Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Tue, 1 Mar 2022 20:42:32 +0000 Subject: [PATCH 40/92] Fix warning when using delete --- Source/ApparentHorizonFinder/AHFinder.impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/ApparentHorizonFinder/AHFinder.impl.hpp b/Source/ApparentHorizonFinder/AHFinder.impl.hpp index 1009c3a21..8bf5c2a2e 100644 --- a/Source/ApparentHorizonFinder/AHFinder.impl.hpp +++ b/Source/ApparentHorizonFinder/AHFinder.impl.hpp @@ -251,7 +251,7 @@ void AHFinder::solve(double a_dt, double a_time, } } - delete ah_solved; + delete[] ah_solved; } template From 1bef0463d54dc5e3f8fae6505cd2c784d642f8d9 Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Wed, 2 Mar 2022 01:14:48 +0000 Subject: [PATCH 41/92] Fix clang warnings --- Examples/ScalarField/ScalarFieldLevel.hpp | 16 +++++++++------- .../AMRInterpolator/SurfaceExtraction.impl.hpp | 2 +- Source/BoxUtils/NanCheck.hpp | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/Examples/ScalarField/ScalarFieldLevel.hpp b/Examples/ScalarField/ScalarFieldLevel.hpp index 0522c7102..668d529e3 100644 --- a/Examples/ScalarField/ScalarFieldLevel.hpp +++ b/Examples/ScalarField/ScalarFieldLevel.hpp @@ -34,30 +34,32 @@ class ScalarFieldLevel : public GRAMRLevel typedef ScalarField ScalarFieldWithPotential; //! Things to do at the end of the advance step, after RK4 calculation - virtual void specificAdvance(); + virtual void specificAdvance() override; //! Initialize data for the field and metric variables - virtual void initialData(); + virtual void initialData() override; #ifdef CH_USE_HDF5 //! routines to do before outputting plot file - virtual void prePlotLevel(); + virtual void prePlotLevel() override; #endif //! RHS routines used at each RK4 step virtual void specificEvalRHS(GRLevelData &a_soln, GRLevelData &a_rhs, - const double a_time); + const double a_time) override; //! Things to do in UpdateODE step, after soln + rhs update virtual void specificUpdateODE(GRLevelData &a_soln, - const GRLevelData &a_rhs, Real a_dt); + const GRLevelData &a_rhs, + Real a_dt) override; /// Things to do before tagging cells (i.e. filling ghosts) virtual void preTagCells() override; //! Tell Chombo how to tag cells for regridding - virtual void computeTaggingCriterion(FArrayBox &tagging_criterion, - const FArrayBox ¤t_state); + virtual void + computeTaggingCriterion(FArrayBox &tagging_criterion, + const FArrayBox ¤t_state) override; //! to do post each time step on every level virtual void specificPostTimeStep() override; diff --git a/Source/AMRInterpolator/SurfaceExtraction.impl.hpp b/Source/AMRInterpolator/SurfaceExtraction.impl.hpp index 2632c37fa..f8a27757a 100644 --- a/Source/AMRInterpolator/SurfaceExtraction.impl.hpp +++ b/Source/AMRInterpolator/SurfaceExtraction.impl.hpp @@ -420,7 +420,7 @@ void SurfaceExtraction::write_extraction( #if CH_SPACEDIM == 3 extraction_file.write_data_line(data, {u, v}); #elif CH_SPACEDIM == 2 - extraction_file.write_data_line(data, {u}); + extraction_file.write_data_line(data, u); #endif } } diff --git a/Source/BoxUtils/NanCheck.hpp b/Source/BoxUtils/NanCheck.hpp index 2d700c5d4..c54803e90 100644 --- a/Source/BoxUtils/NanCheck.hpp +++ b/Source/BoxUtils/NanCheck.hpp @@ -23,7 +23,7 @@ class NanCheck public: NanCheck(const std::string a_error_info = "NanCheck", const double a_max_abs = 1e20) - : m_dx(-1), m_center{}, m_error_info(a_error_info), m_max_abs(a_max_abs) + : NanCheck(-1, {0.}, a_error_info, a_max_abs) { } From 1c0ac9b520257dfaf1d03fedf2cc675f4e5db709 Mon Sep 17 00:00:00 2001 From: TaigoFr Date: Tue, 1 Mar 2022 22:23:36 +0000 Subject: [PATCH 42/92] Fix warning for 2D --- Source/utils/TensorAlgebra.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/utils/TensorAlgebra.hpp b/Source/utils/TensorAlgebra.hpp index 9ba63bd23..f1399ea1c 100644 --- a/Source/utils/TensorAlgebra.hpp +++ b/Source/utils/TensorAlgebra.hpp @@ -291,9 +291,9 @@ ALWAYS_INLINE Tensor<2, data_t> lower_all(const Tensor<2, data_t> &tensor_UU, constexpr int delta(int i, int j) { return (i == j); } /// Computes the levi-civita symbol (3D, NB, symbol, not the Tensor) -inline Tensor<3, double> epsilon() +inline Tensor<3, double, 3> epsilon() { - Tensor<3, double> epsilon = {0.}; + Tensor<3, double, 3> epsilon = {0.}; epsilon[0][1][2] = 1.0; epsilon[1][2][0] = 1.0; epsilon[2][0][1] = 1.0; From e56cbbcb6914541b61d651b54950e3e5cdf153ac Mon Sep 17 00:00:00 2001 From: te307 Date: Wed, 24 May 2023 14:32:34 +0100 Subject: [PATCH 43/92] remove Examples not in the main branch --- Examples/SingleBH/DiagnosticVariables.hpp | 29 --- Examples/SingleBH/GNUmakefile | 26 --- Examples/SingleBH/Main_SingleBH.cpp | 90 --------- Examples/SingleBH/SimulationParameters.hpp | 82 -------- Examples/SingleBH/SingleBHLevel.cpp | 120 ------------ Examples/SingleBH/SingleBHLevel.hpp | 49 ----- Examples/SingleBH/UserVariables.hpp | 29 --- Examples/SingleBH/params.txt | 182 ------------------ .../BlackHoles/IsotropicBoostedBH.hpp | 37 ---- .../BlackHoles/IsotropicBoostedBH.impl.hpp | 121 ------------ .../InitialConditions/BlackHoles/SingleBH.hpp | 52 ----- .../BlackHoles/SingleBH.impl.hpp | 71 ------- 12 files changed, 888 deletions(-) delete mode 100644 Examples/SingleBH/DiagnosticVariables.hpp delete mode 100644 Examples/SingleBH/GNUmakefile delete mode 100644 Examples/SingleBH/Main_SingleBH.cpp delete mode 100644 Examples/SingleBH/SimulationParameters.hpp delete mode 100644 Examples/SingleBH/SingleBHLevel.cpp delete mode 100644 Examples/SingleBH/SingleBHLevel.hpp delete mode 100644 Examples/SingleBH/UserVariables.hpp delete mode 100644 Examples/SingleBH/params.txt delete mode 100644 Source/InitialConditions/BlackHoles/IsotropicBoostedBH.hpp delete mode 100644 Source/InitialConditions/BlackHoles/IsotropicBoostedBH.impl.hpp delete mode 100644 Source/InitialConditions/BlackHoles/SingleBH.hpp delete mode 100644 Source/InitialConditions/BlackHoles/SingleBH.impl.hpp diff --git a/Examples/SingleBH/DiagnosticVariables.hpp b/Examples/SingleBH/DiagnosticVariables.hpp deleted file mode 100644 index 88ff5b7dc..000000000 --- a/Examples/SingleBH/DiagnosticVariables.hpp +++ /dev/null @@ -1,29 +0,0 @@ -/* GRChombo - * Copyright 2012 The GRChombo collaboration. - * Please refer to LICENSE in GRChombo's root directory. - */ - -#ifndef DIAGNOSTICVARIABLES_HPP -#define DIAGNOSTICVARIABLES_HPP - -// assign an enum to each variable -enum -{ - c_Ham, - - c_Mom1, - c_Mom2, - c_Mom3, - - NUM_DIAGNOSTIC_VARS -}; - -namespace DiagnosticVariables -{ -static const std::array variable_names = { - "Ham", - - "Mom1", "Mom2", "Mom3"}; -} - -#endif /* DIAGNOSTICVARIABLES_HPP */ diff --git a/Examples/SingleBH/GNUmakefile b/Examples/SingleBH/GNUmakefile deleted file mode 100644 index b7c938eac..000000000 --- a/Examples/SingleBH/GNUmakefile +++ /dev/null @@ -1,26 +0,0 @@ -# -*- Mode: Makefile -*- - -### This makefile produces an executable for each name in the `ebase' -### variable using the libraries named in the `LibNames' variable. - -# Included makefiles need an absolute path to the Chombo installation -# CHOMBO_HOME := Please set the CHOMBO_HOME locally (e.g. export CHOMBO_HOME=... in bash) - -GRCHOMBO_SOURCE = ../../Source - -ebase := Main_SingleBH - -LibNames := AMRTimeDependent AMRTools BoxTools - -src_dirs := $(GRCHOMBO_SOURCE)/utils \ - $(GRCHOMBO_SOURCE)/simd \ - $(GRCHOMBO_SOURCE)/CCZ4 \ - $(GRCHOMBO_SOURCE)/BoxUtils \ - $(GRCHOMBO_SOURCE)/GRChomboCore \ - $(GRCHOMBO_SOURCE)/AMRInterpolator \ - $(GRCHOMBO_SOURCE)/TaggingCriteria \ - $(GRCHOMBO_SOURCE)/ApparentHorizonFinder \ - $(GRCHOMBO_SOURCE)/InitialConditions/BlackHoles \ - $(GRCHOMBO_SOURCE)/BlackHoles - -include $(CHOMBO_HOME)/mk/Make.test diff --git a/Examples/SingleBH/Main_SingleBH.cpp b/Examples/SingleBH/Main_SingleBH.cpp deleted file mode 100644 index ce3c7592c..000000000 --- a/Examples/SingleBH/Main_SingleBH.cpp +++ /dev/null @@ -1,90 +0,0 @@ -/* GRChombo - * Copyright 2012 The GRChombo collaboration. - * Please refer to LICENSE in GRChombo's root directory. - */ - -#include "CH_Timer.H" -#include "parstream.H" //Gives us pout() -#include -#include - -#include "BHAMR.hpp" -#include "DefaultLevelFactory.hpp" -#include "GRParmParse.hpp" -#include "SetupFunctions.hpp" -#include "SimulationParameters.hpp" - -// Problem specific includes: -#include "SingleBHLevel.hpp" - -int runGRChombo(int argc, char *argv[]) -{ - // Load the parameter file and construct the SimulationParameter class - // To add more parameters edit the SimulationParameters file. - char *in_file = argv[1]; - GRParmParse pp(argc - 2, argv + 2, NULL, in_file); - SimulationParameters sim_params(pp); - - if (sim_params.just_check_params) - return 0; - - // The line below selects the problem that is simulated - // (To simulate a different problem, define a new child of AMRLevel - // and an associated LevelFactory) - BHAMR bh_amr; - DefaultLevelFactory single_bh_level_fact(bh_amr, sim_params); - setupAMRObject(bh_amr, single_bh_level_fact); - - // call this after amr object setup so grids known - // and need it to stay in scope throughout run - AMRInterpolator> interpolator( - bh_amr, sim_params.origin, sim_params.dx, sim_params.boundary_params, - sim_params.verbosity); - bh_amr.set_interpolator(&interpolator); - -#ifdef USE_AHFINDER - if (sim_params.AH_activate) - { - AHSurfaceGeometry sph(sim_params.bh_params.center); -#ifndef USE_ISOTROPIC_BOOSTED_BH - bh_amr.m_ah_finder.add_ah(sph, sim_params.AH_initial_guess, - sim_params.AH_params); -#else - bh_amr.m_ah_finder.add_ah(sph, sim_params.AH_initial_guess_ellipsoid, - sim_params.AH_params); -#endif - } -#endif - - using Clock = std::chrono::steady_clock; - using Minutes = std::chrono::duration>; - - std::chrono::time_point start_time = Clock::now(); - - bh_amr.run(sim_params.stop_time, sim_params.max_steps); - - auto now = Clock::now(); - auto duration = std::chrono::duration_cast(now - start_time); - pout() << "Total simulation time (mins): " << duration.count() << ".\n"; - - bh_amr.conclude(); - - CH_TIMER_REPORT(); // Report results when running with Chombo timers. - - return 0; -} - -int main(int argc, char *argv[]) -{ - mainSetup(argc, argv); - - int status = runGRChombo(argc, argv); - - if (status == 0) - pout() << "GRChombo finished." << std::endl; - else - pout() << "GRChombo failed with return code " << status << std::endl; - - mainFinalize(); - return status; -} diff --git a/Examples/SingleBH/SimulationParameters.hpp b/Examples/SingleBH/SimulationParameters.hpp deleted file mode 100644 index c6391ad6a..000000000 --- a/Examples/SingleBH/SimulationParameters.hpp +++ /dev/null @@ -1,82 +0,0 @@ -/* GRChombo - * Copyright 2012 The GRChombo collaboration. - * Please refer to LICENSE in GRChombo's root directory. - */ - -#ifndef SIMULATIONPARAMETERS_HPP -#define SIMULATIONPARAMETERS_HPP - -// General includes -#include "GRParmParse.hpp" -#include "SimulationParametersBase.hpp" - -// Problem specific includes: -#include "BoostedBH.hpp" - -// define to use IsotropicBoostedBH initial data instead of SingleBH ID -#define USE_ISOTROPIC_BOOSTED_BH - -#ifdef USE_ISOTROPIC_BOOSTED_BH -#ifdef USE_AHFINDER -#include "AHInitialGuess.hpp" -#endif -#endif - -class SimulationParameters : public SimulationParametersBase -{ - public: - SimulationParameters(GRParmParse &pp) : SimulationParametersBase(pp) - { - readParams(pp); - } - - /// Read parameters from the parameter file - void readParams(GRParmParse &pp) - { - // Initial data - pp.load("mass", bh_params.mass); - pp.load("momentum", bh_params.momentum); - pp.load("activate_extraction", activate_extraction, 0); - - // Get the centers of the BHs either explicitly or as - // an offset (not both, or they will be offset from center - // provided) - std::array offset; - pp.load("offset", offset, {0.0, 0.0, 0.0}); - FOR(idir) { bh_params.center[idir] = center[idir] + offset[idir]; } - -#ifdef USE_AHFINDER - pp.load("AH_initial_guess", AH_initial_guess, 0.5 * bh_params.mass); -#ifdef USE_ISOTROPIC_BOOSTED_BH - double r_x = AH_initial_guess, r_y = AH_initial_guess, - r_z = AH_initial_guess; - - bool ah_use_ellipsoid; - pp.load("AH_use_ellipsoid", ah_use_ellipsoid, true); - if (ah_use_ellipsoid) - { - double vel = bh_params.momentum[0]; - double contraction = sqrt(1. - vel * vel); - r_x *= contraction; - } - - AH_initial_guess_ellipsoid.set_params(r_x, r_y, r_z); -#endif -#endif - } - - // Initial data - int activate_extraction; - - // Collection of parameters necessary for initial conditions - BoostedBH::params_t bh_params; - -#ifdef USE_AHFINDER - double AH_initial_guess; -#ifdef USE_ISOTROPIC_BOOSTED_BH - AHInitialGuessEllipsoid AH_initial_guess_ellipsoid; -#endif -#endif -}; - -#endif /* SIMULATIONPARAMETERS_HPP_ */ diff --git a/Examples/SingleBH/SingleBHLevel.cpp b/Examples/SingleBH/SingleBHLevel.cpp deleted file mode 100644 index d7f512cc9..000000000 --- a/Examples/SingleBH/SingleBHLevel.cpp +++ /dev/null @@ -1,120 +0,0 @@ -/* GRChombo - * Copyright 2012 The GRChombo collaboration. - * Please refer to LICENSE in GRChombo's root directory. - */ - -#include "SingleBHLevel.hpp" -#include "BoxLoops.hpp" -#include "CCZ4.hpp" -#include "ChiExtractionTaggingCriterion.hpp" -#include "ComputePack.hpp" -#include "FourthOrderDerivatives.hpp" -#include "NanCheck.hpp" -#include "NewConstraints.hpp" -#include "PositiveChiAndAlpha.hpp" -#include "SetValue.hpp" -#include "SixthOrderDerivatives.hpp" -#include "TraceARemoval.hpp" - -// where 'USE_ISOTROPIC_BOOSTED_BH' may be defined -#include "SimulationParameters.hpp" - -#ifndef USE_ISOTROPIC_BOOSTED_BH -#include "SingleBH.hpp" -#else -#include "GammaCalculator.hpp" -#include "IsotropicBoostedBH.hpp" -#endif - -void SingleBHLevel::specificAdvance() -{ - // Enforce the trace free A_ij condition and positive chi and alpha - BoxLoops::loop(make_compute_pack(TraceARemoval(), PositiveChiAndAlpha()), - m_state_new, m_state_new, INCLUDE_GHOST_CELLS); - - // Check for nan's - if (m_p.nan_check) - BoxLoops::loop( - NanCheck(m_dx, m_p.center, "NaNCheck in specific Advance"), - m_state_new, m_state_new, EXCLUDE_GHOST_CELLS, disable_simd()); -} - -void SingleBHLevel::initialData() -{ - CH_TIME("SingleBHLevel::initialData"); - if (m_verbosity) - pout() << "SingleBHLevel::initialData " << m_level << endl; - -#ifndef USE_ISOTROPIC_BOOSTED_BH - // Set up the compute class for the SingleBH initial data - SingleBH bh(m_p.bh_params, m_dx); -#else - // Set up the compute class for the SingleBH initial data - IsotropicBoostedBH bh(m_p.bh_params, m_dx); -#endif - - // First set everything to zero (to avoid undefinded values in constraints) - // then calculate initial data - BoxLoops::loop(make_compute_pack(SetValue(0.), bh), m_state_new, - m_state_new, INCLUDE_GHOST_CELLS); - -#ifdef USE_ISOTROPIC_BOOSTED_BH - fillAllGhosts(); - BoxLoops::loop(GammaCalculator(m_dx), m_state_new, m_state_new, - EXCLUDE_GHOST_CELLS); -#endif -} - -void SingleBHLevel::prePlotLevel() -{ - fillAllGhosts(); - BoxLoops::loop(Constraints(m_dx, c_Ham, Interval(c_Mom1, c_Mom3)), - m_state_new, m_state_diagnostics, EXCLUDE_GHOST_CELLS); -} - -void SingleBHLevel::specificEvalRHS(GRLevelData &a_soln, GRLevelData &a_rhs, - const double a_time) -{ - // Enforce positive chi and alpha and trace free A - BoxLoops::loop(make_compute_pack(TraceARemoval(), PositiveChiAndAlpha()), - a_soln, a_soln, INCLUDE_GHOST_CELLS); - - // Calculate CCZ4 right hand side - if (m_p.max_spatial_derivative_order == 4) - { - BoxLoops::loop(CCZ4RHS( - m_p.ccz4_params, m_dx, m_p.sigma, m_p.formulation), - a_soln, a_rhs, EXCLUDE_GHOST_CELLS); - } - else if (m_p.max_spatial_derivative_order == 6) - { - BoxLoops::loop(CCZ4RHS( - m_p.ccz4_params, m_dx, m_p.sigma, m_p.formulation), - a_soln, a_rhs, EXCLUDE_GHOST_CELLS); - } -} - -void SingleBHLevel::specificUpdateODE(GRLevelData &a_soln, - const GRLevelData &a_rhs, Real a_dt) -{ - // Enforce the trace free A_ij condition - BoxLoops::loop(TraceARemoval(), a_soln, a_soln, INCLUDE_GHOST_CELLS); -} - -void SingleBHLevel::computeTaggingCriterion(FArrayBox &tagging_criterion, - const FArrayBox ¤t_state) -{ - BoxLoops::loop(ChiExtractionTaggingCriterion(m_dx, m_level, - m_p.extraction_params, - m_p.activate_extraction), - current_state, tagging_criterion); -} - -void SingleBHLevel::specificPostTimeStep() -{ - CH_TIME("SingleBHLevel::specificPostTimeStep"); -#ifdef USE_AHFINDER - if (m_p.AH_activate && m_level == m_p.AH_params.level_to_run) - m_bh_amr.m_ah_finder.solve(m_dt, m_time, m_restart_time); -#endif -} diff --git a/Examples/SingleBH/SingleBHLevel.hpp b/Examples/SingleBH/SingleBHLevel.hpp deleted file mode 100644 index 8efd617cf..000000000 --- a/Examples/SingleBH/SingleBHLevel.hpp +++ /dev/null @@ -1,49 +0,0 @@ -/* GRChombo - * Copyright 2012 The GRChombo collaboration. - * Please refer to LICENSE in GRChombo's root directory. - */ - -#ifndef SINGLEBHLEVEL_HPP_ -#define SINGLEBHLEVEL_HPP_ - -#include "BHAMR.hpp" -#include "DefaultLevelFactory.hpp" -#include "GRAMRLevel.hpp" - -class SingleBHLevel : public GRAMRLevel -{ - friend class DefaultLevelFactory; - // Inherit the contructors from GRAMRLevel - using GRAMRLevel::GRAMRLevel; - - BHAMR &m_bh_amr = dynamic_cast(m_gr_amr); - - /// Things to do at every full timestep - ///(might include several substeps, e.g. in RK4) - virtual void specificAdvance() override; - - /// Initial data calculation - virtual void initialData() override; - - /// Any actions that should happen just before checkpointing - virtual void prePlotLevel() override; - - /// Calculation of the right hand side for the time stepping - virtual void specificEvalRHS(GRLevelData &a_soln, GRLevelData &a_rhs, - const double a_time) override; - - /// Things to do after dt*rhs has been added to the solution - virtual void specificUpdateODE(GRLevelData &a_soln, - const GRLevelData &a_rhs, - Real a_dt) override; - - /// Identify and tag the cells that need higher resolution - virtual void - computeTaggingCriterion(FArrayBox &tagging_criterion, - const FArrayBox ¤t_state) override; - - // to do post each time step on every level - virtual void specificPostTimeStep() override; -}; - -#endif /* SINGLEBHLEVEL_HPP_ */ diff --git a/Examples/SingleBH/UserVariables.hpp b/Examples/SingleBH/UserVariables.hpp deleted file mode 100644 index b3ca4c817..000000000 --- a/Examples/SingleBH/UserVariables.hpp +++ /dev/null @@ -1,29 +0,0 @@ -/* GRChombo - * Copyright 2012 The GRChombo collaboration. - * Please refer to LICENSE in GRChombo's root directory. - */ - -#ifndef USERVARIABLES_HPP -#define USERVARIABLES_HPP - -#include "ArrayTools.hpp" -#include "CCZ4UserVariables.hpp" -#include "DiagnosticVariables.hpp" - -/// This enum gives the index of every variable stored in the grid -enum -{ - // Note that it is important that the first enum value is set to 1 more than - // the last CCZ4 var enum - NUM_VARS = NUM_CCZ4_VARS, -}; - -namespace UserVariables -{ -static const std::array variable_names = - ccz4_variable_names; -} // namespace UserVariables - -#include "UserVariables.inc.hpp" - -#endif /* USERVARIABLES_HPP */ diff --git a/Examples/SingleBH/params.txt b/Examples/SingleBH/params.txt deleted file mode 100644 index b340baa22..000000000 --- a/Examples/SingleBH/params.txt +++ /dev/null @@ -1,182 +0,0 @@ -# See the wiki page for an explanation of the params! -# https://github.com/GRChombo/GRChombo/wiki/Guide-to-parameters - -################################################# -# Filesystem parameters - -verbosity = 0 - -# location / naming of output files -# output_path = "" # Main path for all files. Must exist! -chk_prefix = SingleBH_ -plot_prefix = SingleBHPlot_ -# restart_file = SingleBH_000004.3d.hdf5 - -# HDF5files are written every dt = L/N*dt_multiplier*checkpoint_interval -checkpoint_interval = 20 -# set to 0 to turn off plot files (except at t=0 and t=stop_time) -# set to -1 to never ever print plotfiles -plot_interval = 10 -num_plot_vars = 2 -plot_vars = chi Ham - -# subpaths - specific directories for hdf5, pout, extraction data -# (these are created at runtime) -hdf5_subpath = "hdf5" -pout_subpath = "pout" -data_subpath = "data" - -# change the name of output files -# pout_prefix = "pout" -print_progress_only_to_rank_0 = 1 - -# ignore_checkpoint_name_mismatch = 0 -# write_plot_ghosts = 0 - -################################################# -# Initial Data parameters - -mass = 1. -offset = 0. 0. 0. -momentum = 0.2 0.0 0.0 # speed if using the IsotropicBoostBH ID - -################################################# -# Grid parameters - -# 'N' is the number of subdivisions in each direction of a cubic box -# 'L' is the length of the longest side of the box, dx_coarsest = L/N -# NB - If you use reflective BC and want to specify the subdivisions and side -# of the box were there are no symmetries, specify 'N_full' and 'L_full' instead -# NB - if you have a non-cubic grid, you can specify 'N1' or 'N1_full', -# 'N2' or 'N2_full' and 'N3' or 'N3_full' ( then dx_coarsest = L/N(max) ) -# NB - the N values need to be multiples of the block_factor -N_full = 128 -L_full = 64 - -# Maximum number of times you can regrid above coarsest level -max_level = 4 # There are (max_level+1) grids, so min is zero - -# Frequency of regridding at each level and thresholds on the tagging -# Need one for each level except the top one, ie max_level items -# Generally you do not need to regrid frequently on every level -# Level Regridding: 0 1 2 3 4 -regrid_interval = 1 2 4 8 16 -regrid_threshold = 0.03 - -# Max and min box sizes -max_box_size = 16 -min_box_size = 16 - -# tag_buffer_size = 3 -# grid_buffer_size = 8 -# fill_ratio = 0.7 -# num_ghosts = 3 -# center = 256.0 256.0 256.0 # defaults to center of the grid - -################################################# -# Boundary Conditions parameters - -# Periodic directions - 0 = false, 1 = true -isPeriodic = 0 0 0 -# if not periodic, then specify the boundary type -# 0 = static, 1 = sommerfeld, 2 = reflective -# (see BoundaryConditions.hpp for details) -hi_boundary = 1 1 1 -lo_boundary = 1 2 2 - -# if reflective boundaries selected, must set -# parity of all vars (in order given by UserVariables.hpp) -# 0 = even -# 1,2,3 = odd x, y, z -# 4,5,6 = odd xy, yz, xz -# 7 = odd xyz -vars_parity = 0 0 4 6 0 5 0 #chi and hij - 0 0 4 6 0 5 0 #K and Aij - 0 1 2 3 #Theta and Gamma - 0 1 2 3 1 2 3 #lapse shift and B -vars_parity_diagnostic = 0 1 2 3 #Ham and Mom - -# if sommerfeld boundaries selected, must select -# non zero asymptotic values -num_nonzero_asymptotic_vars = 5 -nonzero_asymptotic_vars = chi h11 h22 h33 lapse -nonzero_asymptotic_values = 1.0 1.0 1.0 1.0 1.0 - -# if you are using extrapolating BC: -# extrapolation_order = 1 -# num_extrapolating_vars = -1 -# extrapolating_vars = - -################################################# -# Evolution parameters - -# dt will be dx*dt_multiplier on each grid level -dt_multiplier = 0.25 -stop_time = 0.0 -# max_steps = 10 - -# Spatial derivative order (only affects CCZ4 RHS) -max_spatial_derivative_order = 4 # can be 4 or 6 - -nan_check = 1 - -# Lapse evolution -lapse_advec_coeff = 1.0 -lapse_coeff = 2.0 -lapse_power = 1.0 - -# Shift evolution -shift_advec_coeff = 0.0 # Usually no advection for beta -shift_Gamma_coeff = 0.75 -eta = 1.0 # eta of gamma driver, should be of order ~1/M_ADM of spacetime - -# CCZ4 parameters -formulation = 0 # 1 for BSSN, 0 for CCZ4 -kappa1 = 0.1 -kappa2 = 0. -kappa3 = 1. -covariantZ4 = 1 # 0: keep kappa1; 1 [default]: replace kappa1 -> kappa1/lapse - -# coefficient for KO numerical dissipation -sigma = 1.0 - -# min_chi = 1.e-4 -# min_lapse = 1.e-4 - -################################################# -# Apparent Horizon Finder parameters - -AH_activate = 1 -AH_num_ranks = 4 -AH_num_points_u = 41 -AH_num_points_v = 40 -AH_solve_interval = 1 -AH_print_interval = 1 -# AH_track_center = 1 -# AH_predict_origin = 1 -# AH_level_to_run = 0 -# AH_allow_re_attempt = 0 -# AH_stop_if_max_fails = 0 -# AH_start_time = 0. -# AH_give_up_time = -1. # -1 to never -# AH_max_fails_after_lost = 0 # -1 to never -# AH_print_geometry_data = 0 -# AH_re_solve_at_restart = 0 -# AH_verbose = 1 -# AH_expansion_radius_power = 1. - -AH_SNES_max_iterations = 500 - -# AH_initial_guess = 0.5 -# AH_use_ellipsoid = 1 # for IsotropicBoostedBH ID - -# look_for_chi_contour = 0.2 - -AH_num_extra_vars = 2 -AH_extra_vars = chi d1_chi - -# AH_coords_subpath = "data/coords" -# AH_stats_prefix = "stats_AH" -# AH_coords_prefix = "coords_AH" - -################################################# diff --git a/Source/InitialConditions/BlackHoles/IsotropicBoostedBH.hpp b/Source/InitialConditions/BlackHoles/IsotropicBoostedBH.hpp deleted file mode 100644 index 4a38ef7bd..000000000 --- a/Source/InitialConditions/BlackHoles/IsotropicBoostedBH.hpp +++ /dev/null @@ -1,37 +0,0 @@ -/* GRChombo - * Copyright 2012 The GRChombo collaboration. - * Please refer to LICENSE in GRChombo's root directory. - */ - -#ifndef ISOTROPICBOOSTEDBH_HPP_ -#define ISOTROPICBOOSTEDBH_HPP_ - -#include "Cell.hpp" - -// This is a simple class from the 2D code that was useful to test the AHFinder -// for high boosts, as it produces highly deformed (ellipsoidal) AHs -// For simplicity, it only boosts the BH in the 'x' direction of the momentum -// (so only bh_params.momentum[0] matters) -// Note: the momentum parameter of BoostedBH::params_t in this class is used as -// velocity, not momentum, and should be between ]-1,1[ -class IsotropicBoostedBH -{ - - public: - //! The constructor - IsotropicBoostedBH(BoostedBH::params_t a_bh_params, double a_dx) - : m_bh_params(a_bh_params), m_dx(a_dx) - { - } - - //! Function to compute the value of all the initial vars on the grid - template void compute(Cell current_cell) const; - - protected: - BoostedBH::params_t m_bh_params; - double m_dx; -}; - -#include "IsotropicBoostedBH.impl.hpp" - -#endif /* ISOTROPICBOOSTEDBH_HPP_ */ diff --git a/Source/InitialConditions/BlackHoles/IsotropicBoostedBH.impl.hpp b/Source/InitialConditions/BlackHoles/IsotropicBoostedBH.impl.hpp deleted file mode 100644 index 8bae6b2dc..000000000 --- a/Source/InitialConditions/BlackHoles/IsotropicBoostedBH.impl.hpp +++ /dev/null @@ -1,121 +0,0 @@ -/* GRChombo - * Copyright 2012 The GRChombo collaboration. - * Please refer to LICENSE in GRChombo's root directory. - */ - -#if !defined(ISOTROPICBOOSTEDBH_HPP_) -#error "This file should only be included through IsotropicBoostedBH.hpp" -#endif - -#ifndef ISOTROPICBOOSTEDBH_IMPL_HPP_ -#define ISOTROPICBOOSTEDBH_IMPL_HPP_ - -#if CH_SPACEDIM == 3 -#include "CCZ4Vars.hpp" -#elif CH_SPACEDIM == 2 -#include "CCZ4CartoonVars.hpp" -#endif - -#include "Coordinates.hpp" -#include "Tensor.hpp" - -// Compute the value of the initial vars on the grid -template -void IsotropicBoostedBH::compute(Cell current_cell) const -{ - Coordinates coords(current_cell, m_dx, m_bh_params.center); - - const double mass = m_bh_params.mass; - const double vel = m_bh_params.momentum[0]; - const double gamma = 1. / sqrt(1. - vel * vel); - - // coords - data_t x = coords.x * gamma; - double y = coords.y; -#if CH_SPACEDIM == 3 - double z = coords.z; -#elif CH_SPACEDIM == 2 - double z = 0; -#endif - data_t r = sqrt(x * x + y * y + z * z); - - // auxiliary vars - data_t omega = 1. - mass / (2. * r); - data_t psi = 1. + mass / (2. * r); - - data_t omega2 = omega * omega; - data_t psi4 = psi * psi * psi * psi; - data_t psi6 = psi4 * psi * psi; - data_t dx_psi = -mass * x / (2. * pow(r, 3)); - data_t dy_psi = -mass * y / (2. * pow(r, 3)); - data_t dz_psi = -mass * z / (2. * pow(r, 3)); - - data_t psi_omega = sqrt(psi6 - vel * vel * omega2); - data_t beta_x = -vel * (psi6 - omega2) / (psi_omega * psi_omega); - data_t g_xx = gamma * gamma * (psi_omega * psi_omega) / (psi * psi); - data_t g_yy = psi4; - data_t g_zz = psi4; - - // build tensors - Tensor<2, data_t, 3> KLL = {0.}; - Tensor<2, data_t, 3> gammaLL = {0.}; - Tensor<2, data_t, 3> gammaUU = {0.}; - - KLL[0][0] = -gamma * gamma * vel * dx_psi * - (2. * psi6 * (psi + 2. * omega) - 2. * vel * vel * omega2) / - (psi4 * psi * psi_omega); - KLL[1][1] = 2. * vel * psi * omega * dx_psi / psi_omega; - KLL[2][2] = KLL[1][1]; - KLL[0][1] = -gamma * vel * psi * dy_psi * (3 * omega + psi) / psi_omega; - KLL[1][0] = KLL[0][1]; - KLL[2][0] = -gamma * vel * psi * dz_psi * (3 * omega + psi) / psi_omega; - KLL[0][2] = KLL[2][0]; - KLL[2][1] = 0.; - KLL[1][2] = KLL[1][2]; - - gammaLL[0][0] = g_xx; - gammaLL[1][1] = g_yy; - gammaLL[2][2] = g_zz; - gammaUU[0][0] = 1. / g_xx; - gammaUU[1][1] = 1. / g_yy; - gammaUU[2][2] = 1. / g_zz; - - data_t chi = pow(g_xx * g_yy * g_zz, -1. / 3.); - -#if CH_SPACEDIM == 3 - CCZ4Vars::VarsWithGauge vars; -#elif CH_SPACEDIM == 2 - CCZ4CartoonVars::VarsWithGauge vars; -#endif - - // Set only the non-zero components explicitly below - VarsTools::assign(vars, 0.); - - vars.chi = chi; - vars.shift[0] = beta_x; - vars.lapse = sqrt(vars.chi); - - for (int i = 0; i < 3; i++) - { - for (int j = 0; j < 3; j++) - { - vars.K += KLL[i][j] * gammaUU[i][j]; - } - } - - FOR(i, j) { vars.h[i][j] = vars.chi * gammaLL[i][j]; } - FOR(i, j) - { - vars.A[i][j] = vars.chi * (KLL[i][j] - vars.K * gammaLL[i][j] / 3.); - } - -#if CH_SPACEDIM == 2 - vars.hww = vars.chi * gammaLL[2][2]; - vars.Aww = vars.chi * (KLL[2][2] - one_third * vars.K * gammaLL[2][2]); -#endif - - // Store the initial values of the variables - current_cell.store_vars(vars); -} - -#endif /* ISOTROPICBOOSTEDBH _IMPL_HPP_ */ diff --git a/Source/InitialConditions/BlackHoles/SingleBH.hpp b/Source/InitialConditions/BlackHoles/SingleBH.hpp deleted file mode 100644 index 2e57c3840..000000000 --- a/Source/InitialConditions/BlackHoles/SingleBH.hpp +++ /dev/null @@ -1,52 +0,0 @@ -/* GRChombo - * Copyright 2012 The GRChombo collaboration. - * Please refer to LICENSE in GRChombo's root directory. - */ - -#ifndef SINGLEBH_HPP_ -#define SINGLEBH_HPP_ - -#include "BoostedBH.hpp" -#include "Cell.hpp" -#include "Coordinates.hpp" -#include "Tensor.hpp" -#include "UserVariables.hpp" //This files needs NUM_VARS - total number of components -#include "simd.hpp" -#include - -enum Lapse -{ - ONE, - PRE_COLLAPSED, - CHI -}; - -// Just a wrapper of the BoostedBH class, to be used by itself (instead of a -// BinaryBH) -class SingleBH -{ - protected: - double m_dx; - BoostedBH bh; - int m_initial_lapse; - - public: - SingleBH(BoostedBH::params_t a_bh_params, double a_dx, - int a_initial_lapse = Lapse::PRE_COLLAPSED) - : m_dx(a_dx), bh(a_bh_params), m_initial_lapse(a_initial_lapse) - { - } - - template void compute(Cell current_cell) const; - - protected: - template - data_t compute_chi(Coordinates coords) const; - - template - Tensor<2, data_t> compute_A(data_t chi, Coordinates coords) const; -}; - -#include "SingleBH.impl.hpp" - -#endif /* SINGLEBH_HPP_ */ diff --git a/Source/InitialConditions/BlackHoles/SingleBH.impl.hpp b/Source/InitialConditions/BlackHoles/SingleBH.impl.hpp deleted file mode 100644 index 8d2991394..000000000 --- a/Source/InitialConditions/BlackHoles/SingleBH.impl.hpp +++ /dev/null @@ -1,71 +0,0 @@ -/* GRChombo - * Copyright 2012 The GRChombo collaboration. - * Please refer to LICENSE in GRChombo's root directory. - */ - -#if !defined(SINGLEBH_HPP_) -#error "This file should only be included through SingleBH.hpp" -#endif - -#ifndef SINGLEBH_IMPL_HPP_ -#define SINGLEBH_IMPL_HPP_ - -#include "BSSNVars.hpp" -#include "SingleBH.hpp" -#include "VarsTools.hpp" -#include "simd.hpp" - -template void SingleBH::compute(Cell current_cell) const -{ - BSSNVars::VarsWithGauge vars; - VarsTools::assign(vars, - 0.); // Set only the non-zero components explicitly below - Coordinates coords(current_cell, m_dx); - - vars.chi = compute_chi(coords); - - // Conformal metric is flat - FOR(i) vars.h[i][i] = 1.; - - vars.A = compute_A(vars.chi, coords); - - switch (m_initial_lapse) - { - case Lapse::ONE: - vars.lapse = 1.; - break; - case Lapse::PRE_COLLAPSED: - vars.lapse = sqrt(vars.chi); - break; - case Lapse::CHI: - vars.lapse = vars.chi; - break; - default: - MayDay::Error("SingleBH::Supplied initial lapse not supported."); - } - - current_cell.store_vars(vars); -} - -template -data_t SingleBH::compute_chi(Coordinates coords) const -{ - const data_t psi = 1. + bh.psi_minus_one(coords); - return pow(psi, -4); -} - -template -Tensor<2, data_t> SingleBH::compute_A(data_t chi, - Coordinates coords) const -{ - - Tensor<2, data_t> Aij = bh.Aij(coords); - Tensor<2, data_t> out; - - // Aij(CCZ4) = psi^(-6) * Aij(Baumgarte&Shapiro book) - FOR(i, j) out[i][j] = pow(chi, 3 / 2.) * Aij[i][j]; - - return out; -} - -#endif /* SINGLEBH_IMPL_HPP_ */ From b6272b5abcb56a81cfcfc4b502eaf91ac8e44272 Mon Sep 17 00:00:00 2001 From: te307 Date: Tue, 6 Jun 2023 12:03:11 +0100 Subject: [PATCH 44/92] attempt to resolve conflicts --- .github/dependabot.yml | 8 + .github/workflows/build-doxygen.yml | 8 +- .../workflows/run-grchombo-tests-clang.yml | 21 +- .github/workflows/run-grchombo-tests-gcc.yml | 27 ++- ...l => run-grchombo-tests-intel-classic.yml} | 32 ++- .github/workflows/test-clang-format.yml | 6 +- .gitignore | 42 ++-- .pre-commit-config.yaml | 2 +- CITATION.cff | 82 +++++++ Examples/BinaryBH/BinaryBHLevel.cpp | 5 +- Examples/BinaryBH/BinaryBHLevel.hpp | 6 +- Examples/BinaryBH/GNUmakefile | 8 + Examples/BinaryBH/Main_BinaryBH.cpp | 3 +- Examples/BinaryBH/README.md | 11 +- Examples/KerrBH/KerrBHLevel.cpp | 5 +- Examples/KerrBH/KerrBHLevel.hpp | 8 +- Examples/ScalarField/ScalarFieldLevel.cpp | 5 +- Examples/ScalarField/ScalarFieldLevel.hpp | 8 +- .../CSD3-Icelake-Intel-modules.sh | 22 ++ .../CSD3-Icelake-Intel.Make.defs.local | 35 +++ ...modules.sh => CSD3-Skylake-GCC-modules.sh} | 0 ...dules.sh => CSD3-Skylake-Intel-modules.sh} | 0 ...am.modules.sh => Cosma7-Durham-modules.sh} | 0 ...fs.local => Cosma7-Durham.Make.defs.local} | 0 .../Cosma8-Durham-modules.sh | 6 + .../Cosma8-Durham.Make.defs.local | 22 ++ .../Mac-Monterey.Make.defs.local | 32 +++ .../fawcett-KNL.Make.defs.local | 21 -- .../fawcett-Skylake.Make.defs.local | 21 -- .../MakeDefsLocalExamples/fawcett-modules.sh | 7 +- .../fawcett.Make.defs.local | 25 ++ ...l-classic-no-hdf5-minimal.Make.defs.local} | 7 +- .../ubuntu-clang.Make.defs.local | 2 +- InstallNotes/cosmos_install_notes | 8 - InstallNotes/general_install_notes | 19 -- InstallNotes/nesi_install_notes | 10 - README.md | 3 +- .../AMRInterpolator/SphericalExtraction.hpp | 6 +- .../SurfaceExtraction.impl.hpp | 6 +- Source/BoxUtils/Cell.impl.hpp | 13 +- Source/BoxUtils/FourthOrderDerivatives.hpp | 150 ++++++------ Source/BoxUtils/SixthOrderDerivatives.hpp | 155 +++++++------ Source/CCZ4/Constraints.hpp | 12 +- Source/GRChomboCore/ChomboParameters.hpp | 6 +- Source/GRChomboCore/GRAMR.cpp | 10 +- Source/GRChomboCore/GRAMRLevel.cpp | 19 +- Source/GRChomboCore/GRAMRLevel.hpp | 14 +- Source/GRChomboCore/SetupFunctions.hpp | 3 +- Source/simd/arm/arm.hpp | 28 +++ Source/simd/arm/neon.hpp | 178 ++++++++++++++ Source/simd/arm/sve.hpp | 217 ++++++++++++++++++ Source/simd/simd.hpp | 79 +++++-- Source/utils/Coordinates.hpp | 3 +- Source/utils/VarsTools.hpp | 13 +- Source/utils/WeylExtraction.hpp | 5 +- Tests/CCZ4Test/CCZ4Test.cpp | 2 +- Tests/ConstraintTest/ConstraintTest.cpp | 2 +- .../SimdFunctionsUnitTest.cpp | 2 +- .../SphericalExtractionTest.cpp | 5 +- 59 files changed, 1101 insertions(+), 354 deletions(-) create mode 100644 .github/dependabot.yml rename .github/workflows/{run-grchombo-tests-intel.yml => run-grchombo-tests-intel-classic.yml} (53%) create mode 100644 CITATION.cff create mode 100644 InstallNotes/MakeDefsLocalExamples/CSD3-Icelake-Intel-modules.sh create mode 100644 InstallNotes/MakeDefsLocalExamples/CSD3-Icelake-Intel.Make.defs.local rename InstallNotes/MakeDefsLocalExamples/{CSD3-GCC-modules.sh => CSD3-Skylake-GCC-modules.sh} (100%) rename InstallNotes/MakeDefsLocalExamples/{CSD3-Intel-modules.sh => CSD3-Skylake-Intel-modules.sh} (100%) rename InstallNotes/MakeDefsLocalExamples/{Cosma-Durham.modules.sh => Cosma7-Durham-modules.sh} (100%) rename InstallNotes/MakeDefsLocalExamples/{Cosma-Durham.Make.defs.local => Cosma7-Durham.Make.defs.local} (100%) create mode 100644 InstallNotes/MakeDefsLocalExamples/Cosma8-Durham-modules.sh create mode 100644 InstallNotes/MakeDefsLocalExamples/Cosma8-Durham.Make.defs.local create mode 100644 InstallNotes/MakeDefsLocalExamples/Mac-Monterey.Make.defs.local delete mode 100644 InstallNotes/MakeDefsLocalExamples/fawcett-KNL.Make.defs.local delete mode 100644 InstallNotes/MakeDefsLocalExamples/fawcett-Skylake.Make.defs.local create mode 100644 InstallNotes/MakeDefsLocalExamples/fawcett.Make.defs.local rename InstallNotes/MakeDefsLocalExamples/{intel-no-hdf5-minimal.Make.defs.local => intel-classic-no-hdf5-minimal.Make.defs.local} (72%) delete mode 100644 InstallNotes/cosmos_install_notes delete mode 100644 InstallNotes/general_install_notes delete mode 100644 InstallNotes/nesi_install_notes create mode 100644 Source/simd/arm/arm.hpp create mode 100644 Source/simd/arm/neon.hpp create mode 100644 Source/simd/arm/sve.hpp diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..475757376 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,8 @@ +version: 2 +updates: + # Enable version updates for GitHub actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + # Check for updates to GitHub Actions every week + interval: "weekly" \ No newline at end of file diff --git a/.github/workflows/build-doxygen.yml b/.github/workflows/build-doxygen.yml index 5c952da43..6b40057f5 100644 --- a/.github/workflows/build-doxygen.yml +++ b/.github/workflows/build-doxygen.yml @@ -7,11 +7,13 @@ on: jobs: doxygen: + name: Build Doxygen documentation + if: github.repository == 'grchombo/grchombo' runs-on: ubuntu-latest steps: - name: Checkout GRChombo - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: path: GRChombo @@ -29,7 +31,7 @@ jobs: working-directory: 'GRChombo/Doxygen/html' - name: Checkout GRChombo.github.io - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: repository: GRChombo/GRChombo.github.io path: GRChombo.github.io @@ -46,7 +48,7 @@ jobs: echo "pr_reviewer=${PR_REVIEWER}" >> $GITHUB_ENV - name: Make pull request in GRChombo.github.io - uses: peter-evans/create-pull-request@v3 + uses: peter-evans/create-pull-request@v5 with: token: ${{ secrets.REPO_GITHUB_TOKEN }} path: 'GRChombo.github.io' diff --git a/.github/workflows/run-grchombo-tests-clang.yml b/.github/workflows/run-grchombo-tests-clang.yml index a83aea63f..5b8577560 100644 --- a/.github/workflows/run-grchombo-tests-clang.yml +++ b/.github/workflows/run-grchombo-tests-clang.yml @@ -4,7 +4,7 @@ on: [push] jobs: build-and-test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 env: CHOMBO_HOME: ${{ github.workspace }}/Chombo/lib OMP_NUM_THREADS: 1 @@ -12,20 +12,31 @@ jobs: steps: - name: Checkout Chombo - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: repository: GRChombo/Chombo path: Chombo - name: Checkout GRChombo - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: path: GRChombo - - name: Install Chombo dependencies + - name: Update package manager database + id: update-database + continue-on-error: true + run: sudo apt-get update + + # This is quite slow so only do this if the previous command fails + - name: Update package repository mirrors if necessary + if: steps.update-database.outcome == 'failure' run: | + sudo gem install apt-spy2 + sudo apt-spy2 fix --commit --launchpad --country=US sudo apt-get update - sudo apt-get -y --no-install-recommends install csh libhdf5-dev libhdf5-openmpi-dev openmpi-bin libblas-dev liblapack-dev libgetopt-complete-perl + + - name: Install Chombo dependencies + run: sudo apt-get -y --no-install-recommends install csh libhdf5-dev libhdf5-openmpi-dev openmpi-bin libblas-dev liblapack-dev libgetopt-complete-perl - name: Build Chombo run: | diff --git a/.github/workflows/run-grchombo-tests-gcc.yml b/.github/workflows/run-grchombo-tests-gcc.yml index 523fc3342..5a461b8a9 100644 --- a/.github/workflows/run-grchombo-tests-gcc.yml +++ b/.github/workflows/run-grchombo-tests-gcc.yml @@ -4,33 +4,44 @@ on: [push] jobs: build-and-test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 env: CHOMBO_HOME: ${{ github.workspace }}/Chombo/lib OMP_NUM_THREADS: 1 steps: - name: Checkout Chombo - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: repository: GRChombo/Chombo path: Chombo - name: Checkout GRChombo - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: path: GRChombo - - name: Install Chombo dependencies + - name: Update package manager database + id: update-database + continue-on-error: true + run: sudo apt-get update + + # This is quite slow so only do this if the previous command fails + - name: Update package repository mirrors if necessary + if: steps.update-database.outcome == 'failure' run: | + sudo gem install apt-spy2 + sudo apt-spy2 fix --commit --launchpad --country=US sudo apt-get update - sudo apt-get -y --no-install-recommends install csh gfortran-10 g++-10 cpp-10 libhdf5-dev libhdf5-openmpi-dev openmpi-bin libblas-dev liblapack-dev libgetopt-complete-perl + + - name: Install Chombo dependencies + run: sudo apt-get -y --no-install-recommends install csh libhdf5-dev libhdf5-openmpi-dev openmpi-bin libblas-dev liblapack-dev libgetopt-complete-perl - name: Set Compilers run: | - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 100 - sudo update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-10 100 - sudo update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-10 100 + sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 120 + sudo update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-12 120 + sudo update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-12 120 - name: Build Chombo run: | diff --git a/.github/workflows/run-grchombo-tests-intel.yml b/.github/workflows/run-grchombo-tests-intel-classic.yml similarity index 53% rename from .github/workflows/run-grchombo-tests-intel.yml rename to .github/workflows/run-grchombo-tests-intel-classic.yml index 75a708801..6aaa54fae 100644 --- a/.github/workflows/run-grchombo-tests-intel.yml +++ b/.github/workflows/run-grchombo-tests-intel-classic.yml @@ -1,43 +1,55 @@ -name: Run GRChombo Tests (Intel) +name: Run GRChombo Tests (Intel Classic) on: [push] jobs: build-and-test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 env: CHOMBO_HOME: ${{ github.workspace }}/Chombo/lib OMP_NUM_THREADS: 1 steps: - name: Checkout Chombo - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: repository: GRChombo/Chombo path: Chombo - name: Checkout GRChombo - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: path: GRChombo - - name: Install Chombo dependencies + - name: Update package manager database + id: update-database + continue-on-error: true + run: sudo apt-get update + + # This is quite slow so only do this if the previous command fails + - name: Update package repository mirrors if necessary + if: steps.update-database.outcome == 'failure' run: | + sudo gem install apt-spy2 + sudo apt-spy2 fix --commit --launchpad --country=US sudo apt-get update - sudo apt-get -y --no-install-recommends install csh libgetopt-complete-perl + + - name: Install Chombo dependencies + run: sudo apt-get -y --no-install-recommends install csh libgetopt-complete-perl - name: Install Intel compilers/MPI run: | - wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB - sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB - sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main" + wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \ + | gpg --dearmor | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null + echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list + sudo apt-get update sudo apt-get -y install intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic intel-oneapi-compiler-fortran intel-oneapi-mkl intel-oneapi-mpi intel-oneapi-mpi-devel intel-oneapi-openmp working-directory: /tmp - name: Build Chombo run: | source /opt/intel/oneapi/setvars.sh - cp $GITHUB_WORKSPACE/GRChombo/InstallNotes/MakeDefsLocalExamples/intel-no-hdf5-minimal.Make.defs.local $CHOMBO_HOME/mk/Make.defs.local + cp $GITHUB_WORKSPACE/GRChombo/InstallNotes/MakeDefsLocalExamples/intel-classic-no-hdf5-minimal.Make.defs.local $CHOMBO_HOME/mk/Make.defs.local make -j 4 AMRTimeDependent AMRTools BaseTools BoxTools working-directory: ${{ env.CHOMBO_HOME }} diff --git a/.github/workflows/test-clang-format.yml b/.github/workflows/test-clang-format.yml index cfeea428f..8d9997e69 100644 --- a/.github/workflows/test-clang-format.yml +++ b/.github/workflows/test-clang-format.yml @@ -7,9 +7,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: DoozyX/clang-format-lint-action@v0.11 + - uses: actions/checkout@v3 + - uses: DoozyX/clang-format-lint-action@v0.16.2 with: source: '.' extensions: 'hpp,cpp' - clangFormatVersion: 10 + clangFormatVersion: 16 diff --git a/.gitignore b/.gitignore index 290526597..00a0ac00e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,27 +1,39 @@ -jobscript* -pout.* -.bashrc -*.pbs -*.s +# Build files +*_F.H d o p f -*_F.H -.DS_Store -log* *.ex -.*.sw[n-p] +ipo_out.optrpt + +# Scheduler files +jobscript* +*.pbs + +# Example output +Weyl_integral* +*ExtractionOut_* *.out *.dat *.hdf5 -*.dSYM LB.txt -Weyl_integral* -*ExtractionOut_* time.table.* -*.patch -*.log -ipo_out.optrpt +pout.* +hdf5 +pout +data +datasets +# Documentation Doxygen/html + +# Other +.bashrc +*.s +.DS_Store +log* +.*.sw[n-p] +*.dSYM +*.patch +*.log \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 957a57099..a62b0f7be 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ repos: - repo: https://github.com/pre-commit/mirrors-clang-format - rev: v10.0.1 + rev: v16.0.4 hooks: - id: clang-format diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 000000000..10c2eb6c8 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,82 @@ +cff-version: 1.2.0 +message: "If you use GRChombo as part of a paper, please cite our JOSS +publication" +title: "GRChombo" +authors: +- name: "The GRChombo collaboration" +url: "https://www.grchombo.org/" +preferred-citation: + type: article + authors: + - family-names: "Andrade" + given-names: "Tomas" + - family-names: "Salo" + given-names: "Llibert Areste" + - family-names: "Aurrekoetxea" + given-names: "Josu C." + - family-names: "Bamber" + given-names: "Jamie" + - family-names: "Clough" + given-names: "Katy" + - family-names: "Croft" + given-names: "Robin" + - family-names: "de Jong" + given-names: "Eloy" + - family-names: "Drew" + given-names: "Amelia" + - family-names: "Duran" + given-names: "Alejandro" + - family-names: "Ferreira" + given-names: "Pedro G." + - family-names: "Figueras" + given-names: "Pau" + - family-names: "Finkel" + given-names: "Hal" + - family-names: "França" + given-names: "Tiago" + - family-names: "Ge" + given-names: "Bo-Xuan" + - family-names: "Gu" + given-names: "Chenxia" + - family-names: "Helfer" + given-names: "Thomas" + - family-names: "Jäykkä" + given-names: "Juha" + - family-names: "Joana" + given-names: "Cristian" + - family-names: "Kunesch" + given-names: "Markus" + - family-names: "Kornet" + given-names: "Kacper" + - family-names: "Lim" + given-names: "Eugene A." + - family-names: "Muia" + given-names: "Francesco" + - family-names: "Nazari" + given-names: "Zainab" + - family-names: "Radia" + given-names: "Miren" + - family-names: "Ripley" + given-names: "Justin" + - family-names: "Shellard" + given-names: "Paul" + - family-names: "Sperhake" + given-names: "Ulrich" + - family-names: "Traykova" + given-names: "Dina" + - family-names: "Tunyasuvunakool" + given-names: "Saran" + - family-names: "Wang" + given-names: "Zipeng" + - family-names: "Widdicombe" + given-names: "James Y." + - family-names: "Wong" + given-names: "Kaze" + doi: "10.21105/joss.03703" + journal: "Journal of Open Source Software" + title: "GRChombo: An adaptable numerical relativity code for fundamental physics" + issue: 68 + volume: 6 + start: 3703 + month: 12 + year: 2021 \ No newline at end of file diff --git a/Examples/BinaryBH/BinaryBHLevel.cpp b/Examples/BinaryBH/BinaryBHLevel.cpp index ec1a5f225..d491fbf22 100644 --- a/Examples/BinaryBH/BinaryBHLevel.cpp +++ b/Examples/BinaryBH/BinaryBHLevel.cpp @@ -99,8 +99,9 @@ void BinaryBHLevel::preTagCells() } // specify the cells to tag -void BinaryBHLevel::computeTaggingCriterion(FArrayBox &tagging_criterion, - const FArrayBox ¤t_state) +void BinaryBHLevel::computeTaggingCriterion( + FArrayBox &tagging_criterion, const FArrayBox ¤t_state, + const FArrayBox ¤t_state_diagnostics) { if (m_p.track_punctures) { diff --git a/Examples/BinaryBH/BinaryBHLevel.hpp b/Examples/BinaryBH/BinaryBHLevel.hpp index b11a2b579..668be0b08 100644 --- a/Examples/BinaryBH/BinaryBHLevel.hpp +++ b/Examples/BinaryBH/BinaryBHLevel.hpp @@ -42,9 +42,9 @@ class BinaryBHLevel : public GRAMRLevel virtual void preTagCells() override; /// Identify and tag the cells that need higher resolution - virtual void - computeTaggingCriterion(FArrayBox &tagging_criterion, - const FArrayBox ¤t_state) override; + virtual void computeTaggingCriterion( + FArrayBox &tagging_criterion, const FArrayBox ¤t_state, + const FArrayBox ¤t_state_diagnostics) override; // to do post each time step on every level virtual void specificPostTimeStep() override; diff --git a/Examples/BinaryBH/GNUmakefile b/Examples/BinaryBH/GNUmakefile index 9208fa18f..354d42cec 100644 --- a/Examples/BinaryBH/GNUmakefile +++ b/Examples/BinaryBH/GNUmakefile @@ -43,3 +43,11 @@ ifeq ($(USE_TWOPUNCTURES),TRUE) endif include $(CHOMBO_HOME)/mk/Make.test + +ifdef TWOPUNCTURES_SOURCE +all-tp: all-test + $(ECHO)mv $(_app_configs) $(ebase)_TP.$(config).ex +else +all-tp: + @echo "Set TWOPUNCTURES_SOURCE environment variable to build BinaryBH example with TwoPunctures."; +endif diff --git a/Examples/BinaryBH/Main_BinaryBH.cpp b/Examples/BinaryBH/Main_BinaryBH.cpp index 898af893c..fc5a002da 100644 --- a/Examples/BinaryBH/Main_BinaryBH.cpp +++ b/Examples/BinaryBH/Main_BinaryBH.cpp @@ -96,7 +96,8 @@ int runGRChombo(int argc, char *argv[]) std::chrono::time_point start_time = Clock::now(); // Add a scheduler to call specificPostTimeStep on every AMRLevel at t=0 - auto task = [](GRAMRLevel *level) { + auto task = [](GRAMRLevel *level) + { if (level->time() == 0.) level->specificPostTimeStep(); }; diff --git a/Examples/BinaryBH/README.md b/Examples/BinaryBH/README.md index 1f45a4f9f..9ed8368f6 100644 --- a/Examples/BinaryBH/README.md +++ b/Examples/BinaryBH/README.md @@ -7,10 +7,15 @@ e.g. ```bash export TWOPUNCTURES_SOURCE=/path/to/TwoPunctures/Source ``` -Then, simply build as normal e.g. +Then, use `make` to build the `all-tp` target using a command such as ```bash -make all -j 4 +make all-tp -j 4 ``` +The created executable will have the filename +``` +Main_BinaryBH_TP..ex +``` + To stop using TwoPunctures, undefine the `TWOPUNCTURES_SOURCE` environment variable: ```bash unset TWOPUNCTURES_SOURCE @@ -18,7 +23,7 @@ unset TWOPUNCTURES_SOURCE Alternatively, you can avoid defining an environment variable by defining it in the make command: ```bash -make all -j 4 TWOPUNCTURES_SOURCE=/path/to/TwoPunctures/Source +make all-tp -j 4 TWOPUNCTURES_SOURCE=/path/to/TwoPunctures/Source ``` Note that the parameter names for TwoPunctures initial data differ to that of the vanilla example: see [params_two_punctures.txt](./params_two_punctures.txt) diff --git a/Examples/KerrBH/KerrBHLevel.cpp b/Examples/KerrBH/KerrBHLevel.cpp index 829280a2a..5488e66f6 100644 --- a/Examples/KerrBH/KerrBHLevel.cpp +++ b/Examples/KerrBH/KerrBHLevel.cpp @@ -107,8 +107,9 @@ void KerrBHLevel::preTagCells() fillAllGhosts(VariableType::evolution, Interval(c_chi, c_chi)); } -void KerrBHLevel::computeTaggingCriterion(FArrayBox &tagging_criterion, - const FArrayBox ¤t_state) +void KerrBHLevel::computeTaggingCriterion( + FArrayBox &tagging_criterion, const FArrayBox ¤t_state, + const FArrayBox ¤t_state_diagnostics) { BoxLoops::loop(ChiTaggingCriterion(m_dx), current_state, tagging_criterion); } diff --git a/Examples/KerrBH/KerrBHLevel.hpp b/Examples/KerrBH/KerrBHLevel.hpp index 9622be7ca..65fff6e42 100644 --- a/Examples/KerrBH/KerrBHLevel.hpp +++ b/Examples/KerrBH/KerrBHLevel.hpp @@ -42,10 +42,10 @@ class KerrBHLevel : public GRAMRLevel /// Things to do before tagging cells (i.e. filling ghosts) virtual void preTagCells() override; - virtual void - computeTaggingCriterion(FArrayBox &tagging_criterion, - const FArrayBox ¤t_state) override; - + virtual void computeTaggingCriterion( + FArrayBox &tagging_criterion, const FArrayBox ¤t_state, + const FArrayBox ¤t_state_diagnostics) override; + virtual void specificPostTimeStep() override; }; diff --git a/Examples/ScalarField/ScalarFieldLevel.cpp b/Examples/ScalarField/ScalarFieldLevel.cpp index d0c2bd7c8..8f5391b5a 100644 --- a/Examples/ScalarField/ScalarFieldLevel.cpp +++ b/Examples/ScalarField/ScalarFieldLevel.cpp @@ -123,8 +123,9 @@ void ScalarFieldLevel::preTagCells() // used here so don't fill any } -void ScalarFieldLevel::computeTaggingCriterion(FArrayBox &tagging_criterion, - const FArrayBox ¤t_state) +void ScalarFieldLevel::computeTaggingCriterion( + FArrayBox &tagging_criterion, const FArrayBox ¤t_state, + const FArrayBox ¤t_state_diagnostics) { BoxLoops::loop( FixedGridsTaggingCriterion(m_dx, m_level, 2.0 * m_p.L, m_p.center), diff --git a/Examples/ScalarField/ScalarFieldLevel.hpp b/Examples/ScalarField/ScalarFieldLevel.hpp index 668d529e3..2425a6933 100644 --- a/Examples/ScalarField/ScalarFieldLevel.hpp +++ b/Examples/ScalarField/ScalarFieldLevel.hpp @@ -57,10 +57,10 @@ class ScalarFieldLevel : public GRAMRLevel virtual void preTagCells() override; //! Tell Chombo how to tag cells for regridding - virtual void - computeTaggingCriterion(FArrayBox &tagging_criterion, - const FArrayBox ¤t_state) override; - + virtual void computeTaggingCriterion( + FArrayBox &tagging_criterion, const FArrayBox ¤t_state, + const FArrayBox ¤t_state_diagnostics) override; + //! to do post each time step on every level virtual void specificPostTimeStep() override; }; diff --git a/InstallNotes/MakeDefsLocalExamples/CSD3-Icelake-Intel-modules.sh b/InstallNotes/MakeDefsLocalExamples/CSD3-Icelake-Intel-modules.sh new file mode 100644 index 000000000..f30e1ada5 --- /dev/null +++ b/InstallNotes/MakeDefsLocalExamples/CSD3-Icelake-Intel-modules.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# To load the modules below call this file as an argument to `source` + +# Check that we are running on an Icelake (either login or compute) node +if [[ $(hostname) != *"-q-"* ]]; then + echo "These modules will only work on an Icelake (Rocky Linux 8) node." 1>&2 + echo "See https://docs.hpc.cam.ac.uk/hpc/user-guide/connecting.html" + echo "Not loading anything..." 1>&2 + return 1 +fi + +# This should already be loaded but load anyway in case modules have been purged +module load rhel8/default-icl # loads Intel compiler and MPI + +module load intel-oneapi-mkl/2022.1.0/intel/mngj3ad6 +module load hdf5/1.10.8/intel/intel-oneapi-mpi/h75adcal + +# Uncomment if using AHFinder +#module load petsc/3.17-icl + +# Uncomment if using TwoPunctures +#module load gsl/2.7.1/gcc/cjb5f4ui diff --git a/InstallNotes/MakeDefsLocalExamples/CSD3-Icelake-Intel.Make.defs.local b/InstallNotes/MakeDefsLocalExamples/CSD3-Icelake-Intel.Make.defs.local new file mode 100644 index 000000000..494f01a33 --- /dev/null +++ b/InstallNotes/MakeDefsLocalExamples/CSD3-Icelake-Intel.Make.defs.local @@ -0,0 +1,35 @@ +# Use this with the following modules +# intel-oneapi-compilers/2022.1.0/gcc/b6zld2mz (loaded by default with the rhel8/default-icl module +# intel-oneapi-mpi/2021.6.0/intel/guxuvcpm (loaded by default with the rhel8/default-icl module) +# intel-oneapi-mkl/2022.1.0/intel/mngj3ad6 +# hdf5/1.10.8/intel/intel-oneapi-mpi/h75adcal + +DIM ?= 3 +DEBUG = FALSE +OPT = HIGH +PRECISION = DOUBLE +PROFILE = FALSE +CXX = icpc +FC = ifort +MPI = TRUE +OPENMPCC = TRUE +MPICXX = mpiicpc +# Uncomment if using AHFinder +# XTRACPPFLAGS = -DUSE_AHFINDER +XTRACONFIG = .Icelake.Intel2021.6 +USE_64 = TRUE +USE_HDF = TRUE +HDF5_PATH = `pkg-config --variable=prefix hdf5` +HDFINCFLAGS = -I${HDF5_PATH}/include +HDFLIBFLAGS = -L${HDF5_PATH}/lib -lhdf5 -lz +HDFMPIINCFLAGS = -I${HDF5_PATH}/include +HDFMPILIBFLAGS = -L${HDF5_PATH}/lib -lhdf5 -lz +NAMESPACE = TRUE +USE_MT = FALSE +cxxdbgflags = -g +cxxoptflags = -g -ipo -qoverride-limits -no-prec-div -fp-model fast=2 -xICELAKE-SERVER -qopt-zmm-usage=high +ldoptflags = -diag-disable=11003 +fdbgflags = -g +foptflags = -g -no-prec-div -fp-model fast=2 -xICELAKE-SERVER -qopt-zmm-usage=high +# Uncomment end of next line if using AHFinder +syslibflags = -qmkl=sequential #-lpetsc diff --git a/InstallNotes/MakeDefsLocalExamples/CSD3-GCC-modules.sh b/InstallNotes/MakeDefsLocalExamples/CSD3-Skylake-GCC-modules.sh similarity index 100% rename from InstallNotes/MakeDefsLocalExamples/CSD3-GCC-modules.sh rename to InstallNotes/MakeDefsLocalExamples/CSD3-Skylake-GCC-modules.sh diff --git a/InstallNotes/MakeDefsLocalExamples/CSD3-Intel-modules.sh b/InstallNotes/MakeDefsLocalExamples/CSD3-Skylake-Intel-modules.sh similarity index 100% rename from InstallNotes/MakeDefsLocalExamples/CSD3-Intel-modules.sh rename to InstallNotes/MakeDefsLocalExamples/CSD3-Skylake-Intel-modules.sh diff --git a/InstallNotes/MakeDefsLocalExamples/Cosma-Durham.modules.sh b/InstallNotes/MakeDefsLocalExamples/Cosma7-Durham-modules.sh similarity index 100% rename from InstallNotes/MakeDefsLocalExamples/Cosma-Durham.modules.sh rename to InstallNotes/MakeDefsLocalExamples/Cosma7-Durham-modules.sh diff --git a/InstallNotes/MakeDefsLocalExamples/Cosma-Durham.Make.defs.local b/InstallNotes/MakeDefsLocalExamples/Cosma7-Durham.Make.defs.local similarity index 100% rename from InstallNotes/MakeDefsLocalExamples/Cosma-Durham.Make.defs.local rename to InstallNotes/MakeDefsLocalExamples/Cosma7-Durham.Make.defs.local diff --git a/InstallNotes/MakeDefsLocalExamples/Cosma8-Durham-modules.sh b/InstallNotes/MakeDefsLocalExamples/Cosma8-Durham-modules.sh new file mode 100644 index 000000000..8a44034fc --- /dev/null +++ b/InstallNotes/MakeDefsLocalExamples/Cosma8-Durham-modules.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +module load gnu_comp/7.3.0 +module load intel_comp/2022.1.2 +module load compiler/2022.0.2 mkl/2022.0.2 mpi/2021.5.1 +module load parallel_hdf5/1.12.0 diff --git a/InstallNotes/MakeDefsLocalExamples/Cosma8-Durham.Make.defs.local b/InstallNotes/MakeDefsLocalExamples/Cosma8-Durham.Make.defs.local new file mode 100644 index 000000000..500eff46f --- /dev/null +++ b/InstallNotes/MakeDefsLocalExamples/Cosma8-Durham.Make.defs.local @@ -0,0 +1,22 @@ +DIM = 3 +CXX = icpc +FC = ifort +OPT = HIGH +DEBUG = FALSE +MPI = TRUE +OPENMPCC = TRUE +USE_64 = TRUE +USE_HDF = TRUE +MPICXX = mpiicpc +HDFINCFLAGS = -I${HDF5_HOME}/include +HDFLIBFLAGS = -L${HDF5_HOME}/lib -lhdf5 -lz +HDFMPIINCFLAGS = -I${HDF5_HOME}/include +HDFMPILIBFLAGS = -Wl,-rpath=${HDF5_HOME}/lib -L${HDF5_HOME}/lib -lhdf5 -lz +NAMESPACE = TRUE +cxxdbgflags = -g +cxxoptflags = -g ${COSMA_8_OPT} -ipo -fp-model fast=2 -no-prec-div +fdbgflags = -g +foptflags = -g ${COSMA_8_OPT} -fp-model fast=2 -no-prec-div +ldoptflags = -diag-disable=11003 # disable warning about no IPO for Fortran files +syslibflags = -qmkl=sequential -Wl,-rpath=${GNUROOT}/lib64 +XTRACONFIG = .COSMA8.Intel2022 diff --git a/InstallNotes/MakeDefsLocalExamples/Mac-Monterey.Make.defs.local b/InstallNotes/MakeDefsLocalExamples/Mac-Monterey.Make.defs.local new file mode 100644 index 000000000..d89c5a1e4 --- /dev/null +++ b/InstallNotes/MakeDefsLocalExamples/Mac-Monterey.Make.defs.local @@ -0,0 +1,32 @@ +# This make defs file is for Mac OS Monterey +# It assumes that you have used homebrew to install +# the relevant packages as indicated for the commands below +# You should also have obtained up to date Xcode command line tools +# using 'xcode-select --install' +DIM = 3 +DEBUG = FALSE +OPT = TRUE +PRECISION = DOUBLE +USE_64 = TRUE +# brew install gcc +CXX = g++-12 +FC = gfortran-12 +# brew install libomp +OPENMPCC = TRUE +# brew install open-mpi +MPI = TRUE +MPI_CXXFLAGS = `mpicxx --showme:compile` +MPI_LINKFLAGS = `mpicxx --showme:link` +MPICXX = g++-12 ${MPI_CXXFLAGS} +USE_HDF = TRUE +# brew install hdf5 +HDF5_DIR = `brew --cellar hdf5`/`brew ls --versions hdf5 | head -n 1 | cut -d ' ' -f2` +HDFINCFLAGS = -I${HDF5_DIR}/include +HDFLIBFLAGS = -L${HDF5_DIR}/lib -lhdf5 -lz +# brew install hdf5-mpi +HDF5_MPI_DIR = `brew --cellar hdf5-mpi`/`brew ls --versions hdf5-mpi | head -n 1 | cut -d ' ' -f2` +HDFMPIINCFLAGS = -I${HDF5_MPI_DIR}/include +HDFMPILIBFLAGS = -L${HDF5_MPI_DIR}/lib -lhdf5 -lz +cxxoptflags = -march=native -O3 +foptflags = -march=native -O3 +syslibflags = ${MPI_LINKFLAGS} -lblas -llapack -framework Accelerate diff --git a/InstallNotes/MakeDefsLocalExamples/fawcett-KNL.Make.defs.local b/InstallNotes/MakeDefsLocalExamples/fawcett-KNL.Make.defs.local deleted file mode 100644 index d96c1e925..000000000 --- a/InstallNotes/MakeDefsLocalExamples/fawcett-KNL.Make.defs.local +++ /dev/null @@ -1,21 +0,0 @@ -DIM = 3 -DEBUG = TRUE -OPT = TRUE -PRECISION = DOUBLE -PROFILE = FALSE -CXX = icpc -std=c++14 -mkl=sequential -qopenmp -FC = ifort -fpe-all0 -mkl=sequential -qopenmp -MPI = TRUE -OPENMPCC = TRUE -MPICXX = mpiicpc -std=c++14 -mkl=sequential -qopenmp -XTRACONFIG = .KNL -USE_64 = TRUE -USE_HDF = TRUE -HDFINCFLAGS = -I/nfs/software/spack/linux-sles12-x86_64/gcc-7.3.0/hdf5-1.10.1-mjgmsccmhio74tqbyazd4mavff4jhu5s/include -HDFLIBFLAGS = -L/nfs/software/spack/linux-sles12-x86_64/gcc-7.3.0/hdf5-1.10.1-mjgmsccmhio74tqbyazd4mavff4jhu5s/lib64 -lhdf5 -lz -HDFMPIINCFLAGS = -I/nfs/software/spack/linux-sles12-x86_64/intel-18.0.3/hdf5-1.10.1-wiegkjgw4yui6nqqxgv7kt2pgxwrmue7/include -HDFMPILIBFLAGS = -L/nfs/software/spack/linux-sles12-x86_64/intel-18.0.3/hdf5-1.10.1-wiegkjgw4yui6nqqxgv7kt2pgxwrmue7/lib -lhdf5 -lz -cxxdbgflags = -g -cxxoptflags = -O3 -xMIC-AVX512 -fdbgflags = -g -foptflags = -O3 -xMIC-AVX512 diff --git a/InstallNotes/MakeDefsLocalExamples/fawcett-Skylake.Make.defs.local b/InstallNotes/MakeDefsLocalExamples/fawcett-Skylake.Make.defs.local deleted file mode 100644 index c3b3a9c66..000000000 --- a/InstallNotes/MakeDefsLocalExamples/fawcett-Skylake.Make.defs.local +++ /dev/null @@ -1,21 +0,0 @@ -DIM = 3 -DEBUG = TRUE -OPT = TRUE -PRECISION = DOUBLE -PROFILE = FALSE -CXX = icpc -std=c++14 -mkl=sequential -qopenmp -FC = ifort -fpe-all0 -mkl=sequential -qopenmp -MPI = TRUE -OPENMPCC = TRUE -MPICXX = mpiicpc -std=c++14 -mkl=sequential -qopenmp -XTRACONFIG = .Skylake -USE_64 = TRUE -USE_HDF = TRUE -HDFINCFLAGS = -I/nfs/software/spack/linux-sles12-x86_64/gcc-7.3.0/hdf5-1.10.1-mjgmsccmhio74tqbyazd4mavff4jhu5s/include -HDFLIBFLAGS = -L/nfs/software/spack/linux-sles12-x86_64/gcc-7.3.0/hdf5-1.10.1-mjgmsccmhio74tqbyazd4mavff4jhu5s/lib64 -lhdf5 -lz -HDFMPIINCFLAGS = -I/nfs/software/spack/linux-sles12-x86_64/intel-18.0.3/hdf5-1.10.1-wiegkjgw4yui6nqqxgv7kt2pgxwrmue7/include -HDFMPILIBFLAGS = -L/nfs/software/spack/linux-sles12-x86_64/intel-18.0.3/hdf5-1.10.1-wiegkjgw4yui6nqqxgv7kt2pgxwrmue7/lib -lhdf5 -lz -cxxdbgflags = -g -cxxoptflags = -O3 -xCORE-AVX512 -fdbgflags = -g -foptflags = -O3 -xCORE-AVX512 diff --git a/InstallNotes/MakeDefsLocalExamples/fawcett-modules.sh b/InstallNotes/MakeDefsLocalExamples/fawcett-modules.sh index f1c9c80c1..5f2d84561 100644 --- a/InstallNotes/MakeDefsLocalExamples/fawcett-modules.sh +++ b/InstallNotes/MakeDefsLocalExamples/fawcett-modules.sh @@ -1,5 +1,6 @@ #!/bin/bash # To load the modules below call this file as an argument to `source` -module load intel/compilers/2018.3 -module load intel/impi/2018.3/intel -module load hdf5/1.10.1/intel-18.0.3-wiegkjg +module load gcc/7.5.0/gcc-7.5.0-4zdnknt # in default environment but load anyway in case of purge +module load intel-oneapi-compilers/2022.0.2/gcc-7.5.0-3clffac intel-oneapi-mkl/2022.0.2/gcc-7.5.0-rnpbvnf +module load hdf5/1.12.1/intel-oneapi-mpi-2021.5.1/gcc-7.5.0-34j4qx4 +module load petsc/3.16.5/intel-oneapi-mpi-2021.5.1/gcc-7.5.0-67nf5rm # for AHFinder diff --git a/InstallNotes/MakeDefsLocalExamples/fawcett.Make.defs.local b/InstallNotes/MakeDefsLocalExamples/fawcett.Make.defs.local new file mode 100644 index 000000000..060ab9062 --- /dev/null +++ b/InstallNotes/MakeDefsLocalExamples/fawcett.Make.defs.local @@ -0,0 +1,25 @@ +DIM ?= 3 +DEBUG = FALSE +OPT = HIGH +PRECISION = DOUBLE +PROFILE = FALSE +CXX = icpx +FC = ifort +MPI = TRUE +OPENMPCC = TRUE +MPICXX = mpiicpc -cxx=icpx +XTRACONFIG = .ICPX2022.0 +XTRACPPFLAGS = -DUSE_AHFINDER +USE_64 = TRUE +USE_HDF = TRUE +HDFINCFLAGS = -I${HDF5_ROOT}/include +HDFLIBFLAGS = -L${HDF5_ROOT}/lib -lhdf5 -lz +HDFMPIINCFLAGS = -I${HDF5_ROOT}/include +HDFMPILIBFLAGS = -L${HDF5_ROOT}/lib -lhdf5 -lz +NAMESPACE = TRUE +cxxdbgflags = -g +cxxoptflags = -g -O3 -fp-model=fast -xCOMMON-AVX512 +fdbgflags = -g +foptflags = -g -O3 -fp-model=fast -xCOMMON-AVX512 +syslibflags = -qmkl=sequential -lpetsc +RUN = mpirun -bootstrap ssh -n 2 ./ diff --git a/InstallNotes/MakeDefsLocalExamples/intel-no-hdf5-minimal.Make.defs.local b/InstallNotes/MakeDefsLocalExamples/intel-classic-no-hdf5-minimal.Make.defs.local similarity index 72% rename from InstallNotes/MakeDefsLocalExamples/intel-no-hdf5-minimal.Make.defs.local rename to InstallNotes/MakeDefsLocalExamples/intel-classic-no-hdf5-minimal.Make.defs.local index e77a8f446..22e8d3c94 100644 --- a/InstallNotes/MakeDefsLocalExamples/intel-no-hdf5-minimal.Make.defs.local +++ b/InstallNotes/MakeDefsLocalExamples/intel-classic-no-hdf5-minimal.Make.defs.local @@ -1,8 +1,8 @@ #begin -- dont change this line # This is used for the github action that builds and runs the tests -# with the Intel C++ compiler and MPI implementation +# with the Intel C++ Classic compiler and MPI implementation -DIM ?= 3 +DIM = 3 DEBUG = TRUE PRECISION = DOUBLE OPT = TRUE @@ -10,6 +10,7 @@ PROFILE = FALSE CXX = icpc FC = ifort MPI = TRUE +XTRACXXFLAGS = -diag-disable=10441 # disable deprecation notice OPENMPCC = TRUE MPICXX = mpiicpc USE_64 = TRUE @@ -17,6 +18,6 @@ USE_HDF = FALSE USE_MT = FALSE cxxoptflags = -xHost -O3 foptflags = -xHost -O3 -syslibflags = -mkl=sequential +syslibflags = -qmkl=sequential #end -- dont change this line diff --git a/InstallNotes/MakeDefsLocalExamples/ubuntu-clang.Make.defs.local b/InstallNotes/MakeDefsLocalExamples/ubuntu-clang.Make.defs.local index a1b77d140..9371e1958 100644 --- a/InstallNotes/MakeDefsLocalExamples/ubuntu-clang.Make.defs.local +++ b/InstallNotes/MakeDefsLocalExamples/ubuntu-clang.Make.defs.local @@ -3,7 +3,7 @@ # To use clang++ with the ubuntu openmpi package, set the environment variable # export OMPI_CXX=clang++ -DIM = 3 +DIM ?= 3 DEBUG = TRUE PRECISION = DOUBLE OPT = TRUE diff --git a/InstallNotes/cosmos_install_notes b/InstallNotes/cosmos_install_notes deleted file mode 100644 index 0dfd309e2..000000000 --- a/InstallNotes/cosmos_install_notes +++ /dev/null @@ -1,8 +0,0 @@ -Install notes for cosmos/universe/cosmos2 etc at DAMTP, University of Cambridge (SGI Altix UV2000) - -1) Use Cosmos.Make.defs.local in this folder as Make.defs.local in CHOMBO_HOME/lib/mk/Make.defs.local -2) Compile with intel: - module load gcc/4.9 (to get the correct intel compiler compatibility) - module load icomp/16.0.3 - module load hdf5 - (Note: you may need to run "module unload" on already loaded versions) diff --git a/InstallNotes/general_install_notes b/InstallNotes/general_install_notes deleted file mode 100644 index 3f2988993..000000000 --- a/InstallNotes/general_install_notes +++ /dev/null @@ -1,19 +0,0 @@ -GRChombo has been compiled and tested with gcc-4.9 and higher and -intel icc-16.0.3 and higher. For icc note that the gcc compatibility has to be set to gcc-4.9 or higher. - -GRChombo requires c++ standard 14 or higher so the compiler should be invoked with -std=c++14 (or higher) - -This folder also contains specific install notes for a collection of machines GRChombo has been run on. - -To compile Chombo: -You should check out the adapted Chombo version from the git repository (currently GRChombo will not compile -with the standard version of Chombo). The installation parameters can be set in Make.defs.local in the folder -${CHOMBO_HOME}/mk/. This folder contains examples for Make.defs.local files for machines we have run GRChombo on. -Once the Make.defs.local file has been set just run "make all DIM=3 (or 2,4,5... if you are thus inclined) -j 32 -(compiles in parallel on 32 ranks)" in the folder ${CHOMBO_HOME}. This will also compile the tests which is a good -thing to do once but subsequently it is faster to only compile the library by invoking "make lib DIM=3 -j 32". - -To compile GRChombo: -Export CHOMBO_HOME (the path to the Chombo installation) and compile whathever example or test you want to -run by invoking make all DIM=3 (or 2,4,5... if you are thus inclined. This is the number of spatial dimensions) -j 32 (compiles in parallel on 32 ranks) -in the relevant directory. In addition, follow any other specific instructions for your machine in this folder. diff --git a/InstallNotes/nesi_install_notes b/InstallNotes/nesi_install_notes deleted file mode 100644 index 37b4e4ded..000000000 --- a/InstallNotes/nesi_install_notes +++ /dev/null @@ -1,10 +0,0 @@ -Install notes for pan/nesi at University of Auckland - -/* 1) Use Cosmos.Make.defs.local in this folder as Make.defs.local in CHOMBO_HOME/lib/mk/Make.defs.local */ -2) Compile with intel: - module load GCC/4.9.2 (to get the correct intel compiler compatibility) - module load intel - module load intel-mpi - module load HDF5 # Then set path in Make.defs.local - module load PAPI/5.4.3-intel-2017a - (Note: you may need to run "module unload" on already loaded versions) diff --git a/README.md b/README.md index 73f9a30ac..9ca34c764 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,8 @@ for our coding style and testing policy before filing a pull request. GRChombo is licensed under the BSD 3-Clause License. Please see LICENSE for details. ## Citation -Please cite our JOSS publication using the following bibtex reference: +If you use GRChombo as part of a paper, please cite our JOSS publication which +you can do using the following BibTeX entry: ``` @article{Andrade2021, diff --git a/Source/AMRInterpolator/SphericalExtraction.hpp b/Source/AMRInterpolator/SphericalExtraction.hpp index 5708cebd7..3b14778df 100644 --- a/Source/AMRInterpolator/SphericalExtraction.hpp +++ b/Source/AMRInterpolator/SphericalExtraction.hpp @@ -99,7 +99,8 @@ class ModeExtraction : public SurfaceExtraction auto integrand_re = [dirs, center, &geom = this->m_geom, es, el, em, &a_function](std::vector &a_data_here, - double r, double theta, double phi) { + double r, double theta, double phi) + { // note that spin_Y_lm requires the coordinates with the center // at the origin double x = geom.get_grid_coord(dirs[0], r, theta, phi) - center[0]; @@ -117,7 +118,8 @@ class ModeExtraction : public SurfaceExtraction auto integrand_im = [dirs, center, &geom = this->m_geom, es, el, em, &a_function](std::vector &a_data_here, - double r, double theta, double phi) { + double r, double theta, double phi) + { // note that spin_Y_lm requires the coordinates with the center // at the origin double x = geom.get_grid_coord(dirs[0], r, theta, phi) - center[0]; diff --git a/Source/AMRInterpolator/SurfaceExtraction.impl.hpp b/Source/AMRInterpolator/SurfaceExtraction.impl.hpp index f8a27757a..cf47572c5 100644 --- a/Source/AMRInterpolator/SurfaceExtraction.impl.hpp +++ b/Source/AMRInterpolator/SurfaceExtraction.impl.hpp @@ -244,9 +244,9 @@ void SurfaceExtraction::add_var_integrand( const bool a_broadcast_integral) { CH_assert(a_var >= 0 && a_var < m_vars.size()); - integrand_t var_integrand = [var = a_var](std::vector &data, double, - double, - double) { return data[var]; }; + integrand_t var_integrand = + [var = a_var](std::vector &data, double, double, double) + { return data[var]; }; add_integrand(var_integrand, out_integrals, a_method_u, a_method_v, a_broadcast_integral); } diff --git a/Source/BoxUtils/Cell.impl.hpp b/Source/BoxUtils/Cell.impl.hpp index 020bd00dc..ff9eef4f2 100644 --- a/Source/BoxUtils/Cell.impl.hpp +++ b/Source/BoxUtils/Cell.impl.hpp @@ -40,9 +40,9 @@ template template