testing

InstituteforDiseaseModeling · Sep 21, 2023 · 3cf478f · 3cf478f
1 parent 44d479b
commit 3cf478f
Show file tree

Hide file tree

Showing 3 changed files with 144 additions and 143 deletions.
diff --git a/examples/Snakefile b/examples/Snakefile
@@ -28,15 +28,16 @@ def get_command(script="run_simulations.py", python_version: str = None):
 
 
 # doing it manually because... too complicated otherwise? re
-expected_outputs = [r"monique/calibration/baseline_calibration/01_serialize_transmission_sweep/experiment_id.txt",
-                    r"monique/calibration/baseline_calibration/02_run_transmission_sweep/experiment_id.txt",
-                    r"monthly_U5_PfPR.csv",
-                    r"../data/example_files/simulation_output/calibration/baseline_calibration/LL_allxLH_each_admin_run/CC_run3.csv",
-                    r"../data/example_files/simulation_inputs/larval_habitats/monthly_habitats_1.csv",
-                    r"monique/calibration/seasonality_calibration/01_burnin_for_seasonalityCalib/experiment_id.txt",
-                    r"monique/calibration/seasonality_calibration/02_seasonality_calibration/experiment_id.txt",
-                    r"monique/run_to_present/experiment_id.txt",
-                    r"monique/run_future_scenarios/experiment_id.txt",
+expected_outputs = [
+    #                 r"monique/calibration/baseline_calibration/01_serialize_transmission_sweep/experiment_id.txt",
+#                     r"monique/calibration/baseline_calibration/02_run_transmission_sweep/experiment_id.txt",
+#                     r"monthly_U5_PfPR.csv",
+#                     r"../data/example_files/simulation_output/calibration/baseline_calibration/LL_allxLH_each_admin_run/CC_run3.csv",
+#                     r"../data/example_files/simulation_inputs/larval_habitats/monthly_habitats_1.csv",
+#                     r"monique/calibration/seasonality_calibration/01_burnin_for_seasonalityCalib/experiment_id.txt",
+#                     r"monique/calibration/seasonality_calibration/02_seasonality_calibration/experiment_id.txt",
+#                     r"monique/run_to_present/experiment_id.txt",
+#                     r"monique/run_future_scenarios/experiment_id.txt",
                     r"ben/example/run_1960-2004/experiment_id.txt",
                     r"ben/example/run_2005-2022/experiment_id.txt"]
 
@@ -62,134 +63,134 @@ rule all:
 
 # # monique/calibration/baseline_calibration/ run
 baseline_calibration = r"monique/calibration/baseline_calibration/"
-
-rule serialize_transmission_sweep:
-    input: baseline_calibration + r"01_serialize_transmission_sweep/params.py",
-           baseline_calibration + r"01_serialize_transmission_sweep/manifest.py"
-    output: baseline_calibration + r"01_serialize_transmission_sweep/experiment_id.txt"
-    run:
-        update_parameters_in_file(input[0],{"years = 30": f"years = 10\n",
-                                            "num_seeds = ": "num_seeds = 1\n"})
-        update_parameters_in_file(input[1],{"USER_PATH = ": f"USER_PATH = r'../data'\n",
-                                            "sif_id = None": "sif_id = 'dtk_sif.id'\n"})
-        shell(get_command(script=baseline_calibration + r"01_serialize_transmission_sweep/run_simulations.py",
-            python_version=config['python_version']))
-
-rule run_transmission_sweep:
-    input: baseline_calibration + r"01_serialize_transmission_sweep/experiment_id.txt",
-           baseline_calibration + r"02_run_transmission_sweep/params.py",
-           baseline_calibration + r"02_run_transmission_sweep/manifest.py",
-    output: baseline_calibration + r"02_run_transmission_sweep/experiment_id.txt"
-    run:
-        with open(input[0]) as exp_id_file:
-            burnin_id = exp_id_file.read()
-        update_parameters_in_file(input[1],{"burnin_id = ": f"burnin_id = '{burnin_id}'\n",
-                                            "num_seeds = ": "num_seeds = 1\n",
-                                            "num_burnin_seeds = ": "num_burnin_seeds = 1\n"})
-        update_parameters_in_file(input[2],{"USER_PATH = ": f"USER_PATH = r'../data'\n",
-                                            "sif_id = None": "sif_id = 'dtk_sif.id'\n"})
-        shell(get_command(script=baseline_calibration + r"02_run_transmission_sweep/run_simulations.py",
-            python_version=config['python_version']))
-
-rule analyze_ssmt_monthy_U5_PfPR:
-    input: baseline_calibration + r"02_run_transmission_sweep/experiment_id.txt",
-           baseline_calibration + r"03_analyze_ssmt_monthly_U5_PfPR.py"
-    output: r"monthly_U5_PfPR.csv"
-    run:
-        with open(input[0]) as exp_id_file:
-            burnin_id = exp_id_file.read()
-            update_parameters_in_file(input[1],
-                {"experiments = ": f"experiments = {{'PfPR_sweep_main_example': '{burnin_id}'}}\n"})
-        shell(get_command(script=baseline_calibration + r"03_analyze_ssmt_monthly_U5_PfPR.py",
-            python_version=config['python_version']))
-
-rule find_best_xLH_fits:
-    output:
-        r"../data/example_files/simulation_output/calibration/baseline_calibration/LL_allxLH_each_admin_run/CC_run3.csv"
-    run:
-        shell(get_command(script=r"monique/calibration/baseline_calibration/04_find_best_xLH_fits.py",
-            python_version=config['python_version']))
-
-
-# monique/calibration/baseline_calibration/ run
-seasonality_cal = r"monique/calibration/seasonality_calibration/"
-
-rule burnin_for_seasonalityCalib:
-    input: seasonality_cal + r"01_burnin_for_seasonalityCalib/params.py",
-           seasonality_cal + r"01_burnin_for_seasonalityCalib/manifest.py"
-    output:
-        seasonality_cal + r"01_burnin_for_seasonalityCalib/experiment_id.txt"
-    run:
-        update_parameters_in_file(input[0],{"years = 20": f"years = 5\n"})
-        update_parameters_in_file(input[1],{"USER_PATH = ": f"USER_PATH = r'../data'\n",
-                                            "sif_id = None": "sif_id = 'dtk_sif.id'\n"})
-        shell(get_command(script=seasonality_cal + r"01_burnin_for_seasonalityCalib/run_simulations.py",
-            python_version=config['python_version']))
-
-
-rule seasonality_calibration:
-    input: seasonality_cal + r"01_burnin_for_seasonalityCalib/experiment_id.txt",
-           seasonality_cal + r"02_seasonality_calibration/params.py",
-           seasonality_cal + r"02_seasonality_calibration/manifest.py"
-    output:
-        seasonality_cal + r"02_seasonality_calibration/experiment_id.txt"
-    run:
-        with open(input[0]) as exp_id_file:
-            burnin_id = exp_id_file.read()
-        update_parameters_in_file(input[1],{"burnin_ids =": f"burnin_ids = {{'AA': '{burnin_id}'}}\n",
-                                            "burnin_years = ": "    burnin_years = 5\n"})
-        update_parameters_in_file(input[2],{"USER_PATH = ": f"USER_PATH = r'../data'\n",
-                                            "sif_id = None": "sif_id = 'dtk_sif.id'\n"})
-        shell(get_command(script=r"monique/calibration/seasonality_calibration/02_seasonality_calibration/run_calibration.py",
-            python_version=config['python_version']))
-
-rule save_best_seasonality_fit:
-    input: seasonality_cal + r"/03_save_best_seasonality_fit.py"
-    output:
-        r"../data/example_files/simulation_inputs/larval_habitats/monthly_habitats_1.csv"
-    run:
-        update_parameters_in_file(input[0],{"USER_PATH = ": f"USER_PATH = r'../data'\n"})
-        shell(get_command(script=seasonality_cal + r"03_save_best_seasonality_fit.py",
-            python_version=config['python_version']))
-
-# run to present
-run_to_pre = r"monique/run_to_present/"
-
-rule run_to_present:
-    input: baseline_calibration + r"01_serialize_transmission_sweep/experiment_id.txt",
-           run_to_pre + r"params.py",run_to_pre + r"manifest.py"
-    output: run_to_pre + r"experiment_id.txt"
-    run:
-        with open(input[0]) as exp_id_file:
-            burnin_id = exp_id_file.read()
-        update_parameters_in_file(input[1],{"burnin_id = ": f"burnin_id = '{burnin_id}'\n",
-                                            "num_seeds = ": "num_seeds = 1\n",
-                                            "num_burnin_seeds = ": "num_burnin_seeds = 1\n",
-                                            "ser_date = 30 * 365": "ser_date = 10 * 365\n"})
-        update_parameters_in_file(input[2],{"USER_PATH = ": f"USER_PATH = r'../data'\n",
-                                            "sif_id = None": "sif_id = 'dtk_sif.id'\n"})
-        shell(get_command(script=run_to_pre + r"run_simulations.py",
-            python_version=config['python_version']))
-
 #
-# run to future
-run_to_futu = r"monique/run_future_scenarios/"
-
-rule run_to_future:
-    input: run_to_pre + r"experiment_id.txt",
-           run_to_futu + r"params.py",run_to_futu + r"manifest.py"
-    output: run_to_futu + r"experiment_id.txt"
-    run:
-        with open(input[0]) as exp_id_file:
-            burnin_id = exp_id_file.read()
-        update_parameters_in_file(input[1],{"burnin_id = ": f"burnin_id = '{burnin_id}'\n",
-                                            "num_seeds = ": "num_seeds = 1\n",
-                                            "num_burnin_seeds = ": "num_burnin_seeds = 1\n",
-                                            "num_burnin_seeds_calib = 5": "num_burnin_seeds_calib = 1\n"})
-        update_parameters_in_file(input[2],{"USER_PATH = ": f"USER_PATH = r'../data'\n",
-                                            "sif_id = None": "sif_id = 'dtk_sif.id'\n"})
-        shell(get_command(script=run_to_futu + r"run_simulations.py",
-            python_version=config['python_version']))
+# rule serialize_transmission_sweep:
+#     input: baseline_calibration + r"01_serialize_transmission_sweep/params.py",
+#            baseline_calibration + r"01_serialize_transmission_sweep/manifest.py"
+#     output: baseline_calibration + r"01_serialize_transmission_sweep/experiment_id.txt"
+#     run:
+#         update_parameters_in_file(input[0],{"years = 30": f"years = 10\n",
+#                                             "num_seeds = ": "num_seeds = 1\n"})
+#         update_parameters_in_file(input[1],{"USER_PATH = ": f"USER_PATH = r'../data'\n",
+#                                             "sif_id = None": "sif_id = 'dtk_sif.id'\n"})
+#         shell(get_command(script=baseline_calibration + r"01_serialize_transmission_sweep/run_simulations.py",
+#             python_version=config['python_version']))
+#
+# rule run_transmission_sweep:
+#     input: baseline_calibration + r"01_serialize_transmission_sweep/experiment_id.txt",
+#            baseline_calibration + r"02_run_transmission_sweep/params.py",
+#            baseline_calibration + r"02_run_transmission_sweep/manifest.py",
+#     output: baseline_calibration + r"02_run_transmission_sweep/experiment_id.txt"
+#     run:
+#         with open(input[0]) as exp_id_file:
+#             burnin_id = exp_id_file.read()
+#         update_parameters_in_file(input[1],{"burnin_id = ": f"burnin_id = '{burnin_id}'\n",
+#                                             "num_seeds = ": "num_seeds = 1\n",
+#                                             "num_burnin_seeds = ": "num_burnin_seeds = 1\n"})
+#         update_parameters_in_file(input[2],{"USER_PATH = ": f"USER_PATH = r'../data'\n",
+#                                             "sif_id = None": "sif_id = 'dtk_sif.id'\n"})
+#         shell(get_command(script=baseline_calibration + r"02_run_transmission_sweep/run_simulations.py",
+#             python_version=config['python_version']))
+#
+# rule analyze_ssmt_monthy_U5_PfPR:
+#     input: baseline_calibration + r"02_run_transmission_sweep/experiment_id.txt",
+#            baseline_calibration + r"03_analyze_ssmt_monthly_U5_PfPR.py"
+#     output: r"monthly_U5_PfPR.csv"
+#     run:
+#         with open(input[0]) as exp_id_file:
+#             burnin_id = exp_id_file.read()
+#             update_parameters_in_file(input[1],
+#                 {"experiments = ": f"experiments = {{'PfPR_sweep_main_example': '{burnin_id}'}}\n"})
+#         shell(get_command(script=baseline_calibration + r"03_analyze_ssmt_monthly_U5_PfPR.py",
+#             python_version=config['python_version']))
+#
+# rule find_best_xLH_fits:
+#     output:
+#         r"../data/example_files/simulation_output/calibration/baseline_calibration/LL_allxLH_each_admin_run/CC_run3.csv"
+#     run:
+#         shell(get_command(script=r"monique/calibration/baseline_calibration/04_find_best_xLH_fits.py",
+#             python_version=config['python_version']))
+#
+#
+# # monique/calibration/baseline_calibration/ run
+# seasonality_cal = r"monique/calibration/seasonality_calibration/"
+#
+# rule burnin_for_seasonalityCalib:
+#     input: seasonality_cal + r"01_burnin_for_seasonalityCalib/params.py",
+#            seasonality_cal + r"01_burnin_for_seasonalityCalib/manifest.py"
+#     output:
+#         seasonality_cal + r"01_burnin_for_seasonalityCalib/experiment_id.txt"
+#     run:
+#         update_parameters_in_file(input[0],{"years = 20": f"years = 5\n"})
+#         update_parameters_in_file(input[1],{"USER_PATH = ": f"USER_PATH = r'../data'\n",
+#                                             "sif_id = None": "sif_id = 'dtk_sif.id'\n"})
+#         shell(get_command(script=seasonality_cal + r"01_burnin_for_seasonalityCalib/run_simulations.py",
+#             python_version=config['python_version']))
+#
+#
+# rule seasonality_calibration:
+#     input: seasonality_cal + r"01_burnin_for_seasonalityCalib/experiment_id.txt",
+#            seasonality_cal + r"02_seasonality_calibration/params.py",
+#            seasonality_cal + r"02_seasonality_calibration/manifest.py"
+#     output:
+#         seasonality_cal + r"02_seasonality_calibration/experiment_id.txt"
+#     run:
+#         with open(input[0]) as exp_id_file:
+#             burnin_id = exp_id_file.read()
+#         update_parameters_in_file(input[1],{"burnin_ids =": f"burnin_ids = {{'AA': '{burnin_id}'}}\n",
+#                                             "burnin_years = ": "    burnin_years = 5\n"})
+#         update_parameters_in_file(input[2],{"USER_PATH = ": f"USER_PATH = r'../data'\n",
+#                                             "sif_id = None": "sif_id = 'dtk_sif.id'\n"})
+#         shell(get_command(script=r"monique/calibration/seasonality_calibration/02_seasonality_calibration/run_calibration.py",
+#             python_version=config['python_version']))
+#
+# rule save_best_seasonality_fit:
+#     input: seasonality_cal + r"/03_save_best_seasonality_fit.py"
+#     output:
+#         r"../data/example_files/simulation_inputs/larval_habitats/monthly_habitats_1.csv"
+#     run:
+#         update_parameters_in_file(input[0],{"USER_PATH = ": f"USER_PATH = r'../data'\n"})
+#         shell(get_command(script=seasonality_cal + r"03_save_best_seasonality_fit.py",
+#             python_version=config['python_version']))
+#
+# # run to present
+# run_to_pre = r"monique/run_to_present/"
+#
+# rule run_to_present:
+#     input: baseline_calibration + r"01_serialize_transmission_sweep/experiment_id.txt",
+#            run_to_pre + r"params.py",run_to_pre + r"manifest.py"
+#     output: run_to_pre + r"experiment_id.txt"
+#     run:
+#         with open(input[0]) as exp_id_file:
+#             burnin_id = exp_id_file.read()
+#         update_parameters_in_file(input[1],{"burnin_id = ": f"burnin_id = '{burnin_id}'\n",
+#                                             "num_seeds = ": "num_seeds = 1\n",
+#                                             "num_burnin_seeds = ": "num_burnin_seeds = 1\n",
+#                                             "ser_date = 30 * 365": "ser_date = 10 * 365\n"})
+#         update_parameters_in_file(input[2],{"USER_PATH = ": f"USER_PATH = r'../data'\n",
+#                                             "sif_id = None": "sif_id = 'dtk_sif.id'\n"})
+#         shell(get_command(script=run_to_pre + r"run_simulations.py",
+#             python_version=config['python_version']))
+#
+# #
+# # run to future
+# run_to_futu = r"monique/run_future_scenarios/"
+#
+# rule run_to_future:
+#     input: run_to_pre + r"experiment_id.txt",
+#            run_to_futu + r"params.py",run_to_futu + r"manifest.py"
+#     output: run_to_futu + r"experiment_id.txt"
+#     run:
+#         with open(input[0]) as exp_id_file:
+#             burnin_id = exp_id_file.read()
+#         update_parameters_in_file(input[1],{"burnin_id = ": f"burnin_id = '{burnin_id}'\n",
+#                                             "num_seeds = ": "num_seeds = 1\n",
+#                                             "num_burnin_seeds = ": "num_burnin_seeds = 1\n",
+#                                             "num_burnin_seeds_calib = 5": "num_burnin_seeds_calib = 1\n"})
+#         update_parameters_in_file(input[2],{"USER_PATH = ": f"USER_PATH = r'../data'\n",
+#                                             "sif_id = None": "sif_id = 'dtk_sif.id'\n"})
+#         shell(get_command(script=run_to_futu + r"run_simulations.py",
+#             python_version=config['python_version']))
 
 # run 1960 to 2004
 run_1960_to_2004 = r"ben/example/run_1960-2004/"

diff --git a/examples/ben/example/run_1960-2004/run_simulations.py b/examples/ben/example/run_1960-2004/run_simulations.py
@@ -42,8 +42,8 @@ def _post_run(experiment: Experiment, **kwargs):
     Return:
         None
     """
-    wait_until_done = kwargs.get('wait_until_done', None)
-    if wait_until_done and experiment.succeeded:
+    # wait_until_done = kwargs.get('wait_until_done', None)
+    if experiment.succeeded:
         with open("ben/example/run_1960-2004/experiment_id.txt", "w") as fd:
             fd.write(experiment.uid.hex)
 
@@ -88,7 +88,7 @@ def run_experiment(**kwargs):
 
     experiment = _config_experiment(**kwargs)
     _pre_run(experiment, **kwargs)
-    experiment.run(**kwargs)
+    experiment.run(wait_until_done=True, **kwargs)
     _post_run(experiment, **kwargs)
 
 
@@ -111,4 +111,4 @@ def run_experiment(**kwargs):
     # dtk.setup(pathlib.Path(manifest.eradication_path).parent)
     # os.chdir(os.path.dirname(__file__))
     # print("...done.")
-    run_experiment(show_warnings_once=True, wait_until_done=True)
+    run_experiment(show_warnings_once=True)
diff --git a/examples/ben/example/run_2005-2022/run_simulations.py b/examples/ben/example/run_2005-2022/run_simulations.py
@@ -43,7 +43,7 @@ def _post_run(experiment: Experiment, **kwargs):
         None
     """
     wait_until_done = kwargs.get('wait_until_done', None)
-    if wait_until_done and experiment.succeeded:
+    if experiment.succeeded:
         with open("ben/example/run_2005-2022/experiment_id.txt", "w") as fd:
             fd.write(experiment.uid.hex)
 
@@ -88,7 +88,7 @@ def run_experiment(**kwargs):
 
     experiment = _config_experiment(**kwargs)
     _pre_run(experiment, **kwargs)
-    experiment.run(**kwargs)
+    experiment.run(wait_until_done=True, **kwargs)
     _post_run(experiment, **kwargs)
 
 
@@ -111,4 +111,4 @@ def run_experiment(**kwargs):
     # dtk.setup(pathlib.Path(manifest.eradication_path).parent)
     # os.chdir(os.path.dirname(__file__))
     # print("...done.")
-    run_experiment(show_warnings_once=True, wait_until_done=True)
+    run_experiment(show_warnings_once=True )