ILAMB task revision

setup.py

@@ -18,7 +18,7 @@ def package_files(directory, prefixes, extensions):
 
 
 data_files = package_files(
-    "zppy", prefixes=[], extensions=["bash", "csh", "cfg", "ini", "sh"]
+    "zppy", prefixes=[], extensions=["bash", "csh", "cfg", "ini", "sh", "json"]
 )
 
 setup(

tests/integration/test_complete_run.cfg

@@ -32,6 +32,7 @@ years = "1850:1854:2",
   frequency = "monthly"
   input_files = "eam.h0"
   input_subdir = "archive/atm/hist"
+  ts_fmt = "cmip"
 
   [[ atm_daily_180x360_aave ]]
   frequency = "daily"
@@ -50,7 +51,8 @@ years = "1850:1854:2",
   frequency = "monthly"
   input_files = "elm.h0"
   input_subdir = "archive/lnd/hist"
-  vars = "FSH,RH2M"
+  vars = "FSH,LAISHA,LAISUN,RH2M"
+  ts_fmt = "cmip"
 
   [[ rof_monthly ]]
   extra_vars = 'areatotal2'
@@ -140,3 +142,10 @@ ts_num_years = 5
 ts_years = "1850-1854", "1850-1860",
 walltime = "00:30:00"
 years = "1850-1860",
+
+[ilamb_run]
+active = True
+grid = '180x360_aave'
+short_name = 'v2.LR.historical_0201'
+ts_num_years = 2
+years = "1850:1854:2",

tests/test_sections.py

@@ -93,10 +93,12 @@ def test_sections(self):
         expected_section = {
             "active": "True",
             "area_nm": "area",
+            "cmip_metadata": "e3sm_to_cmip/default_metadata.json",
             "dpf": 30,
             "extra_vars": "",
             "mapping_file": "MAPPING_FILE_TS",
             "tpd": 1,
+            "ts_fmt": "ts_only",
             "vars": "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT",
             "years": ["0001:0020:5"],
         }
@@ -109,6 +111,7 @@ def test_sections(self):
             "area_nm": "area",
             "campaign": "none",
             "case": "CASE",
+            "cmip_metadata": "e3sm_to_cmip/default_metadata.json",
             "debug": False,
             "dpf": 30,
             "dry_run": False,
@@ -127,6 +130,7 @@ def test_sections(self):
             "subsection": None,
             "templateDir": "zppy/templates",
             "tpd": 1,
+            "ts_fmt": "ts_only",
             "vars": "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT",
             "walltime": "02:00:00",
             "www": "WWWW",
@@ -226,23 +230,29 @@ def test_subsections(self):
         expected_section = {
             "active": "True",
             "area_nm": "area",
+            "cmip_metadata": "e3sm_to_cmip/default_metadata.json",
             "dpf": 30,
             "extra_vars": "",
             "tpd": 1,
+            "ts_fmt": "ts_only",
             "ts_grid1": {
                 "area_nm": None,
+                "cmip_metadata": None,
                 "dpf": None,
                 "extra_vars": None,
                 "mapping_file": "MAPPING_FILE_TS_GRID1",
                 "tpd": None,
+                "ts_fmt": None,
                 "years": ["0001:0020:5"],
             },
             "ts_grid2": {
                 "area_nm": None,
+                "cmip_metadata": None,
                 "dpf": None,
                 "extra_vars": None,
                 "mapping_file": "MAPPING_FILE_TS_GRID2",
                 "tpd": None,
+                "ts_fmt": None,
                 "years": ["0001:0020:10"],
             },
             "vars": "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT",
@@ -256,6 +266,7 @@ def test_subsections(self):
             "area_nm": "area",
             "campaign": "none",
             "case": "CASE",
+            "cmip_metadata": "e3sm_to_cmip/default_metadata.json",
             "debug": False,
             "dpf": 30,
             "dry_run": False,
@@ -274,6 +285,7 @@ def test_subsections(self):
             "subsection": "ts_grid1",
             "templateDir": "zppy/templates",
             "tpd": 1,
+            "ts_fmt": "ts_only",
             "vars": "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT",
             "walltime": "02:00:00",
             "www": "WWWW",
@@ -286,6 +298,7 @@ def test_subsections(self):
             "area_nm": "area",
             "campaign": "none",
             "case": "CASE",
+            "cmip_metadata": "e3sm_to_cmip/default_metadata.json",
             "debug": False,
             "dpf": 30,
             "dry_run": False,
@@ -304,6 +317,7 @@ def test_subsections(self):
             "subsection": "ts_grid2",
             "templateDir": "zppy/templates",
             "tpd": 1,
+            "ts_fmt": "ts_only",
             "vars": "FSNTOA,FLUT,FSNT,FLNT,FSNS,FLNS,SHFLX,QFLX,PRECC,PRECL,PRECSC,PRECSL,TS,TREFHT",
             "walltime": "02:00:00",
             "www": "WWWW",

zppy/__main__.py

@@ -9,6 +9,7 @@
 from zppy.climo import climo
 from zppy.e3sm_diags import e3sm_diags
 from zppy.global_time_series import global_time_series
+from zppy.ilamb_run import ilamb_run
 from zppy.mpas_analysis import mpas_analysis
 from zppy.tc_analysis import tc_analysis
 from zppy.ts import ts
@@ -117,6 +118,9 @@ def main():
     # global time series tasks
     global_time_series(config, scriptDir)
 
+    # ilamb_run tasks
+    ilamb_run(config, scriptDir)
+
 
 def _validate_config(config):
     validator = Validator()

zppy/ilamb_run.py

@@ -0,0 +1,100 @@
+import os
+import pprint
+
+import jinja2
+
+from zppy.utils import checkStatus, getTasks, getYears, submitScript
+
+
+# -----------------------------------------------------------------------------
+def ilamb_run(config, scriptDir):
+
+    # Initialize jinja2 template engine
+    templateLoader = jinja2.FileSystemLoader(
+        searchpath=config["default"]["templateDir"]
+    )
+    templateEnv = jinja2.Environment(loader=templateLoader)
+    template = templateEnv.get_template("ilamb_run.bash")
+
+    # --- List of ilamb_run tasks ---
+    tasks = getTasks(config, "ilamb_run")
+    if len(tasks) == 0:
+        return
+
+    # --- Generate and submit ilamb_run scripts ---
+    dependencies = []
+
+    for c in tasks:
+
+        if "ts_num_years" in c.keys():
+            c["ts_num_years"] = int(c["ts_num_years"])
+
+        # Loop over year sets
+        year_sets = getYears(c["years"])
+        for s in year_sets:
+            c["year1"] = s[0]
+            c["year2"] = s[1]
+            c["scriptDir"] = scriptDir
+            if c["subsection"]:
+                c["sub"] = c["subsection"]
+            else:
+                c["sub"] = c["grid"]
+
+            # List of dependencies
+            dependencies.append(
+                os.path.join(
+                    scriptDir,
+                    "ts_%s_%04d-%04d-%04d.status"
+                    % (
+                        c["ts_land_subsection"],
+                        c["year1"],
+                        c["year2"],
+                        c["ts_num_years"],
+                    ),
+                ),
+            )
+            if not c["land_only"]:
+                dependencies.append(
+                    os.path.join(
+                        scriptDir,
+                        "ts_%s_%04d-%04d-%04d.status"
+                        % (
+                            c["ts_atm_subsection"],
+                            c["year1"],
+                            c["year2"],
+                            c["ts_num_years"],
+                        ),
+                    ),
+                )
+
+            prefix = "ilamb_run_%04d-%04d" % (
+                c["year1"],
+                c["year2"],
+            )
+            c["prefix"] = prefix
+            print(prefix)
+            scriptFile = os.path.join(scriptDir, "%s.bash" % (prefix))
+            statusFile = os.path.join(scriptDir, "%s.status" % (prefix))
+            settingsFile = os.path.join(scriptDir, "%s.settings" % (prefix))
+            skip = checkStatus(statusFile)
+            if skip:
+                continue
+
+            # Create script
+            with open(scriptFile, "w") as f:
+                f.write(template.render(**c))
+
+            with open(settingsFile, "w") as sf:
+                p = pprint.PrettyPrinter(indent=2, stream=sf)
+                p.pprint(c)
+                p.pprint(s)
+
+            if not c["dry_run"]:
+                # Submit job
+                # Note --export=All is needed to make sure the executable is copied and executed on the nodes.
+                jobid = submitScript(scriptFile, dependFiles=dependencies, export="ALL")
+
+                if jobid != -1:
+                    # Update status file
+                    with open(statusFile, "w") as f:
+                        f.write("WAITING %d\n" % (jobid))

zppy/templates/default.ini

@@ -59,17 +59,21 @@ vars = string(default="")
 
 [ts]
 area_nm = string(default="area")
+cmip_metadata = string(default="e3sm_to_cmip/default_metadata.json")
 # Days per file
 dpf = integer(default=30)
 extra_vars = string(default="")
 # Time-steps per day
 tpd = integer(default=1)
+ts_fmt = string(default="ts_only")
 
   [[__many__]]
   area_nm = string(default=None)
+  cmip_metadata = string(default=None)
   dpf = integer(default=None)
   extra_vars = string(default=None)
   tpd = integer(default=None)
+  ts_fmt = string(default=None)
 
 [tc_analysis]
 # NOTE: always overrides value in [default]
@@ -253,3 +257,9 @@ moc_file = string(default="")
 ts_num_years = integer(default=10)
 ts_years = string_list(default=list(""))
 # `years = "1-100",` would plot years 1 to 100 on the graphs.
+
+[ilamb_run]
+cfg = string(default="ilamb_run/cmip.cfg")
+land_only = boolean(default=False)
+ts_atm_subsection = string(default="atm_monthly_180x360_aave")
+ts_land_subsection = string(default="land_monthly")

zppy/templates/e3sm_to_cmip/default_metadata.json

@@ -0,0 +1,107 @@
+{
+  "#note_source_type": "explanation of what source_type is goes here",
+
+  "source_type": "AOGCM AER",
+
+  "#note_experiment_id": "CMIP6 valid experiment_ids are found in CMIP6_CV.json",
+
+  "experiment_id": "piControl",
+
+  "activity_id": "CMIP",
+
+  "sub_experiment_id": "none",
+
+  "realization_index": "1",
+
+  "initialization_index": "1",
+
+  "physics_index": "1",
+
+  "forcing_index": "1",
+
+  "#note_run_variant": "Text stored in attribute variant_info (recommended, not required description of run variant)",
+
+  "run_variant": "",
+
+  "parent_experiment_id": "piControl-spinup",
+
+  "parent_activity_id": "CMIP",
+
+  "parent_source_id": "E3SM-1-0",
+
+  "parent_variant_label": "r1i1p1f1",
+
+  "parent_time_units": "days since 0001-01-01",
+
+  "branch_method": "standard",
+
+  "branch_time_in_child": 0.0,
+
+  "branch_time_in_parent": 0.0,
+
+  "#note_institution_id": "institution_id must be registered at https://github.com/WCRP-CMIP/CMIP6_CVs/issues/new ",
+
+  "institution_id": "E3SM-Project",
+
+  "#note_source_id": "source_id (model name) must be registered at https://github.com/WCRP-CMIP/CMIP6_CVs/issues/new ",
+
+  "source_id": "E3SM-1-0",
+
+  "calendar": "noleap",
+
+  "grid": "data regridded to a CMIP6 standard 1x1 degree lonxlat grid from the native grid using an area-average preserving method.",
+
+  "grid_label": "gr",
+
+  "nominal_resolution": "100 km",
+
+  "license": "CMIP6 model data produced by E3SM is licensed under a Creative Commons Attribution ShareAlike 4.0 International License (https://creativecommons.org/licenses). Consult https://pcmdi.llnl.gov/CMIP6/TermsOfUse for terms of use governing CMIP6 output, including citation requirements and proper acknowledgment. Further information about this data, including some limitations, can be found via the further_info_url (recorded as a global attribute in this file) and at https:///pcmdi.llnl.gov/. The data producers and data providers make no warranty, either express or implied, including, but not limited to, warranties of merchantability and fitness for a particular purpose. All liabilities arising from the supply of the information (including any liability arising in negligence) are excluded to the fullest extent permitted by law.",
+
+  "#output": "Root directory for output (can be either a relative or full path)",
+
+  "outpath": "CMIP6",
+
+  "#note_optional": " **** The following descriptors are optional and may be set to an empty string ",
+
+  "project PI": "Dave Bader ([email protected])",
+
+  "data contact": "[email protected]",
+
+  "history": "Output from 20180129.DECKv1b_piControl.ne30_oEC.edison. compset = A_WCYCL1850S_CMIP6",
+
+  "comment": "This is the default metadata file, any data  should only be used for testing purposes",
+
+  "references": "Golaz, J.-C., P. M. Caldwell, L. P. Van Roekel and co-authors, 2019: The DOE E3SM coupled model version 1: Overview and evaluation at standard resolution. JAMES, doi: 10.1029/2018MS001603; http://e3sm.org'",
+
+  "#note_CV": " **** The following will be obtained from the CV and do not need to be defined here",
+
+  "sub_experiment": "none",
+
+  "institution": "E3SM-Project",
+
+  "source": "E3SM 1.0 (2018)",
+
+  "#note_CMIP6": " **** The following are set correctly for CMIP6 and should not normally need editing",
+
+  "_control_vocabulary_file": "CMIP6_CV.json",
+
+  "_AXIS_ENTRY_FILE": "CMIP6_coordinate.json",
+
+  "_FORMULA_VAR_FILE": "CMIP6_formula_terms.json",
+
+  "_cmip6_option": "CMIP6",
+
+  "mip_era": "CMIP6",
+
+  "parent_mip_era": "CMIP6",
+
+  "tracking_prefix": "hdl:21.14100",
+
+  "_history_template": "%s ;rewrote data to be consistent with <activity_id> for variable <variable_id> found in table <table_id>.",
+
+  "#output_path_template": "Template for output path directory using tables keys or global attributes, these should follow the relevant data reference syntax",
+
+  "output_path_template": "<mip_era><activity_id><institution_id><source_id><experiment_id><_member_id><table><variable_id><grid_label><version>",
+
+  "output_file_template": "<variable_id><table><source_id><experiment_id><_member_id><grid_label>"
+}