integrate CMIP time-series generation

setup.py

@@ -18,7 +18,7 @@ def package_files(directory, prefixes, extensions):
 
 
 data_files = package_files(
-    "zppy", prefixes=[], extensions=["bash", "csh", "cfg", "ini", "sh"]
+    "zppy", prefixes=[], extensions=["bash", "csh", "cfg", "ini", "sh", "json"]
 )
 
 setup(

zppy/__main__.py

@@ -9,6 +9,7 @@
 from zppy.climo import climo
 from zppy.e3sm_diags import e3sm_diags
 from zppy.global_time_series import global_time_series
+from zppy.ilamb_run import ilamb_run
 from zppy.mpas_analysis import mpas_analysis
 from zppy.tc_analysis import tc_analysis
 from zppy.ts import ts
@@ -117,6 +118,9 @@ def main():
     # global time series tasks
     global_time_series(config, scriptDir)
 
+    # ilamb_run tasks
+    ilamb_run(config, scriptDir)
+
 
 def _validate_config(config):
     validator = Validator()

zppy/ilamb_run.py

@@ -0,0 +1,92 @@
+import os
+import pprint
+
+import jinja2
+
+from zppy.utils import checkStatus, getTasks, getYears, submitScript
+
+
+# -----------------------------------------------------------------------------
+def ilamb_run(config, scriptDir):
+
+    # Initialize jinja2 template engine
+    templateLoader = jinja2.FileSystemLoader(
+        searchpath=config["default"]["templateDir"]
+    )
+    templateEnv = jinja2.Environment(loader=templateLoader)
+    template = templateEnv.get_template("ilamb_run.bash")
+
+    # --- List of ilamb_run tasks ---
+    tasks = getTasks(config, "ilamb_run")
+    if len(tasks) == 0:
+        return
+
+    # --- Generate and submit ilamb_run scripts ---
+    dependencies = []
+
+    for c in tasks:
+
+        if "ts_num_years" in c.keys():
+            c["ts_num_years"] = int(c["ts_num_years"])
+
+        # Loop over year sets
+        year_sets = getYears(c["years"])
+        for s in year_sets:
+            c["year1"] = s[0]
+            c["year2"] = s[1]
+            c["scriptDir"] = scriptDir
+
+            # List of dependencies
+            dependencies.append(
+                os.path.join(
+                    scriptDir,
+                    "ts_%s_%04d-%04d-%04d.status"
+                    % ("land_monthly", c["year1"], c["year2"], c["ts_num_years"]),
+                ),
+            )
+            if not c["land_only"]:
+                dependencies.append(
+                    os.path.join(
+                        scriptDir,
+                        "ts_%s_%04d-%04d-%04d.status"
+                        % (
+                            "atm_monthly_180x360_aave",
+                            c["year1"],
+                            c["year2"],
+                            c["ts_num_years"],
+                        ),
+                    ),
+                )
+
+            prefix = "ilamb_run_%04d-%04d" % (
+                c["year1"],
+                c["year2"],
+            )
+            c["prefix"] = prefix
+            print(prefix)
+            scriptFile = os.path.join(scriptDir, "%s.bash" % (prefix))
+            statusFile = os.path.join(scriptDir, "%s.status" % (prefix))
+            settingsFile = os.path.join(scriptDir, "%s.settings" % (prefix))
+            skip = checkStatus(statusFile)
+            if skip:
+                continue
+
+            # Create script
+            with open(scriptFile, "w") as f:
+                f.write(template.render(**c))
+
+            with open(settingsFile, "w") as sf:
+                p = pprint.PrettyPrinter(indent=2, stream=sf)
+                p.pprint(c)
+                p.pprint(s)
+
+            if not c["dry_run"]:
+                # Submit job
+                jobid = submitScript(
+                    scriptFile, dependFiles=dependencies, export="NONE"
+                )
+
+                if jobid != -1:
+                    # Update status file
+                    with open(statusFile, "w") as f:
+                        f.write("WAITING %d\n" % (jobid))

zppy/templates/default.ini

@@ -64,6 +64,9 @@ dpf = integer(default=30)
 extra_vars = string(default="")
 # Time-steps per day
 tpd = integer(default=1)
+ts_fmt = string(default="ts_only")
+cmip_metadata = string(default="e3sm_to_cmip/default_metadata.json")
+
 
   [[__many__]]
   area_nm = string(default=None)
@@ -253,3 +256,10 @@ moc_file = string(default="")
 ts_num_years = integer(default=10)
 ts_years = string_list(default=list(""))
 # `years = "1-100",` would plot years 1 to 100 on the graphs.
+
+[ilamb_run]
+qos = string(default="regular")
+nodes = integer(default=1)
+walltime = string(default="02:00:00")
+land_only = boolean(default=False)
+cfg = string(default="ilamb_run/cmip.cfg")

zppy/templates/e3sm_to_cmip/default_metadata.json

@@ -0,0 +1,107 @@
+{
+  "#note_source_type": "explanation of what source_type is goes here",
+
+  "source_type": "AOGCM AER",
+
+  "#note_experiment_id": "CMIP6 valid experiment_ids are found in CMIP6_CV.json",
+
+  "experiment_id": "piControl",
+
+  "activity_id": "CMIP",
+
+  "sub_experiment_id": "none",
+
+  "realization_index": "1",
+
+  "initialization_index": "1",
+
+  "physics_index": "1",
+
+  "forcing_index": "1",
+
+  "#note_run_variant": "Text stored in attribute variant_info (recommended, not required description of run variant)",
+
+  "run_variant": "",
+
+  "parent_experiment_id": "piControl-spinup",
+
+  "parent_activity_id": "CMIP",
+
+  "parent_source_id": "E3SM-1-0",
+
+  "parent_variant_label": "r1i1p1f1",
+
+  "parent_time_units": "days since 0001-01-01",
+
+  "branch_method": "standard",
+
+  "branch_time_in_child": 0.0,
+
+  "branch_time_in_parent": 0.0,
+
+  "#note_institution_id": "institution_id must be registered at https://github.com/WCRP-CMIP/CMIP6_CVs/issues/new ",
+
+  "institution_id": "E3SM-Project",
+
+  "#note_source_id": "source_id (model name) must be registered at https://github.com/WCRP-CMIP/CMIP6_CVs/issues/new ",
+
+  "source_id": "E3SM-1-0",
+
+  "calendar": "noleap",
+
+  "grid": "data regridded to a CMIP6 standard 1x1 degree lonxlat grid from the native grid using an area-average preserving method.",
+
+  "grid_label": "gr",
+
+  "nominal_resolution": "100 km",
+
+  "license": "CMIP6 model data produced by E3SM is licensed under a Creative Commons Attribution ShareAlike 4.0 International License (https://creativecommons.org/licenses). Consult https://pcmdi.llnl.gov/CMIP6/TermsOfUse for terms of use governing CMIP6 output, including citation requirements and proper acknowledgment. Further information about this data, including some limitations, can be found via the further_info_url (recorded as a global attribute in this file) and at https:///pcmdi.llnl.gov/. The data producers and data providers make no warranty, either express or implied, including, but not limited to, warranties of merchantability and fitness for a particular purpose. All liabilities arising from the supply of the information (including any liability arising in negligence) are excluded to the fullest extent permitted by law.",
+
+  "#output": "Root directory for output (can be either a relative or full path)",
+
+  "outpath": "CMIP6",
+
+  "#note_optional": " **** The following descriptors are optional and may be set to an empty string ",
+
+  "project PI": "Dave Bader ([email protected])",
+
+  "data contact": "[email protected]",
+
+  "history": "Output from 20180129.DECKv1b_piControl.ne30_oEC.edison. compset = A_WCYCL1850S_CMIP6",
+
+  "comment": "This is the default metadata file, any data  should only be used for testing purposes",
+
+  "references": "Golaz, J.-C., P. M. Caldwell, L. P. Van Roekel and co-authors, 2019: The DOE E3SM coupled model version 1: Overview and evaluation at standard resolution. JAMES, doi: 10.1029/2018MS001603; http://e3sm.org'",
+
+  "#note_CV": " **** The following will be obtained from the CV and do not need to be defined here",
+
+  "sub_experiment": "none",
+
+  "institution": "E3SM-Project",
+
+  "source": "E3SM 1.0 (2018)",
+
+  "#note_CMIP6": " **** The following are set correctly for CMIP6 and should not normally need editing",
+
+  "_control_vocabulary_file": "CMIP6_CV.json",
+
+  "_AXIS_ENTRY_FILE": "CMIP6_coordinate.json",
+
+  "_FORMULA_VAR_FILE": "CMIP6_formula_terms.json",
+
+  "_cmip6_option": "CMIP6",
+
+  "mip_era": "CMIP6",
+
+  "parent_mip_era": "CMIP6",
+
+  "tracking_prefix": "hdl:21.14100",
+
+  "_history_template": "%s ;rewrote data to be consistent with <activity_id> for variable <variable_id> found in table <table_id>.",
+
+  "#output_path_template": "Template for output path directory using tables keys or global attributes, these should follow the relevant data reference syntax",
+
+  "output_path_template": "<mip_era><activity_id><institution_id><source_id><experiment_id><_member_id><table><variable_id><grid_label><version>",
+
+  "output_file_template": "<variable_id><table><source_id><experiment_id><_member_id><grid_label>"
+}

zppy/templates/ilamb_run.bash

@@ -0,0 +1,112 @@
+#!/bin/bash
+{% include 'slurm_header.sh' %}
+
+{{ environment_commands }}
+
+# Turn on debug output if needed
+debug={{ debug }}
+if [[ "${debug,,}" == "true" ]]; then
+  set -x
+fi
+
+# Point to obsersvation data
+# TODO: need to update these data to other supported machines
+export ILAMB_ROOT=/lcrc/group/acme/public_html/diagnostics/ilamb_data
+
+# Script dir
+cd {{ scriptDir }}
+
+# Get jobid
+id=${SLURM_JOBID}
+
+# Update status file
+STARTTIME=$(date +%s)
+echo "RUNNING ${id}" > {{ prefix }}.status
+
+# Basic definitions
+case="{{ case }}"
+short="{{ short_name }}"
+www="{{ www }}"
+y1={{ year1 }}
+y2={{ year2 }}
+Y1="{{ '%04d' % (year1) }}"
+Y2="{{ '%04d' % (year2) }}"
+scriptDir="{{ scriptDir }}"
+
+# Create temporary workdir
+workdir=`mktemp -d tmp.${id}.XXXX`
+workdir=${scriptDir}/${workdir}
+model_root=${workdir}/model_data
+
+if [ $short != "" ]; then
+    case=${short}
+fi
+
+mkdir -p ${model_root}/${case}
+cd ${model_root}/${case}
+
+# Create output directory
+# Create local links to input cmip time-series files
+lnd_ts_for_ilamb={{ output }}/post/lnd/180x360_aave/cmip_ts/monthly/
+atm_ts_for_ilamb={{ output }}/post/atm/180x360_aave/cmip_ts/monthly/
+cp -s ${lnd_ts_for_ilamb}/*_*_*_*_*_*_${Y1}??-${Y2}??.nc .
+cp -s ${atm_ts_for_ilamb}/*_*_*_*_*_*_${Y1}??-${Y2}??.nc .
+cd ../..
+
+echo
+echo ===== RUN ILAMB =====
+echo
+
+# Run diagnostics
+# Note --export=All is needed to make sure the executable is copied and executed  on the nodes.
+# TODO: find the mpi run format for different platforms
+
+# include cfg file
+cat > ilamb_run.cfg << EOF
+{% include cfg %}
+EOF
+
+echo ${workdir}
+echo {{ scriptDir }}
+
+srun --export=ALL -N 1 ilamb-run --config ilamb_run.cfg --model_root $model_root  --regions global --model_year ${Y1} 2000
+
+if [ $? != 0 ]; then
+  cd {{ scriptDir }}
+  echo 'ERROR (2)' > {{ prefix }}.status
+  exit 2
+fi
+
+# Copy output to web server
+echo
+echo ===== COPY FILES TO WEB SERVER =====
+echo
+
+# TODO copy _build from $workdir to web server
+
+## Create top-level directory
+#f=${www}/${case}/ilamb/{{ grid }}
+#mkdir -p ${f}
+#if [ $? != 0 ]; then
+#  cd {{ scriptDir }}
+#  echo 'ERROR (3)' > {{ prefix }}.status
+#  exit 3
+#fi
+#
+## Delete temporary workdir
+#cd ${workdir}
+#cd ..
+#if [[ "${debug,,}" != "true" ]]; then
+#  rm -rf ${workdir}
+#fi
+
+# Update status file and exit
+{% raw %}
+ENDTIME=$(date +%s)
+ELAPSEDTIME=$(($ENDTIME - $STARTTIME))
+{% endraw %}
+echo ==============================================
+echo "Elapsed time: $ELAPSEDTIME seconds"
+echo ==============================================
+echo 'OK' > {{ prefix }}.status
+exit 0