Merge pull request #260 from bstephens82/prect_mods

Adding mods to derive PRECT

.github/scripts/pylint_threshold_test.py

@@ -87,7 +87,7 @@ def pylint_check(pyfile_list, rcfile, threshold=10.0):
 
         #Run linter:
         lint_results = lint.Run([rcstr, '--exit-zero', pyfile],
-                                reporter=pylint_report, do_exit=False)
+                                reporter=pylint_report, exit=False)
 
         #Extract linter score:
         lint_score = lint_results.linter.stats.global_note

lib/adf_diag.py

@@ -379,6 +379,9 @@ def call_ncrcat(cmd):
             hist_str = 'cam.h0'
         #End if
 
+        # get info about variable defaults
+        res = self.variable_defaults
+
         #Loop over cases:
         for case_idx, case_name in enumerate(case_names):
 
@@ -510,12 +513,24 @@ def call_ncrcat(cmd):
 
             #Loop over CAM history variables:
             list_of_commands = []
-            for var in self.diag_var_list:
+            vars_to_derive = []
+            #create copy of var list that can be modified for derivable variables
+            diag_var_list = self.diag_var_list
+            for var in diag_var_list:
                 if var not in hist_file_var_list:
-                    msg = f"WARNING: {var} is not in the file {hist_files[0]}."
-                    msg += " No time series will be generated."
-                    print(msg)
-                    continue
+                    vres = res.get(var, {})
+                    if "derivable_from" in vres:
+                        constit_list = vres['derivable_from']
+                        for constit in constit_list:
+                            if constit not in diag_var_list:
+                                diag_var_list.append(constit)
+                        vars_to_derive.append(var)
+                        continue
+                    else:
+                        msg = f"WARNING: {var} is not in the file {hist_files[0]}."
+                        msg += " No time series will be generated."
+                        print(msg)
+                        continue
 
                 #Check if variable has a "lev" dimension according to first file:
                 has_lev = bool('lev' in hist_file_ds[var].dims)
@@ -592,6 +607,9 @@ def call_ncrcat(cmd):
             #Now run the "ncrcat" subprocesses in parallel:
             with mp.Pool(processes=self.num_procs) as mpool:
                 _ = mpool.map(call_ncrcat, list_of_commands)
+
+            if vars_to_derive:
+                self.derive_variables(vars_to_derive=vars_to_derive,ts_dir=ts_dir[case_idx])
             #End with
 
         #End cases loop
@@ -915,4 +933,30 @@ def setup_run_cvdp(self):
         print('For CVDP information visit: https://www.cesm.ucar.edu/working_groups/CVC/cvdp/')
         print('   ')
 
+    #########
+
+    def derive_variables(self,vars_to_derive=None,ts_dir=None):
+
+        """
+        Derive variables acccording to steps given here.  Since derivations will depend on the 
+        variable, each variable to derive will need its own set of steps below.
+        """
+
+        for var in vars_to_derive:
+            if var == "PRECT":
+                 # PRECT can be found by simply adding PRECL and PRECC
+                 # grab file names for the PRECL and PRECC files from the case ts directory
+                 if glob.glob(os.path.join(ts_dir,"*PRECC*")) and glob.glob(os.path.join(ts_dir,"*PRECL*")):
+                     constit_files=sorted(glob.glob(os.path.join(ts_dir,"*PREC*")))
+                 else:
+                     ermsg = "PRECC and PRECL were not both present; PRECT cannot be calculated."
+                     ermsg += " Please remove PRECT from diag_var_list or find the relevant CAM files."
+                     raise FileNotFoundError(ermsg)
+                 # create new file name for PRECT
+                 prect_file = constit_files[0].replace('PRECC','PRECT')
+                 # append PRECC to the file containing PRECL
+                 os.system(f"ncks -A -v PRECC {constit_files[0]} {constit_files[1]}")
+                 # create new file with the sum of PRECC and PRECL
+                 os.system(f"ncap2 -s 'PRECT=(PRECC+PRECL)' {constit_files[1]} {prect_file}")
+
 ###############

lib/adf_variable_defaults.yaml

@@ -49,6 +49,12 @@
 # category  -> The website category the variable will be placed under.
 #
 #
+# DERIVING:
+#
+# derivable_from -> If not present in the available output files, the variable can be derived from
+#                   other variables that are present (e.g. PRECT can be derived from PRECC and PRECL),
+#                   which are specified in this list
+#
 # Final Note:  Please do not modify this file unless you plan to push your changes back to the ADF repo.
 #              If you would like to modify this file for your personal ADF runs then it is recommended
 #              to make a copy of this file, make modifications in that copy, and then point the ADF to
@@ -445,6 +451,7 @@ PRECT:
   obs_name: "ERAI"
   obs_var_name: "PRECT"
   category: "Hydrologic cycle"
+  derivable_from: ['PRECL','PRECC']
 
 QFLX:
   category: "Hydrologic cycle"