Add cmor_name as an extra tag for getting variable info

.pre-commit-config.yaml

@@ -31,9 +31,9 @@ repos:
     hooks:
       - id: isort
   - repo: https://github.com/pre-commit/mirrors-yapf
-    rev: 'v0.30.0'
+    rev: 'v0.31.0'
     hooks:
-      - id: yapf
+      - id: yapf-diff
   - repo: https://github.com/myint/docformatter
     rev: 'v1.4'
     hooks:

esmvalcore/_recipe.py

@@ -71,7 +71,7 @@ def _add_cmor_info(variable, override=False):
     derive = variable.get('derive', False)
     table = CMOR_TABLES.get(project)
     if table:
-        table_entry = table.get_variable(mip, short_name, derive)
+        table_entry = table.get_variable(mip, variable['cmor_name'], derive)
     else:
         table_entry = None
     if table_entry is None:
@@ -141,7 +141,7 @@ def _update_target_levels(variable, variables, settings, config_user):
                 filename=filename,
                 project=variable_data['project'],
                 dataset=dataset,
-                short_name=variable_data['short_name'],
+                cmor_name=variable_data['cmor_name'],
                 mip=variable_data['mip'],
                 frequency=variable_data['frequency'],
                 fix_dir=os.path.splitext(variable_data['filename'])[0] +
@@ -201,7 +201,7 @@ def _dataset_to_file(variable, config_user):
     """Find the first file belonging to dataset from variable info."""
     (files, dirnames, filenames) = _get_input_files(variable, config_user)
     if not files and variable.get('derive'):
-        required_vars = get_required(variable['short_name'],
+        required_vars = get_required(variable['cmor_name'],
                                      variable['project'])
         for required_var in required_vars:
             _augment(required_var, variable)
@@ -264,7 +264,7 @@ def _get_default_settings(variable, config_user, derive=False):
     fix = {
         'project': variable['project'],
         'dataset': variable['dataset'],
-        'short_name': variable['short_name'],
+        'cmor_name': variable['cmor_name'],
         'mip': variable['mip'],
     }
     # File fixes
@@ -287,6 +287,7 @@ def _get_default_settings(variable, config_user, derive=False):
 
     if derive:
         settings['derive'] = {
+            'cmor_name': variable['cmor_name'],
             'short_name': variable['short_name'],
             'standard_name': variable['standard_name'],
             'long_name': variable['long_name'],
@@ -297,7 +298,7 @@ def _get_default_settings(variable, config_user, derive=False):
     settings['cmor_check_metadata'] = {
         'cmor_table': variable['project'],
         'mip': variable['mip'],
-        'short_name': variable['short_name'],
+        'cmor_name': variable['cmor_name'],
         'frequency': variable['frequency'],
         'check_level': config_user.get('check_level', CheckLevels.DEFAULT)
     }
@@ -330,6 +331,8 @@ def _add_fxvar_keys(fx_info, variable):
     fx_variable = deepcopy(variable)
     fx_variable.update(fx_info)
     fx_variable['variable_group'] = fx_info['short_name']
+    if 'cmor_name' not in fx_variable:
+        fx_variable['cmor_name'] = fx_info['short_name']
 
     # add special ensemble for CMIP5 only
     if fx_variable['project'] == 'CMIP5':
@@ -344,7 +347,7 @@ def _add_fxvar_keys(fx_info, variable):
 def _search_fx_mip(tables, found_mip, variable, fx_info, config_user):
     fx_files = None
     for mip in tables:
-        fx_cmor = tables[mip].get(fx_info['short_name'])
+        fx_cmor = tables[mip].get(fx_info['cmor_name'])
         if fx_cmor:
             found_mip = True
             fx_info['mip'] = mip
@@ -381,7 +384,7 @@ def _get_fx_files(variable, fx_info, config_user):
         found_mip, fx_info, fx_files = _search_fx_mip(
             project_tables, found_mip, variable, fx_info, config_user)
     else:
-        fx_cmor = project_tables[fx_info['mip']].get(fx_info['short_name'])
+        fx_cmor = project_tables[fx_info['mip']].get(fx_info['cmor_name'])
         if fx_cmor:
             found_mip = True
             fx_info = _add_fxvar_keys(fx_info, variable)
@@ -436,6 +439,8 @@ def _update_fx_files(step_name, settings, variable, config_user, fx_vars):
             fx_info.update({'mip': None})
         if 'short_name' not in fx_info:
             fx_info.update({'short_name': fx_var})
+        if 'cmor_name' not in fx_info:
+            fx_info.update({'cmor_name': fx_info['short_name']})
         fx_files, fx_info = _get_fx_files(variable, fx_info, config_user)
         if fx_files:
             fx_info['filename'] = fx_files
@@ -1097,6 +1102,8 @@ def _initialize_variables(self, raw_variable, raw_datasets):
                 if activity:
                     variable['activity'] = activity
             check.variable(variable, required_keys)
+            if 'cmor_name' not in variable:
+                variable['cmor_name'] = variable['short_name']
         variables = self._expand_ensemble(variables)
         return variables
 

esmvalcore/cmor/_fixes/fix.py

@@ -121,7 +121,7 @@ def __ne__(self, other):
         return not self.__eq__(other)
 
     @staticmethod
-    def get_fixes(project, dataset, mip, short_name):
+    def get_fixes(project, dataset, mip, cmor_name):
         """
         Get the fixes that must be applied for a given dataset.
 
@@ -141,19 +141,19 @@ def get_fixes(project, dataset, mip, short_name):
         project: str
         dataset: str
         mip: str
-        short_name: str
+        cmor_name: str
 
         Returns
         -------
         list(Fix)
             Fixes to apply for the given data
         """
         cmor_table = CMOR_TABLES[project]
-        vardef = cmor_table.get_variable(mip, short_name)
+        vardef = cmor_table.get_variable(mip, cmor_name)
 
         project = project.replace('-', '_').lower()
         dataset = dataset.replace('-', '_').lower()
-        short_name = short_name.replace('-', '_').lower()
+        cmor_name = cmor_name.replace('-', '_').lower()
 
         fixes = []
         try:
@@ -162,7 +162,7 @@ def get_fixes(project, dataset, mip, short_name):
 
             classes = inspect.getmembers(fixes_module, inspect.isclass)
             classes = dict((name.lower(), value) for name, value in classes)
-            for fix_name in (short_name, mip.lower(), 'allvars'):
+            for fix_name in (cmor_name, mip.lower(), 'allvars'):
                 try:
                     fixes.append(classes[fix_name](vardef))
                 except KeyError:

esmvalcore/cmor/check.py

@@ -939,7 +939,7 @@ def report_debug_message(self, message, *args):
 
 def _get_cmor_checker(table,
                       mip,
-                      short_name,
+                      cmor_name,
                       frequency,
                       fail_on_error=False,
                       check_level=CheckLevels.DEFAULT,
@@ -952,9 +952,9 @@ def _get_cmor_checker(table,
                 table, ', '.join(CMOR_TABLES)))
 
     cmor_table = CMOR_TABLES[table]
-    var_info = cmor_table.get_variable(mip, short_name)
+    var_info = cmor_table.get_variable(mip, cmor_name)
     if var_info is None:
-        var_info = CMOR_TABLES['custom'].get_variable(mip, short_name)
+        var_info = CMOR_TABLES['custom'].get_variable(mip, cmor_name)
 
     def _checker(cube):
         return CMORCheck(cube,
@@ -970,7 +970,7 @@ def _checker(cube):
 def cmor_check_metadata(cube,
                         cmor_table,
                         mip,
-                        short_name,
+                        cmor_name,
                         frequency,
                         check_level=CheckLevels.DEFAULT):
     """Check if metadata conforms to variable's CMOR definition.
@@ -985,16 +985,16 @@ def cmor_check_metadata(cube,
         CMOR definitions to use.
     mip:
         Variable's mip.
-    short_name: basestring
-        Variable's short name.
+    cmor_name: basestring
+        Variable's cmor name.
     frequency: basestring
         Data frequency.
     check_level: CheckLevels
         Level of strictness of the checks.
     """
     checker = _get_cmor_checker(cmor_table,
                                 mip,
-                                short_name,
+                                cmor_name,
                                 frequency,
                                 check_level=check_level)
     checker(cube).check_metadata()
@@ -1004,7 +1004,7 @@ def cmor_check_metadata(cube,
 def cmor_check_data(cube,
                     cmor_table,
                     mip,
-                    short_name,
+                    cmor_name,
                     frequency,
                     check_level=CheckLevels.DEFAULT):
     """Check if data conforms to variable's CMOR definition.
@@ -1019,23 +1019,23 @@ def cmor_check_data(cube,
         CMOR definitions to use.
     mip:
         Variable's mip.
-    short_name: basestring
-        Variable's short name
+    cmor_name: basestring
+        Variable's cmor name
     frequency: basestring
         Data frequency
     check_level: CheckLevels
         Level of strictness of the checks.
     """
     checker = _get_cmor_checker(cmor_table,
                                 mip,
-                                short_name,
+                                cmor_name,
                                 frequency,
                                 check_level=check_level)
     checker(cube).check_data()
     return cube
 
 
-def cmor_check(cube, cmor_table, mip, short_name, frequency, check_level):
+def cmor_check(cube, cmor_table, mip, cmor_name, frequency, check_level):
     """Check if cube conforms to variable's CMOR definition.
 
     Equivalent to calling cmor_check_metadata and cmor_check_data
@@ -1049,8 +1049,8 @@ def cmor_check(cube, cmor_table, mip, short_name, frequency, check_level):
         CMOR definitions to use.
     mip:
         Variable's mip.
-    short_name: basestring
-        Variable's short name.
+    cmor_name: basestring
+        Variable's cmor name.
     frequency: basestring
         Data frequency.
     check_level: enum.IntEnum
@@ -1059,13 +1059,13 @@ def cmor_check(cube, cmor_table, mip, short_name, frequency, check_level):
     cmor_check_metadata(cube,
                         cmor_table,
                         mip,
-                        short_name,
+                        cmor_name,
                         frequency,
                         check_level=check_level)
     cmor_check_data(cube,
                     cmor_table,
                     mip,
-                    short_name,
+                    cmor_name,
                     frequency,
                     check_level=check_level)
     return cube

esmvalcore/cmor/fix.py

@@ -4,6 +4,7 @@
 for the given dataset. Therefore is recommended to apply them to all
 variables to be sure that all known errors are fixed.
 """
+from esmvalcore.cmor.table import get_var_info
 import logging
 from collections import defaultdict
 
@@ -15,7 +16,7 @@
 logger = logging.getLogger(__name__)
 
 
-def fix_file(file, short_name, project, dataset, mip, output_dir):
+def fix_file(file, cmor_name, project, dataset, mip, output_dir):
     """Fix files before ESMValTool can load them.
 
     This fixes are only for issues that prevent iris from loading the cube or
@@ -27,7 +28,7 @@ def fix_file(file, short_name, project, dataset, mip, output_dir):
     ----------
     file: str
         Path to the original file
-    short_name: str
+    cmor_name: str
         Variable's short name
     project: str
     dataset:str
@@ -42,13 +43,13 @@ def fix_file(file, short_name, project, dataset, mip, output_dir):
     for fix in Fix.get_fixes(project=project,
                              dataset=dataset,
                              mip=mip,
-                             short_name=short_name):
+                             cmor_name=cmor_name):
         file = fix.fix_file(file, output_dir)
     return file
 
 
 def fix_metadata(cubes,
-                 short_name,
+                 cmor_name,
                  project,
                  dataset,
                  mip,
@@ -65,8 +66,8 @@ def fix_metadata(cubes,
     ----------
     cubes: iris.cube.CubeList
         Cubes to fix
-    short_name: str
-        Variable's short name
+    cmor_name: str
+        Variable's cmor name
     project: str
 
     dataset: str
@@ -92,22 +93,24 @@ def fix_metadata(cubes,
     fixes = Fix.get_fixes(project=project,
                           dataset=dataset,
                           mip=mip,
-                          short_name=short_name)
+                          cmor_name=cmor_name)
     fixed_cubes = []
     by_file = defaultdict(list)
     for cube in cubes:
         by_file[cube.attributes.get('source_file', '')].append(cube)
 
+    vardef = get_var_info(project, mip, cmor_name)
+
     for cube_list in by_file.values():
         cube_list = CubeList(cube_list)
         for fix in fixes:
             cube_list = fix.fix_metadata(cube_list)
 
-        cube = _get_single_cube(cube_list, short_name, project, dataset)
+        cube = _get_single_cube(cube_list, vardef.short_name, project, dataset)
         checker = _get_cmor_checker(frequency=frequency,
                                     table=project,
                                     mip=mip,
-                                    short_name=short_name,
+                                    cmor_name=cmor_name,
                                     check_level=check_level,
                                     fail_on_error=False,
                                     automatic_fixes=True)
@@ -142,7 +145,7 @@ def _get_single_cube(cube_list, short_name, project, dataset):
 
 
 def fix_data(cube,
-             short_name,
+             cmor_name,
              project,
              dataset,
              mip,
@@ -161,8 +164,8 @@ def fix_data(cube,
     ----------
     cube: iris.cube.Cube
         Cube to fix
-    short_name: str
-        Variable's short name
+    cmor_name: str
+        Variable's cmor name
     project: str
     dataset: str
     mip: str
@@ -185,12 +188,12 @@ def fix_data(cube,
     for fix in Fix.get_fixes(project=project,
                              dataset=dataset,
                              mip=mip,
-                             short_name=short_name):
+                             cmor_name=cmor_name):
         cube = fix.fix_data(cube)
     checker = _get_cmor_checker(frequency=frequency,
                                 table=project,
                                 mip=mip,
-                                short_name=short_name,
+                                cmor_name=cmor_name,
                                 fail_on_error=False,
                                 automatic_fixes=True,
                                 check_level=check_level)