Convert yeb to eb

easybuild/framework/easyconfig/easyconfig.py

@@ -1068,7 +1068,7 @@ def all_dependencies(self):
 
         return self._all_dependencies
 
-    def dump(self, fp, always_overwrite=True, backup=False):
+    def dump(self, fp, always_overwrite=True, backup=False, convert_yeb=False):
         """
         Dump this easyconfig to file, with the given filename.
 
@@ -1097,13 +1097,16 @@ def dump(self, fp, always_overwrite=True, backup=False):
             if self.template_values[key] not in templ_val and len(self.template_values[key]) > 2:
                 templ_val[self.template_values[key]] = key
 
-        try:
-            ectxt = self.parser.dump(self, default_values, templ_const, templ_val)
-        except NotImplementedError as err:
-            # need to restore enable_templating value in case this method is caught in a try/except block and ignored
-            # (the ability to dump is not a hard requirement for build success)
-            self.enable_templating = orig_enable_templating
-            raise NotImplementedError(err)
+        if convert_yeb and is_yeb_format(self.path, None):
+            ectxt = self.parser.convert_and_dump_yeb_as_eb(self, default_values, templ_const, templ_val)
+        else:
+            try:
+                ectxt = self.parser.dump(self, default_values, templ_const, templ_val)
+            except NotImplementedError as err:
+                # restore enable_templating value in case this method is caught in a try/except block and ignored
+                # (the ability to dump is not a hard requirement for build success)
+                self.enable_templating = orig_enable_templating
+                raise NotImplementedError(err)
 
         self.log.debug("Dumped easyconfig: %s", ectxt)
 

easybuild/framework/easyconfig/format/one.py

@@ -127,7 +127,7 @@ def parse(self, txt):
         """
         super(FormatOneZero, self).parse(txt, strict_section_markers=True)
 
-    def _reformat_line(self, param_name, param_val, outer=False, addlen=0):
+    def _reformat_line(self, param_name, param_val, outer=False, addlen=0, comments=True):
         """
         Construct formatted string representation of iterable parameter (list/tuple/dict), including comments.
 
@@ -162,7 +162,8 @@ def _reformat_line(self, param_name, param_val, outer=False, addlen=0):
                     ordered_item_keys = REFORMAT_ORDERED_ITEM_KEYS.get(param_name, sorted(param_val.keys()))
                     for item_key in ordered_item_keys:
                         item_val = param_val[item_key]
-                        comment = self._get_item_comments(param_name, item_val).get(str(item_val), '')
+                        comment = self._get_item_comments(param_name, item_val,
+                                                          comments=comments).get(str(item_val), '')
                         key_pref = quote_py_str(item_key) + ': '
                         addlen = addlen + len(INDENT_4SPACES) + len(key_pref) + len(comment)
                         formatted_item_val = self._reformat_line(param_name, item_val, addlen=addlen)
@@ -172,7 +173,7 @@ def _reformat_line(self, param_name, param_val, outer=False, addlen=0):
                         }
                 else:  # list, tuple
                     for item in param_val:
-                        comment = self._get_item_comments(param_name, item).get(str(item), '')
+                        comment = self._get_item_comments(param_name, item, comments=comments).get(str(item), '')
                         addlen = addlen + len(INDENT_4SPACES) + len(comment)
                         # the tuples are really strings here that are constructed from the dependency dicts
                         # so for a plain list of builddependencies param_val is a list of strings here;
@@ -200,35 +201,39 @@ def _reformat_line(self, param_name, param_val, outer=False, addlen=0):
 
         return res
 
-    def _get_item_comments(self, key, val):
+    def _get_item_comments(self, key, val, comments=True):
         """Get per-item comments for specified parameter name/value."""
         item_comments = {}
-        for comment_key, comment_val in self.comments['iter'].get(key, {}).items():
-            if str(val) in comment_key:
-                item_comments[str(val)] = comment_val
+        if comments:
+            for comment_key, comment_val in self.comments['iter'].get(key, {}).items():
+                if str(val) in comment_key:
+                    item_comments[str(val)] = comment_val
 
         return item_comments
 
-    def _find_param_with_comments(self, key, val, templ_const, templ_val):
+    def _find_param_with_comments(self, key, val, templ_const, templ_val, comments=True):
         """Find parameter definition and accompanying comments, to include in dumped easyconfig file."""
         res = []
 
-        val = self._reformat_line(key, val, outer=True)
+        val = self._reformat_line(key, val, outer=True, comments=comments)
 
         # templates
         if key not in EXCLUDED_KEYS_REPLACE_TEMPLATES:
             val = to_template_str(key, val, templ_const, templ_val)
 
-        if key in self.comments['inline']:
-            res.append("%s = %s%s" % (key, val, self.comments['inline'][key]))
+        if comments:
+            if key in self.comments['inline']:
+                res.append("%s = %s%s" % (key, val, self.comments['inline'][key]))
+            else:
+                if key in self.comments['above']:
+                    res.extend(self.comments['above'][key])
+                res.append("%s = %s" % (key, val))
         else:
-            if key in self.comments['above']:
-                res.extend(self.comments['above'][key])
             res.append("%s = %s" % (key, val))
 
         return res
 
-    def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_val):
+    def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_val, comments=True):
         """
         Determine parameters in the dumped easyconfig file which have a non-default value.
         """
@@ -258,7 +263,8 @@ def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_
                     else:
                         valstr = quote_py_str(ecfg[key])
 
-                    eclines.extend(self._find_param_with_comments(key, valstr, templ_const, templ_val))
+                    eclines.extend(self._find_param_with_comments(key, valstr, templ_const, templ_val,
+                                                                  comments=comments))
 
                     printed_keys.append(key)
                     printed = True
@@ -267,7 +273,7 @@ def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_
 
         return eclines, printed_keys
 
-    def dump(self, ecfg, default_values, templ_const, templ_val):
+    def dump(self, ecfg, default_values, templ_const, templ_val, comments=True):
         """
         Dump easyconfig in format v1.
 
@@ -277,24 +283,30 @@ def dump(self, ecfg, default_values, templ_const, templ_val):
         :param templ_val: known template values
         """
         # include header comments first
-        dump = self.comments['header'][:]
+        dump = []
+        if comments:
+            dump = self.comments['header'][:]
 
         # print easyconfig parameters ordered and in groups specified above
-        params, printed_keys = self._find_defined_params(ecfg, GROUPED_PARAMS, default_values, templ_const, templ_val)
+        params, printed_keys = self._find_defined_params(ecfg, GROUPED_PARAMS, default_values, templ_const, templ_val,
+                                                         comments=comments)
         dump.extend(params)
 
         # print other easyconfig parameters at the end
         keys_to_ignore = printed_keys + LAST_PARAMS
         for key in default_values:
             if key not in keys_to_ignore and ecfg[key] != default_values[key]:
-                dump.extend(self._find_param_with_comments(key, quote_py_str(ecfg[key]), templ_const, templ_val))
+                dump.extend(self._find_param_with_comments(key, quote_py_str(ecfg[key]), templ_const, templ_val,
+                                                           comments=comments))
         dump.append('')
 
         # print last parameters
-        params, _ = self._find_defined_params(ecfg, [[k] for k in LAST_PARAMS], default_values, templ_const, templ_val)
+        params, _ = self._find_defined_params(ecfg, [[k] for k in LAST_PARAMS], default_values, templ_const, templ_val,
+                                              comments=comments)
         dump.extend(params)
 
-        dump.extend(self.comments['tail'])
+        if comments:
+            dump.extend(self.comments['tail'])
 
         return '\n'.join(dump)
 

easybuild/framework/easyconfig/parser.py

@@ -226,6 +226,18 @@ def get_config_dict(self, validate=True):
 
         return cfg
 
+    def convert_and_dump_yeb_as_eb(self, ecfg, default_values, templ_const, templ_val):
+        """Convert a parsed yeb easyconfig as a format 1 file"""
+        # First need to fake the format version
+        original_formatter = self._formatter
+        klass = self._get_format_version_class()
+        self._formatter = klass()
+        # Comments are not retained by current yeb format
+        dump_txt = self._formatter.dump(ecfg, default_values, templ_const, templ_val, comments=False)
+        # Return to original
+        self._formatter = original_formatter
+        return dump_txt
+
     def dump(self, ecfg, default_values, templ_const, templ_val):
         """Dump easyconfig in format it was parsed from."""
         return self._formatter.dump(ecfg, default_values, templ_const, templ_val)

test/framework/yeb.py

@@ -110,6 +110,53 @@ def test_parse_yeb(self):
 
                 self.assertEqual(yeb_val, eb_val)
 
+    def test_yeb_to_eb_dump(self):
+        """Test parsing of .yeb easyconfig and dumping as .eb easyconfig."""
+        if 'yaml' not in sys.modules:
+            print("Skipping test_parse_yeb (no PyYAML available)")
+            return
+
+        build_options = {
+            'check_osdeps': False,
+            'external_modules_metadata': {},
+            'valid_module_classes': module_classes(),
+        }
+        init_config(build_options=build_options)
+        easybuild.tools.build_log.EXPERIMENTAL = True
+
+        testdir = os.path.dirname(os.path.abspath(__file__))
+        test_easyconfigs = os.path.join(testdir, 'easyconfigs')
+        test_yeb_easyconfigs = os.path.join(testdir, 'easyconfigs', 'yeb')
+
+        # test parsing
+        test_files = [
+            'bzip2-1.0.6-GCC-4.9.2',
+            'CrayCCE-5.1.29',
+            'gzip-1.6-GCC-4.9.2',
+            'intel-2018a',
+            'toy-0.0',
+        ]
+
+        for filename in test_files:
+            ec_yeb = EasyConfig(os.path.join(test_yeb_easyconfigs, '%s.yeb' % filename))
+            # Temporarily dump the parsed yeb
+            dumped_file = '%s/%s.eb' % (self.test_prefix, filename)
+            ec_yeb.dump(dumped_file, convert_yeb=True)
+            # Read the dumped file again
+            dumped_ec_eb = EasyConfig(dumped_file)
+            # compare with parsed result of .eb easyconfig
+            ec_file = glob.glob(os.path.join(test_easyconfigs, 'test_ecs', '*', '*', '%s.eb' % filename))[0]
+            ec_eb = EasyConfig(ec_file)
+
+            for key in sorted(dumped_ec_eb.asdict()):
+                eb_val = ec_eb[key]
+                yeb_val = dumped_ec_eb[key]
+                if key == 'description':
+                    # multi-line string is always terminated with '\n' in YAML, so strip it off
+                    yeb_val = yeb_val.strip()
+
+                self.assertEqual(yeb_val, eb_val)
+
     def test_is_yeb_format(self):
         """ Test is_yeb_format function """
         testdir = os.path.dirname(os.path.abspath(__file__))