Feature Parity for Open Embedded (ros-infrastructure#79)

* Switch to use util functions.

* Use the generate_installers function from superflore.generate_installers

* Linting

* Change the ros_meta class to be RosMeta, as well as changed the inheritance
to be a member of the class (see ros-infrastructure#52).

* Removed the sym-linked structure to behave more like the Gentoo generator (see ros-infrastructure#51).

* Removed a commented line from the ebuild logic copypasta, changed the print format slightly for the exception output after a failed pr, and called the file_pr function.

* Add temporary file manager code, as suggested by @tfoote.

* Switch to utils print functions for TempfileManager output.

* Use tempfile manager to clean up temp directories at exit.

* Linting.

* Remove hard-coded org and repo from RepoInstance call.

* Reorder import statements.

* Ok, that's pretty cool.

* Apply new TempfileManager usage to OpenEmbedded logic.

* OpenEmbedded generation is now running. ToDo: automatically clean up the tar files.

* Files were not generating in the directory, but are now.

* Fixed argument mapping for ebuild generator.

superflore/TempfileManager.py

@@ -0,0 +1,57 @@
+# Copyright 2017 Open Source Robotics Foundation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import errno
+import os
+import shutil
+import subprocess
+import tempfile
+
+from superflore.utils import err
+from superflore.utils import info
+
+
+class TempfileManager:
+    def __init__(self, path):
+        self.arg_path = path
+        self.temp_path = None
+
+    def __enter__(self):
+        if self.arg_path:
+            try:
+                os.mkdir(self.arg_path)
+            except OSError as ex:
+                if ex.errno != errno.EEXIST:
+                    raise
+            return self.arg_path
+        else:
+            self.temp_path = tempfile.mkdtemp()
+            info("Working in temporary directory %s" % self.temp_path)
+        return self.temp_path
+
+    def __exit__(self, *args):
+        if self.temp_path:
+            info("Cleaning up temporary directory %s" % self.temp_path)
+            try:
+                shutil.rmtree(self.temp_path)
+            except OSError as ex:
+                if ex.errno == errno.EPERM:
+                    err("Failed to rmtree %s" % self.temp_path)
+                    err("Escalating to sudo rm -rf %s" % self.temp_path)
+                    subprocess.check_call(
+                        ('sudo rm -rf %s' % self.temp_path).split()
+                    )
+                else:
+                    raise
+            self.temp_path = None

superflore/generators/bitbake/gen_packages.py

@@ -13,173 +13,100 @@
 # limitations under the License.
 
 import glob
-import os
 import sys
 
 from rosdistro.dependency_walker import DependencyWalker
 from rosdistro.manifest_provider import get_release_tag
 from rosdistro.rosdistro import RosPackage
 
 from rosinstall_generator.distro import _generate_rosinstall
-from rosinstall_generator.distro import get_distro
 from rosinstall_generator.distro import get_package_names
 
 from superflore.exceptions import NoPkgXml
 from superflore.exceptions import UnresolvedDependency
 
-from termcolor import colored
+from superflore.generators.bitbake.yocto_recipe import yoctoRecipe
 
-import xmltodict
+from superflore.utils import err
+from superflore.utils import get_pkg_version
+from superflore.utils import info
+from superflore.utils import make_dir
+from superflore.utils import ok
+from superflore.utils import warn
 
-from .yocto_recipe import yoctoRecipe
+import xmltodict
 
 org = "Open Source Robotics Foundation"
 org_license = "BSD"
 
 
-def warn(string):
-    print(colored('>>>> {0}'.format(string), 'yellow'))
-
-
-def ok(string):
-    print(colored('>>>> {0}'.format(string), 'green'))
-
-
-def err(string):
-    print(colored('!!!! {0}'.format(string), 'red'))
-
-
-def info(string):
-    print(colored('>>>> {0}'.format(string), 'cyan'))
-
-
-def make_dir(dirname):
-    try:
-        os.makedirs(dirname)
-    except Exception:
-        pass
-
-
-def get_pkg_version(distro, pkg_name):
-    pkg = distro.release_packages[pkg_name]
-    repo = distro.repositories[pkg.repository_name].release_repository
-    maj_min_patch, deb_inc = repo.version.split('-')
-    if deb_inc != '0':
-        return '{0}-r{1}'.format(maj_min_patch, deb_inc)
-    return maj_min_patch
-
-
-def generate_installers(distro_name, overlay, preserve_existing=True):
-    make_dir("recipes-ros-{}".format(distro_name))
-    distro = get_distro(distro_name)
-    pkg_names = get_package_names(distro)
-    total = float(len(pkg_names[0]))
-    borkd_pkgs = dict()
-    changes = []
-    installers = []
-    succeeded = 0
-    failed = 0
+def regenerate_installer(overlay, pkg, distro, preserve_existing=False):
+    make_dir("{0}/recipes-ros-{1}".format(overlay.repo.repo_dir, distro.name))
+    version = get_pkg_version(distro, pkg)
+    pkg_names = get_package_names(distro)[0]
 
-    for i, pkg in enumerate(sorted(pkg_names[0])):
-        version = get_pkg_version(distro, pkg)
-        """
-        recipe_exists = os.path.exists(
-            'ros-{}/{}/{}-{}.ebuild'.format(distro_name, pkg, pkg, version))
-        patch_path = 'ros-{}/{}/files'.format(distro_name, pkg)
-        has_patches = os.path.exists(patch_path)
-        """
-        percent = '%.1f' % (100 * (float(i) / total))
+    if pkg not in pkg_names:
+        raise RuntimeError("Unknown package '%s'" % pkg)
 
-        """
-        if preserve_existing and ebuild_exists:
-            skip_msg = 'Ebuild for package '
-            skip_msg += '{0} up to date, skipping...'.format(pkg)
-            status = '{0}%: {1}'.format(percent, skip_msg)
-            ok(status)
-            succeeded = succeeded + 1
-            continue
-        """
-        # otherwise, remove a (potentially) existing recipe.
-        existing = glob.glob(
-            'recipes-ros-{0}/{1}/*.bb'.format(distro_name, pkg)
+    # check for an existing recipe
+    existing = glob.glob(
+        '{0}/recipes-ros-{1}/{2}/*.bb'.format(
+            overlay.repo.repo_dir,
+            distro.name,
+            pkg
         )
-        if len(existing) > 0:
-            overlay.remove_file(existing[0])
-        try:
-            current = oe_installer(distro, pkg)
-            current.recipe.name = pkg.replace('_', '-')
-        except Exception as e:
-            err('Failed to generate installer for package {}!'.format(pkg))
-            err('  exception: {0}'.format(e))
-            failed = failed + 1
-            continue
-        try:
-            info(' downloading archive version for package \'%s\'' % pkg)
-            current.recipe.downloadArchive()
-            recipe_text = current.recipe_text()
-        except NoPkgXml:
-            err('  No package.xml file for pkg \'{0}\'!'.format(pkg))
-            err("Failed to generate installer for package {}!".format(pkg))
-            failed = failed + 1
-            continue  # cannot generate package
-        except UnresolvedDependency:
-            dep_err = 'Failed to resolve required dependencies for'
-            err("{0} package {1}!".format(dep_err, pkg))
-            unresolved = current.recipe.get_unresolved()
-            borkd_pkgs[pkg] = list()
-            for dep in unresolved:
-                err(" unresolved: \"{}\"".format(dep))
-                borkd_pkgs[pkg].append(dep)
-            err("Failed to generate installer for package {}!".format(pkg))
-            failed = failed + 1
-            continue  # do not generate an incomplete ebuild
-        """
-        except KeyError:
-            err("Failed to parse data for package {}!".format(pkg))
-            unresolved = current.recipe.get_unresolved()
-            err("Failed to generate installer for package {}!".format(pkg))
-            failed = failed + 1
-            continue  # do not generate an incomplete ebuild
-        """
-        make_dir(
-            "recipes-ros-{}/{}".format(distro_name, pkg.replace('_', '-'))
-        )
-        success_msg = 'Successfully generated installer for package'
-        ok('{0}%: {1} \'{2}\'.'.format(percent, success_msg, pkg))
-        succeeded = succeeded + 1
+    )
 
-        # try:
-        recipe_name = '{0}/{1}/{1}_{2}'.format(
-            distro_name,
-            pkg.replace('_', '-'),
-            version
+    if preserve_existing and existing:
+        ok("recipe for package '%s' up to date, skpping..." % pkg)
+        return None, []
+    elif existing:
+        overlay.remove_file(existing[0])
+    try:
+        current = oe_installer(distro, pkg)
+        current.recipe.name = pkg.replace('_', '-')
+    except Exception as e:
+        err('Failed to generate installer for package {}!'.format(pkg))
+        raise e
+    try:
+        info("downloading archive version for package '%s'..." % pkg)
+        current.recipe.downloadArchive()
+        recipe_text = current.recipe_text()
+    except UnresolvedDependency:
+        dep_err = 'Failed to resolve required dependencies for'
+        err("{0} package {1}!".format(dep_err, pkg))
+        unresolved = current.recipe.get_unresolved()
+        for dep in unresolved:
+            err(" unresolved: \"{}\"".format(dep))
+        return None, current.recipe.get_unresolved()
+    except NoPkgXml:
+        err("Could not fetch pkg!")
+        return None, []
+    except KeyError as ke:
+        err("Failed to parse data for package {}!".format(pkg))
+        raise ke
+    make_dir(
+        "{0}/recipes-ros-{1}/{2}".format(
+            overlay.repo.repo_dir,
+            distro.name,
+            pkg.replace('_', '-')
         )
-        recipe_file = open('recipes-ros-{0}.bb'.format(recipe_name), "w")
-
-        recipe_file.write(recipe_text)
-        changes.append('*{0} --> {1}*'.format(pkg, version))
-        """
-        except Exception as e:
-            err("Failed to write recipe to disk!")
-            err(" exception: %s" % (e))
-            installers.append(current)
-            failed_msg = 'Failed to generate installer'
-            err("{0}%: {1} for package {2}!".format(percent, failed_msg, pkg))
-            bad_installers.append(current)
-            failed = failed + 1
-        """
-        results = 'Generated {0} / {1}'.format(succeeded, failed + succeeded)
-        results += ' for distro {0}'.format(distro_name)
-    print("------ {0} ------".format(results))
-    print()
-
-    if len(borkd_pkgs) > 0:
-        warn("Unresolved:")
-        for broken in borkd_pkgs.keys():
-            warn("{}:".format(broken))
-            warn("  {}".format(borkd_pkgs[broken]))
-
-    return installers, borkd_pkgs, changes
+    )
+    success_msg = 'Successfully generated installer for package'
+    ok('{0} \'{1}\'.'.format(success_msg, pkg))
+    recipe_name = '{0}/recipes-ros-{1}/{2}/{2}_{3}.bb'.format(
+        overlay.repo.repo_dir,
+        distro.name,
+        pkg.replace('_', '-'),
+        version
+    )
+    try:
+        with open('{0}'.format(recipe_name), "w") as recipe_file:
+            recipe_file.write(recipe_text)
+    except Exception as e:
+        err("Failed to write recipe to disk!")
+        raise e
+    return current, []
 
 
 def _gen_recipe_for_package(distro, pkg_name, pkg,

superflore/generators/bitbake/ros_meta.py

@@ -12,67 +12,45 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
-# TODO(allenh1): remove inheritance like the RosOverlay class.
-import random
-import string
 import time
 
-from superflore import RepoInstance as repo_instance
-
-
-def get_random_tmp_dir():
-    rand_str = ''.join(random.choice(string.ascii_letters) for x in range(10))
-    return '/tmp/{0}'.format(rand_str)
-
+from superflore.repo_instance import RepoInstance
 
-def get_random_branch_name():
-    rand_str = ''.join(random.choice(string.ascii_letters) for x in range(10))
-    return 'gentoo-bot-{0}'.format(rand_str)
+from superflore.utils import info
+from superflore.utils import rand_ascii_str
 
 
-class ros_meta(repo_instance):
-    def __init__(self):
-        # clone repo into a random tmp directory.
-        repo_instance.__init__(self, 'allenh1',
-                               'meta-ros', get_random_tmp_dir())
-        self.branch_name = get_random_branch_name()
-        self.clone()
-        branch_msg = 'Creating new branch {0}...'.format(self.branch_name)
-        repo_instance.info(branch_msg)
-        self.create_branch(self.branch_name)
+class RosMeta(object):
+    def __init__(self, repo_dir, do_clone, org='allenh1', repo='meta-ros'):
+        self.repo = RepoInstance(org, repo, repo_dir, do_clone)
+        self.branch_name = 'yocto-bot-%s' % rand_ascii_str()
+        info('Creating new branch {0}...'.format(self.branch_name))
+        self.repo.create_branch(self.branch_name)
 
-    def clean_ros_ebuild_dirs(self, distro=None):
-        if distro is not None:
-            self.info(
-                'Cleaning up recipes-ros-{0} directory...'.format(distro)
-            )
-            self.git.rm('-rf', 'recipes-ros-{0}'.format(distro))
+    def clean_ros_recipe_dirs(self, distro=None):
+        if distro:
+            info('Cleaning up recipes-ros-{0} directory...'.format(distro))
+            self.repo.git.rm('-rf', 'recipes-ros-{0}'.format(distro))
         else:
-            self.info('Cleaning up recipes-ros-* directories...')
-            self.git.rm('-rf', 'recipes-ros-*')
+            info('Cleaning up recipes-ros-* directories...')
+            self.repo.git.rm('-rf', 'recipes-ros-*')
 
     def commit_changes(self, distro):
-        self.info('Adding changes...')
+        info('Adding changes...')
         if distro == 'all' or distro == 'update':
-            self.git.add('recipes-ros-*')
+            self.repo.git.add('recipes-ros-*')
         else:
-            self.git.add('recipes-ros-{0}'.format(distro))
+            self.repo.git.add('recipes-ros-{0}'.format(distro))
         commit_msg = {
             'update': 'rosdistro sync, {0}',
             'all': 'regenerate all distros, {0}',
             'lunar': 'regenerate ros-lunar, {0}',
             'indigo': 'regenerate ros-indigo, {0}',
             'kinetic': 'regenerate ros-kinetic, {0}',
         }[distro].format(time.ctime())
-        self.info('Committing to branch {0}...'.format(self.branch_name))
-        self.git.commit(m='{0}'.format(commit_msg))
+        info('Committing to branch {0}...'.format(self.branch_name))
+        self.repo.git.commit(m='{0}'.format(commit_msg))
 
     def pull_request(self, message):
-        self.info('Filing pull-request for allenh1/meta-ros...')
         pr_title = 'rosdistro sync, {0}'.format(time.ctime())
-        self.git.pull_request(m='{0}'.format(message),
-                              title='{0}'.format(pr_title))
-        good_msg = 'Successfully filed a pull request with the {0} repo.'
-        good_msg = good_msg.format('allenh1/meta-ros')
-        self.happy(good_msg)
+        self.repo.pull_request(message, pr_title)