Skip to content

Commit

Permalink
Feature Parity for Open Embedded (#79)
Browse files Browse the repository at this point in the history 
* Switch to use util functions.

* Use the generate_installers function from superflore.generate_installers

* Linting

* Change the ros_meta class to be RosMeta, as well as changed the inheritance
to be a member of the class (see #52).

* Removed the sym-linked structure to behave more like the Gentoo generator (see #51).

* Removed a commented line from the ebuild logic copypasta, changed the print format slightly for the exception output after a failed pr, and called the file_pr function.

* Add temporary file manager code, as suggested by @tfoote.

* Switch to utils print functions for TempfileManager output.

* Use tempfile manager to clean up temp directories at exit.

* Linting.

* Remove hard-coded org and repo from RepoInstance call.

* Reorder import statements.

* Ok, that's pretty cool.

* Apply new TempfileManager usage to OpenEmbedded logic.

* OpenEmbedded generation is now running. ToDo: automatically clean up the tar files.

* Files were not generating in the directory, but are now.

* Fixed argument mapping for ebuild generator.
  • Loading branch information
@allenh1
allenh1 authored Nov 2, 2017
1 parent fef7c79 commit f0cfcff
Show file tree
Hide file tree
Showing 7 changed files with 378 additions and 458 deletions.
57 changes: 57 additions & 0 deletions superflore/TempfileManager.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
# Copyright 2017 Open Source Robotics Foundation

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import errno
import os
import shutil
import subprocess
import tempfile

from superflore.utils import err
from superflore.utils import info


class TempfileManager:
def __init__(self, path):
self.arg_path = path
self.temp_path = None

def __enter__(self):
if self.arg_path:
try:
os.mkdir(self.arg_path)
except OSError as ex:
if ex.errno != errno.EEXIST:
raise
return self.arg_path
else:
self.temp_path = tempfile.mkdtemp()
info("Working in temporary directory %s" % self.temp_path)
return self.temp_path

def __exit__(self, *args):
if self.temp_path:
info("Cleaning up temporary directory %s" % self.temp_path)
try:
shutil.rmtree(self.temp_path)
except OSError as ex:
if ex.errno == errno.EPERM:
err("Failed to rmtree %s" % self.temp_path)
err("Escalating to sudo rm -rf %s" % self.temp_path)
subprocess.check_call(
('sudo rm -rf %s' % self.temp_path).split()
)
else:
raise
self.temp_path = None
213 changes: 70 additions & 143 deletions superflore/generators/bitbake/gen_packages.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,173 +13,100 @@
# limitations under the License.

import glob
import os
import sys

from rosdistro.dependency_walker import DependencyWalker
from rosdistro.manifest_provider import get_release_tag
from rosdistro.rosdistro import RosPackage

from rosinstall_generator.distro import _generate_rosinstall
from rosinstall_generator.distro import get_distro
from rosinstall_generator.distro import get_package_names

from superflore.exceptions import NoPkgXml
from superflore.exceptions import UnresolvedDependency

from termcolor import colored
from superflore.generators.bitbake.yocto_recipe import yoctoRecipe

import xmltodict
from superflore.utils import err
from superflore.utils import get_pkg_version
from superflore.utils import info
from superflore.utils import make_dir
from superflore.utils import ok
from superflore.utils import warn

from .yocto_recipe import yoctoRecipe
import xmltodict

org = "Open Source Robotics Foundation"
org_license = "BSD"


def warn(string):
print(colored('>>>> {0}'.format(string), 'yellow'))


def ok(string):
print(colored('>>>> {0}'.format(string), 'green'))


def err(string):
print(colored('!!!! {0}'.format(string), 'red'))


def info(string):
print(colored('>>>> {0}'.format(string), 'cyan'))


def make_dir(dirname):
try:
os.makedirs(dirname)
except Exception:
pass


def get_pkg_version(distro, pkg_name):
pkg = distro.release_packages[pkg_name]
repo = distro.repositories[pkg.repository_name].release_repository
maj_min_patch, deb_inc = repo.version.split('-')
if deb_inc != '0':
return '{0}-r{1}'.format(maj_min_patch, deb_inc)
return maj_min_patch


def generate_installers(distro_name, overlay, preserve_existing=True):
make_dir("recipes-ros-{}".format(distro_name))
distro = get_distro(distro_name)
pkg_names = get_package_names(distro)
total = float(len(pkg_names[0]))
borkd_pkgs = dict()
changes = []
installers = []
succeeded = 0
failed = 0
def regenerate_installer(overlay, pkg, distro, preserve_existing=False):
make_dir("{0}/recipes-ros-{1}".format(overlay.repo.repo_dir, distro.name))
version = get_pkg_version(distro, pkg)
pkg_names = get_package_names(distro)[0]

for i, pkg in enumerate(sorted(pkg_names[0])):
version = get_pkg_version(distro, pkg)
"""
recipe_exists = os.path.exists(
'ros-{}/{}/{}-{}.ebuild'.format(distro_name, pkg, pkg, version))
patch_path = 'ros-{}/{}/files'.format(distro_name, pkg)
has_patches = os.path.exists(patch_path)
"""
percent = '%.1f' % (100 * (float(i) / total))
if pkg not in pkg_names:
raise RuntimeError("Unknown package '%s'" % pkg)

"""
if preserve_existing and ebuild_exists:
skip_msg = 'Ebuild for package '
skip_msg += '{0} up to date, skipping...'.format(pkg)
status = '{0}%: {1}'.format(percent, skip_msg)
ok(status)
succeeded = succeeded + 1
continue
"""
# otherwise, remove a (potentially) existing recipe.
existing = glob.glob(
'recipes-ros-{0}/{1}/*.bb'.format(distro_name, pkg)
# check for an existing recipe
existing = glob.glob(
'{0}/recipes-ros-{1}/{2}/*.bb'.format(
overlay.repo.repo_dir,
distro.name,
pkg
)
if len(existing) > 0:
overlay.remove_file(existing[0])
try:
current = oe_installer(distro, pkg)
current.recipe.name = pkg.replace('_', '-')
except Exception as e:
err('Failed to generate installer for package {}!'.format(pkg))
err(' exception: {0}'.format(e))
failed = failed + 1
continue
try:
info(' downloading archive version for package \'%s\'' % pkg)
current.recipe.downloadArchive()
recipe_text = current.recipe_text()
except NoPkgXml:
err(' No package.xml file for pkg \'{0}\'!'.format(pkg))
err("Failed to generate installer for package {}!".format(pkg))
failed = failed + 1
continue # cannot generate package
except UnresolvedDependency:
dep_err = 'Failed to resolve required dependencies for'
err("{0} package {1}!".format(dep_err, pkg))
unresolved = current.recipe.get_unresolved()
borkd_pkgs[pkg] = list()
for dep in unresolved:
err(" unresolved: \"{}\"".format(dep))
borkd_pkgs[pkg].append(dep)
err("Failed to generate installer for package {}!".format(pkg))
failed = failed + 1
continue # do not generate an incomplete ebuild
"""
except KeyError:
err("Failed to parse data for package {}!".format(pkg))
unresolved = current.recipe.get_unresolved()
err("Failed to generate installer for package {}!".format(pkg))
failed = failed + 1
continue # do not generate an incomplete ebuild
"""
make_dir(
"recipes-ros-{}/{}".format(distro_name, pkg.replace('_', '-'))
)
success_msg = 'Successfully generated installer for package'
ok('{0}%: {1} \'{2}\'.'.format(percent, success_msg, pkg))
succeeded = succeeded + 1
)

# try:
recipe_name = '{0}/{1}/{1}_{2}'.format(
distro_name,
pkg.replace('_', '-'),
version
if preserve_existing and existing:
ok("recipe for package '%s' up to date, skpping..." % pkg)
return None, []
elif existing:
overlay.remove_file(existing[0])
try:
current = oe_installer(distro, pkg)
current.recipe.name = pkg.replace('_', '-')
except Exception as e:
err('Failed to generate installer for package {}!'.format(pkg))
raise e
try:
info("downloading archive version for package '%s'..." % pkg)
current.recipe.downloadArchive()
recipe_text = current.recipe_text()
except UnresolvedDependency:
dep_err = 'Failed to resolve required dependencies for'
err("{0} package {1}!".format(dep_err, pkg))
unresolved = current.recipe.get_unresolved()
for dep in unresolved:
err(" unresolved: \"{}\"".format(dep))
return None, current.recipe.get_unresolved()
except NoPkgXml:
err("Could not fetch pkg!")
return None, []
except KeyError as ke:
err("Failed to parse data for package {}!".format(pkg))
raise ke
make_dir(
"{0}/recipes-ros-{1}/{2}".format(
overlay.repo.repo_dir,
distro.name,
pkg.replace('_', '-')
)
recipe_file = open('recipes-ros-{0}.bb'.format(recipe_name), "w")

recipe_file.write(recipe_text)
changes.append('*{0} --> {1}*'.format(pkg, version))
"""
except Exception as e:
err("Failed to write recipe to disk!")
err(" exception: %s" % (e))
installers.append(current)
failed_msg = 'Failed to generate installer'
err("{0}%: {1} for package {2}!".format(percent, failed_msg, pkg))
bad_installers.append(current)
failed = failed + 1
"""
results = 'Generated {0} / {1}'.format(succeeded, failed + succeeded)
results += ' for distro {0}'.format(distro_name)
print("------ {0} ------".format(results))
print()

if len(borkd_pkgs) > 0:
warn("Unresolved:")
for broken in borkd_pkgs.keys():
warn("{}:".format(broken))
warn(" {}".format(borkd_pkgs[broken]))

return installers, borkd_pkgs, changes
)
success_msg = 'Successfully generated installer for package'
ok('{0} \'{1}\'.'.format(success_msg, pkg))
recipe_name = '{0}/recipes-ros-{1}/{2}/{2}_{3}.bb'.format(
overlay.repo.repo_dir,
distro.name,
pkg.replace('_', '-'),
version
)
try:
with open('{0}'.format(recipe_name), "w") as recipe_file:
recipe_file.write(recipe_text)
except Exception as e:
err("Failed to write recipe to disk!")
raise e
return current, []


def _gen_recipe_for_package(distro, pkg_name, pkg,
Expand Down
64 changes: 21 additions & 43 deletions superflore/generators/bitbake/ros_meta.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,67 +12,45 @@
# See the License for the specific language governing permissions and
# limitations under the License.


# TODO(allenh1): remove inheritance like the RosOverlay class.
import random
import string
import time

from superflore import RepoInstance as repo_instance


def get_random_tmp_dir():
rand_str = ''.join(random.choice(string.ascii_letters) for x in range(10))
return '/tmp/{0}'.format(rand_str)

from superflore.repo_instance import RepoInstance

def get_random_branch_name():
rand_str = ''.join(random.choice(string.ascii_letters) for x in range(10))
return 'gentoo-bot-{0}'.format(rand_str)
from superflore.utils import info
from superflore.utils import rand_ascii_str


class ros_meta(repo_instance):
def __init__(self):
# clone repo into a random tmp directory.
repo_instance.__init__(self, 'allenh1',
'meta-ros', get_random_tmp_dir())
self.branch_name = get_random_branch_name()
self.clone()
branch_msg = 'Creating new branch {0}...'.format(self.branch_name)
repo_instance.info(branch_msg)
self.create_branch(self.branch_name)
class RosMeta(object):
def __init__(self, repo_dir, do_clone, org='allenh1', repo='meta-ros'):
self.repo = RepoInstance(org, repo, repo_dir, do_clone)
self.branch_name = 'yocto-bot-%s' % rand_ascii_str()
info('Creating new branch {0}...'.format(self.branch_name))
self.repo.create_branch(self.branch_name)

def clean_ros_ebuild_dirs(self, distro=None):
if distro is not None:
self.info(
'Cleaning up recipes-ros-{0} directory...'.format(distro)
)
self.git.rm('-rf', 'recipes-ros-{0}'.format(distro))
def clean_ros_recipe_dirs(self, distro=None):
if distro:
info('Cleaning up recipes-ros-{0} directory...'.format(distro))
self.repo.git.rm('-rf', 'recipes-ros-{0}'.format(distro))
else:
self.info('Cleaning up recipes-ros-* directories...')
self.git.rm('-rf', 'recipes-ros-*')
info('Cleaning up recipes-ros-* directories...')
self.repo.git.rm('-rf', 'recipes-ros-*')

def commit_changes(self, distro):
self.info('Adding changes...')
info('Adding changes...')
if distro == 'all' or distro == 'update':
self.git.add('recipes-ros-*')
self.repo.git.add('recipes-ros-*')
else:
self.git.add('recipes-ros-{0}'.format(distro))
self.repo.git.add('recipes-ros-{0}'.format(distro))
commit_msg = {
'update': 'rosdistro sync, {0}',
'all': 'regenerate all distros, {0}',
'lunar': 'regenerate ros-lunar, {0}',
'indigo': 'regenerate ros-indigo, {0}',
'kinetic': 'regenerate ros-kinetic, {0}',
}[distro].format(time.ctime())
self.info('Committing to branch {0}...'.format(self.branch_name))
self.git.commit(m='{0}'.format(commit_msg))
info('Committing to branch {0}...'.format(self.branch_name))
self.repo.git.commit(m='{0}'.format(commit_msg))

def pull_request(self, message):
self.info('Filing pull-request for allenh1/meta-ros...')
pr_title = 'rosdistro sync, {0}'.format(time.ctime())
self.git.pull_request(m='{0}'.format(message),
title='{0}'.format(pr_title))
good_msg = 'Successfully filed a pull request with the {0} repo.'
good_msg = good_msg.format('allenh1/meta-ros')
self.happy(good_msg)
self.repo.pull_request(message, pr_title)
Loading

0 comments on commit f0cfcff

Please sign in to comment.