diff --git a/superflore/TempfileManager.py b/superflore/TempfileManager.py new file mode 100644 index 00000000..248da1a4 --- /dev/null +++ b/superflore/TempfileManager.py @@ -0,0 +1,57 @@ +# Copyright 2017 Open Source Robotics Foundation + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import errno +import os +import shutil +import subprocess +import tempfile + +from superflore.utils import err +from superflore.utils import info + + +class TempfileManager: + def __init__(self, path): + self.arg_path = path + self.temp_path = None + + def __enter__(self): + if self.arg_path: + try: + os.mkdir(self.arg_path) + except OSError as ex: + if ex.errno != errno.EEXIST: + raise + return self.arg_path + else: + self.temp_path = tempfile.mkdtemp() + info("Working in temporary directory %s" % self.temp_path) + return self.temp_path + + def __exit__(self, *args): + if self.temp_path: + info("Cleaning up temporary directory %s" % self.temp_path) + try: + shutil.rmtree(self.temp_path) + except OSError as ex: + if ex.errno == errno.EPERM: + err("Failed to rmtree %s" % self.temp_path) + err("Escalating to sudo rm -rf %s" % self.temp_path) + subprocess.check_call( + ('sudo rm -rf %s' % self.temp_path).split() + ) + else: + raise + self.temp_path = None diff --git a/superflore/generators/bitbake/gen_packages.py b/superflore/generators/bitbake/gen_packages.py index 11def5be..c21dfef1 100644 --- a/superflore/generators/bitbake/gen_packages.py +++ b/superflore/generators/bitbake/gen_packages.py @@ -13,7 +13,6 @@ # limitations under the License. import glob -import os import sys from rosdistro.dependency_walker import DependencyWalker @@ -21,165 +20,93 @@ from rosdistro.rosdistro import RosPackage from rosinstall_generator.distro import _generate_rosinstall -from rosinstall_generator.distro import get_distro from rosinstall_generator.distro import get_package_names from superflore.exceptions import NoPkgXml from superflore.exceptions import UnresolvedDependency -from termcolor import colored +from superflore.generators.bitbake.yocto_recipe import yoctoRecipe -import xmltodict +from superflore.utils import err +from superflore.utils import get_pkg_version +from superflore.utils import info +from superflore.utils import make_dir +from superflore.utils import ok +from superflore.utils import warn -from .yocto_recipe import yoctoRecipe +import xmltodict org = "Open Source Robotics Foundation" org_license = "BSD" -def warn(string): - print(colored('>>>> {0}'.format(string), 'yellow')) - - -def ok(string): - print(colored('>>>> {0}'.format(string), 'green')) - - -def err(string): - print(colored('!!!! {0}'.format(string), 'red')) - - -def info(string): - print(colored('>>>> {0}'.format(string), 'cyan')) - - -def make_dir(dirname): - try: - os.makedirs(dirname) - except Exception: - pass - - -def get_pkg_version(distro, pkg_name): - pkg = distro.release_packages[pkg_name] - repo = distro.repositories[pkg.repository_name].release_repository - maj_min_patch, deb_inc = repo.version.split('-') - if deb_inc != '0': - return '{0}-r{1}'.format(maj_min_patch, deb_inc) - return maj_min_patch - - -def generate_installers(distro_name, overlay, preserve_existing=True): - make_dir("recipes-ros-{}".format(distro_name)) - distro = get_distro(distro_name) - pkg_names = get_package_names(distro) - total = float(len(pkg_names[0])) - borkd_pkgs = dict() - changes = [] - installers = [] - succeeded = 0 - failed = 0 +def regenerate_installer(overlay, pkg, distro, preserve_existing=False): + make_dir("{0}/recipes-ros-{1}".format(overlay.repo.repo_dir, distro.name)) + version = get_pkg_version(distro, pkg) + pkg_names = get_package_names(distro)[0] - for i, pkg in enumerate(sorted(pkg_names[0])): - version = get_pkg_version(distro, pkg) - """ - recipe_exists = os.path.exists( - 'ros-{}/{}/{}-{}.ebuild'.format(distro_name, pkg, pkg, version)) - patch_path = 'ros-{}/{}/files'.format(distro_name, pkg) - has_patches = os.path.exists(patch_path) - """ - percent = '%.1f' % (100 * (float(i) / total)) + if pkg not in pkg_names: + raise RuntimeError("Unknown package '%s'" % pkg) - """ - if preserve_existing and ebuild_exists: - skip_msg = 'Ebuild for package ' - skip_msg += '{0} up to date, skipping...'.format(pkg) - status = '{0}%: {1}'.format(percent, skip_msg) - ok(status) - succeeded = succeeded + 1 - continue - """ - # otherwise, remove a (potentially) existing recipe. - existing = glob.glob( - 'recipes-ros-{0}/{1}/*.bb'.format(distro_name, pkg) + # check for an existing recipe + existing = glob.glob( + '{0}/recipes-ros-{1}/{2}/*.bb'.format( + overlay.repo.repo_dir, + distro.name, + pkg ) - if len(existing) > 0: - overlay.remove_file(existing[0]) - try: - current = oe_installer(distro, pkg) - current.recipe.name = pkg.replace('_', '-') - except Exception as e: - err('Failed to generate installer for package {}!'.format(pkg)) - err(' exception: {0}'.format(e)) - failed = failed + 1 - continue - try: - info(' downloading archive version for package \'%s\'' % pkg) - current.recipe.downloadArchive() - recipe_text = current.recipe_text() - except NoPkgXml: - err(' No package.xml file for pkg \'{0}\'!'.format(pkg)) - err("Failed to generate installer for package {}!".format(pkg)) - failed = failed + 1 - continue # cannot generate package - except UnresolvedDependency: - dep_err = 'Failed to resolve required dependencies for' - err("{0} package {1}!".format(dep_err, pkg)) - unresolved = current.recipe.get_unresolved() - borkd_pkgs[pkg] = list() - for dep in unresolved: - err(" unresolved: \"{}\"".format(dep)) - borkd_pkgs[pkg].append(dep) - err("Failed to generate installer for package {}!".format(pkg)) - failed = failed + 1 - continue # do not generate an incomplete ebuild - """ - except KeyError: - err("Failed to parse data for package {}!".format(pkg)) - unresolved = current.recipe.get_unresolved() - err("Failed to generate installer for package {}!".format(pkg)) - failed = failed + 1 - continue # do not generate an incomplete ebuild - """ - make_dir( - "recipes-ros-{}/{}".format(distro_name, pkg.replace('_', '-')) - ) - success_msg = 'Successfully generated installer for package' - ok('{0}%: {1} \'{2}\'.'.format(percent, success_msg, pkg)) - succeeded = succeeded + 1 + ) - # try: - recipe_name = '{0}/{1}/{1}_{2}'.format( - distro_name, - pkg.replace('_', '-'), - version + if preserve_existing and existing: + ok("recipe for package '%s' up to date, skpping..." % pkg) + return None, [] + elif existing: + overlay.remove_file(existing[0]) + try: + current = oe_installer(distro, pkg) + current.recipe.name = pkg.replace('_', '-') + except Exception as e: + err('Failed to generate installer for package {}!'.format(pkg)) + raise e + try: + info("downloading archive version for package '%s'..." % pkg) + current.recipe.downloadArchive() + recipe_text = current.recipe_text() + except UnresolvedDependency: + dep_err = 'Failed to resolve required dependencies for' + err("{0} package {1}!".format(dep_err, pkg)) + unresolved = current.recipe.get_unresolved() + for dep in unresolved: + err(" unresolved: \"{}\"".format(dep)) + return None, current.recipe.get_unresolved() + except NoPkgXml: + err("Could not fetch pkg!") + return None, [] + except KeyError as ke: + err("Failed to parse data for package {}!".format(pkg)) + raise ke + make_dir( + "{0}/recipes-ros-{1}/{2}".format( + overlay.repo.repo_dir, + distro.name, + pkg.replace('_', '-') ) - recipe_file = open('recipes-ros-{0}.bb'.format(recipe_name), "w") - - recipe_file.write(recipe_text) - changes.append('*{0} --> {1}*'.format(pkg, version)) - """ - except Exception as e: - err("Failed to write recipe to disk!") - err(" exception: %s" % (e)) - installers.append(current) - failed_msg = 'Failed to generate installer' - err("{0}%: {1} for package {2}!".format(percent, failed_msg, pkg)) - bad_installers.append(current) - failed = failed + 1 - """ - results = 'Generated {0} / {1}'.format(succeeded, failed + succeeded) - results += ' for distro {0}'.format(distro_name) - print("------ {0} ------".format(results)) - print() - - if len(borkd_pkgs) > 0: - warn("Unresolved:") - for broken in borkd_pkgs.keys(): - warn("{}:".format(broken)) - warn(" {}".format(borkd_pkgs[broken])) - - return installers, borkd_pkgs, changes + ) + success_msg = 'Successfully generated installer for package' + ok('{0} \'{1}\'.'.format(success_msg, pkg)) + recipe_name = '{0}/recipes-ros-{1}/{2}/{2}_{3}.bb'.format( + overlay.repo.repo_dir, + distro.name, + pkg.replace('_', '-'), + version + ) + try: + with open('{0}'.format(recipe_name), "w") as recipe_file: + recipe_file.write(recipe_text) + except Exception as e: + err("Failed to write recipe to disk!") + raise e + return current, [] def _gen_recipe_for_package(distro, pkg_name, pkg, diff --git a/superflore/generators/bitbake/ros_meta.py b/superflore/generators/bitbake/ros_meta.py index 60c37ebe..043703bc 100644 --- a/superflore/generators/bitbake/ros_meta.py +++ b/superflore/generators/bitbake/ros_meta.py @@ -12,52 +12,35 @@ # See the License for the specific language governing permissions and # limitations under the License. - -# TODO(allenh1): remove inheritance like the RosOverlay class. -import random -import string import time -from superflore import RepoInstance as repo_instance - - -def get_random_tmp_dir(): - rand_str = ''.join(random.choice(string.ascii_letters) for x in range(10)) - return '/tmp/{0}'.format(rand_str) - +from superflore.repo_instance import RepoInstance -def get_random_branch_name(): - rand_str = ''.join(random.choice(string.ascii_letters) for x in range(10)) - return 'gentoo-bot-{0}'.format(rand_str) +from superflore.utils import info +from superflore.utils import rand_ascii_str -class ros_meta(repo_instance): - def __init__(self): - # clone repo into a random tmp directory. - repo_instance.__init__(self, 'allenh1', - 'meta-ros', get_random_tmp_dir()) - self.branch_name = get_random_branch_name() - self.clone() - branch_msg = 'Creating new branch {0}...'.format(self.branch_name) - repo_instance.info(branch_msg) - self.create_branch(self.branch_name) +class RosMeta(object): + def __init__(self, repo_dir, do_clone, org='allenh1', repo='meta-ros'): + self.repo = RepoInstance(org, repo, repo_dir, do_clone) + self.branch_name = 'yocto-bot-%s' % rand_ascii_str() + info('Creating new branch {0}...'.format(self.branch_name)) + self.repo.create_branch(self.branch_name) - def clean_ros_ebuild_dirs(self, distro=None): - if distro is not None: - self.info( - 'Cleaning up recipes-ros-{0} directory...'.format(distro) - ) - self.git.rm('-rf', 'recipes-ros-{0}'.format(distro)) + def clean_ros_recipe_dirs(self, distro=None): + if distro: + info('Cleaning up recipes-ros-{0} directory...'.format(distro)) + self.repo.git.rm('-rf', 'recipes-ros-{0}'.format(distro)) else: - self.info('Cleaning up recipes-ros-* directories...') - self.git.rm('-rf', 'recipes-ros-*') + info('Cleaning up recipes-ros-* directories...') + self.repo.git.rm('-rf', 'recipes-ros-*') def commit_changes(self, distro): - self.info('Adding changes...') + info('Adding changes...') if distro == 'all' or distro == 'update': - self.git.add('recipes-ros-*') + self.repo.git.add('recipes-ros-*') else: - self.git.add('recipes-ros-{0}'.format(distro)) + self.repo.git.add('recipes-ros-{0}'.format(distro)) commit_msg = { 'update': 'rosdistro sync, {0}', 'all': 'regenerate all distros, {0}', @@ -65,14 +48,9 @@ def commit_changes(self, distro): 'indigo': 'regenerate ros-indigo, {0}', 'kinetic': 'regenerate ros-kinetic, {0}', }[distro].format(time.ctime()) - self.info('Committing to branch {0}...'.format(self.branch_name)) - self.git.commit(m='{0}'.format(commit_msg)) + info('Committing to branch {0}...'.format(self.branch_name)) + self.repo.git.commit(m='{0}'.format(commit_msg)) def pull_request(self, message): - self.info('Filing pull-request for allenh1/meta-ros...') pr_title = 'rosdistro sync, {0}'.format(time.ctime()) - self.git.pull_request(m='{0}'.format(message), - title='{0}'.format(pr_title)) - good_msg = 'Successfully filed a pull request with the {0} repo.' - good_msg = good_msg.format('allenh1/meta-ros') - self.happy(good_msg) + self.repo.pull_request(message, pr_title) diff --git a/superflore/generators/bitbake/run.py b/superflore/generators/bitbake/run.py index 209354bc..f59d2e8d 100644 --- a/superflore/generators/bitbake/run.py +++ b/superflore/generators/bitbake/run.py @@ -13,14 +13,19 @@ # limitations under the License. import argparse -import errno -import os -import shutil import sys -from superflore import RepoInstance -from superflore.generators.bitbake.gen_packages import generate_installers -from superflore.generators.bitbake.ros_meta import ros_meta +from superflore.generate_installers import generate_installers + +from superflore.generators.bitbake.gen_packages import regenerate_installer +from superflore.generators.bitbake.ros_meta import RosMeta + +from superflore.TempfileManager import TempfileManager + +from superflore.utils import err +from superflore.utils import info +from superflore.utils import ok +from superflore.utils import warn # Modify if a new distro is added active_distros = ['indigo', 'kinetic', 'lunar'] @@ -30,39 +35,18 @@ overlay = None -def link_existing_files(mode): - global overlay - sym_link_msg = 'Symbolicly linking files from {0}/recipes-ros-{1}...' - dir_fmt = '{0}/recipes-ros-{1}' - if mode == 'all' or mode == 'update': - for x in active_distros: - ros_meta.info(sym_link_msg.format(overlay.repo_dir, x)) - os.symlink( - dir_fmt.format(overlay.repo_dir, x), './recipes-ros-' + x - ) - else: - # only link the relevant directory. - ros_meta.info(sym_link_msg.format(overlay.repo_dir, mode)) - os.symlink( - dir_fmt.format(overlay.repo_dir, mode), './recipes-ros-' + mode - ) - - -def clean_up(distro): - global overlay - clean_msg = 'Cleaning up tmp directory {0}...'.format(overlay.repo_dir) - ros_meta.info(clean_msg) - shutil.rmtree(overlay.repo_dir) - ros_meta.info('Cleaning up symbolic links...') - if mode != 'all' and mode != 'update': - os.remove('recipes-ros-{0}'.format(distro)) - else: - for x in active_distros: - os.remove('recipes-ros-{0}'.format(x)) +def file_pr(overlay, delta, missing_deps): + try: + overlay.pull_request('{0}\n{1}'.format(delta, missing_deps)) + except Exception as e: + err('Failed to file PR with allenh1/meta-ros repo!') + err(' Exception: {0}'.format(e)) + sys.exit(1) def main(): global overlay + global preserve_existing parser = argparse.ArgumentParser('Deploy ROS packages into Yocto Linux') parser.add_argument( @@ -75,104 +59,92 @@ def main(): help='regenerate all packages in all distros', action="store_true" ) - args = parser.parse_args(sys.argv[1:]) - # clone current repo - overlay = ros_meta() - selected_targets = active_distros - - if args.all: - ros_meta.warn('"All" mode detected... this may take a while!') - elif args.ros_distro: - selected_targets = [args.ros_distro] - preserve_existing = False - - # clone current repo - selected_targets = active_distros - for x in active_distros: - try: - os.remove('recipes-ros-{0}'.format(x)) - warn_msg =\ - 'removing existing symlink "./recipes-ros-{0}"'.format(x) - RepoInstance.warn(warn_msg) - except OSError as e: - if e.errno == errno.ENOENT: - pass - else: - raise e - link_existing_files(args.ros_distro) - - # generate installers - total_installers = dict() - total_broken = set() - total_changes = dict() - - for distro in selected_targets: - distro_installers, distro_broken, distro_changes =\ - generate_installers(distro, overlay, preserve_existing) - for key in distro_broken.keys(): - for pkg in distro_broken[key]: - total_broken.add(pkg) - - total_changes[distro] = distro_changes - total_installers[distro] = distro_installers - - num_changes = 0 - for distro_name in total_changes: - num_changes += len(total_changes[distro_name]) - - if num_changes == 0: - ros_meta.info('ROS distro is up to date.') - ros_meta.info('Exiting...') - clean_up() - sys.exit(0) - - # remove duplicates - inst_list = total_broken - - delta = "Changes:\n" - delta += "========\n" - - if 'indigo' in total_changes and len(total_changes['indigo']) > 0: - delta += "Indigo Changes:\n" - delta += "---------------\n" - - for d in sorted(total_changes['indigo']): - delta += '* {0}\n'.format(d) - delta += "\n" - - if 'kinetic' in total_changes and len(total_changes['kinetic']) > 0: - delta += "Kinetic Changes:\n" - delta += "----------------\n" - - for d in sorted(total_changes['kinetic']): - delta += '* {0}\n'.format(d) - delta += "\n" - - if 'lunar' in total_changes and len(total_changes['lunar']) > 0: - delta += "Lunar Changes:\n" - delta += "--------------\n" - - for d in sorted(total_changes['lunar']): - delta += '* {0}\n'.format(d) - delta += "\n" - - missing_deps = '' - - if len(inst_list) > 0: - missing_deps = "Missing Dependencies:\n" - missing_deps += "=====================\n" - for pkg in sorted(inst_list): - missing_deps += " * [ ] {0}\n".format(pkg) - - # Commit changes and file pull request - # overlay.regenerate_manifests(mode) - overlay.commit_changes(args.ros_distro) - try: - overlay.pull_request('{0}\n{1}'.format(delta, missing_deps)) - except Exception as e: - overlay.error('Failed to file PR with allenh1/meta-ros repo!') - overlay.error('Exception: {0}'.format(e)) - sys.exit(1) + parser.add_argument( + '--output-repository-path', + help='location of the Git repo', + type=str + ) - clean_up() - ros_meta.happy('Successfully synchronized repositories!') + args = parser.parse_args(sys.argv[1:]) + with TempfileManager(args.output_repository_path) as _repo: + overlay = RosMeta(_repo, not args.output_repository_path) + selected_targets = active_distros + if args.all: + warn('"All" mode detected... this may take a while!') + preserve_existing = False + elif args.ros_distro: + selected_targets = [args.ros_distro] + preserve_existing = False + # clone current repo + selected_targets = active_distros + # generate installers + total_installers = dict() + total_broken = set() + total_changes = dict() + + for distro in selected_targets: + distro_installers, distro_broken, distro_changes =\ + generate_installers( + distro_name=distro, + overlay=overlay, + gen_pkg_func=regenerate_installer, + preserve_existing=preserve_existing + ) + for key in distro_broken.keys(): + for pkg in distro_broken[key]: + total_broken.add(pkg) + + total_changes[distro] = distro_changes + total_installers[distro] = distro_installers + + num_changes = 0 + for distro_name in total_changes: + num_changes += len(total_changes[distro_name]) + + if num_changes == 0: + info('ROS distro is up to date.') + info('Exiting...') + sys.exit(0) + + # remove duplicates + inst_list = total_broken + + delta = "Changes:\n" + delta += "========\n" + + if 'indigo' in total_changes and len(total_changes['indigo']) > 0: + delta += "Indigo Changes:\n" + delta += "---------------\n" + + for d in sorted(total_changes['indigo']): + delta += '* {0}\n'.format(d) + delta += "\n" + + if 'kinetic' in total_changes and len(total_changes['kinetic']) > 0: + delta += "Kinetic Changes:\n" + delta += "----------------\n" + + for d in sorted(total_changes['kinetic']): + delta += '* {0}\n'.format(d) + delta += "\n" + + if 'lunar' in total_changes and len(total_changes['lunar']) > 0: + delta += "Lunar Changes:\n" + delta += "--------------\n" + + for d in sorted(total_changes['lunar']): + delta += '* {0}\n'.format(d) + delta += "\n" + + missing_deps = '' + + if len(inst_list) > 0: + missing_deps = "Missing Dependencies:\n" + missing_deps += "=====================\n" + for pkg in sorted(inst_list): + missing_deps += " * [ ] {0}\n".format(pkg) + + # Commit changes and file pull request + overlay.commit_changes(args.ros_distro) + file_pr(overlay, delta, missing_deps) + ok('Successfully synchronized repositories!') diff --git a/superflore/generators/ebuild/overlay_instance.py b/superflore/generators/ebuild/overlay_instance.py index 367cc6f7..f1e2dd3c 100644 --- a/superflore/generators/ebuild/overlay_instance.py +++ b/superflore/generators/ebuild/overlay_instance.py @@ -13,40 +13,20 @@ # limitations under the License. import os -import random -import string import time from superflore.docker import Docker + from superflore.repo_instance import RepoInstance from superflore.utils import info - - -def get_random_tmp_dir(): - rand_str = ''.join(random.choice(string.ascii_letters) for x in range(10)) - return '/tmp/{0}'.format(rand_str) - - -def get_random_branch_name(): - rand_str = ''.join(random.choice(string.ascii_letters) for x in range(10)) - return 'gentoo-bot-{0}'.format(rand_str) +from superflore.utils import rand_ascii_str class RosOverlay(object): - def __init__(self, repo_dir=None): - # clone repo into a random tmp directory. - do_clone = True - if repo_dir: - do_clone = not os.path.exists(os.path.realpath(repo_dir)) - self.repo = RepoInstance( - 'ros', 'ros-overlay', - repo_dir or get_random_tmp_dir(), - do_clone - ) - self.branch_name = get_random_branch_name() - if do_clone: - self.repo.clone() + def __init__(self, repo_dir, do_clone, org='ros', repo='ros-overlay'): + self.repo = RepoInstance(org, repo, repo_dir, do_clone) + self.branch_name = 'gentoo-bot-%s' % rand_ascii_str() info('Creating new branch {0}...'.format(self.branch_name)) self.repo.create_branch(self.branch_name) diff --git a/superflore/generators/ebuild/run.py b/superflore/generators/ebuild/run.py index ea8954bb..20c4f4b7 100755 --- a/superflore/generators/ebuild/run.py +++ b/superflore/generators/ebuild/run.py @@ -15,7 +15,6 @@ import argparse import os -import shutil import sys import time @@ -28,6 +27,8 @@ from superflore.repo_instance import RepoInstance +from superflore.TempfileManager import TempfileManager + from superflore.utils import err from superflore.utils import info from superflore.utils import ok @@ -40,13 +41,7 @@ overlay = None -def clean_up(distro, preserve_repo=False): - global overlay - if not preserve_repo: - clean_msg = \ - 'Cleaning up tmp directory {0}...'.format(overlay.repo.repo_dir) - info(clean_msg) - shutil.rmtree(overlay.repo.repo_dir) +def clean_up(): if os.path.exists('.pr-message.tmp'): os.remove('.pr-message.tmp') if os.path.exists('.pr-title.tmp'): @@ -127,39 +122,120 @@ def main(): ) raise try: - prev_overlay = RepoInstance(args.output_repository_path) + prev_overlay = RepoInstance(args.output_repository_path, False) info('PR message:\n"%s"\n' % msg) info('PR title:\n"%s"\n' % title) prev_overlay.pull_request(msg, title) - clean_up('all') + clean_up() sys.exit(0) except Exception as e: err('Failed to file PR!') err('reason: {0}'.format(e)) sys.exit(1) - # clone current repo - overlay = RosOverlay(args.output_repository_path) - selected_targets = active_distros - # generate installers - total_installers = dict() - total_broken = set() - total_changes = dict() - - if args.only: - for pkg in args.only: - info("Regenerating package '%s'..." % pkg) - regenerate_pkg( - overlay, - pkg=pkg, - distro=get_distro(args.ros_distro) - ) + with TempfileManager(args.output_repository_path) as _repo: + # clone if args.output_repository_path is None + overlay = RosOverlay(_repo, not args.output_repository_path) + selected_targets = active_distros + # generate installers + total_installers = dict() + total_broken = set() + total_changes = dict() + + if args.only: + for pkg in args.only: + info("Regenerating package '%s'..." % pkg) + regenerate_pkg( + overlay, + pkg=pkg, + distro=get_distro(args.ros_distro) + ) + # Commit changes and file pull request + regen_dict = dict() + regen_dict[args.ros_distro] = args.only + overlay.regenerate_manifests(regen_dict) + overlay.commit_changes(args.ros_distro) + delta = "Regenerated: '%s'\n" % args.only + missing_deps = '' + if args.dry_run: + info('Running in dry mode, not filing PR') + title_file = open('.pr-title.tmp', 'w') + title_file.write('rosdistro sync, {0}\n'.format(time.ctime())) + pr_message_file = open('.pr-message.tmp', 'w') + pr_message_file.write('%s\n%s\n' % (delta, missing_deps)) + sys.exit(0) + file_pr(overlay, delta, missing_deps) + + clean_up() + ok('Successfully synchronized repositories!') + sys.exit(0) + + for distro in selected_targets: + distro_installers, distro_broken, distro_changes =\ + generate_installers( + distro_name=distro, + overlay=overlay, + gen_pkg_func=regenerate_pkg, + preserve_existing=preserve_existing + ) + for key in distro_broken.keys(): + for pkg in distro_broken[key]: + total_broken.add(pkg) + + total_changes[distro] = distro_changes + total_installers[distro] = distro_installers + + num_changes = 0 + for distro_name in total_changes: + num_changes += len(total_changes[distro_name]) + + if num_changes == 0: + info('ROS distro is up to date.') + info('Exiting...') + clean_up() + sys.exit(0) + + # remove duplicates + inst_list = total_broken + + delta = "Changes:\n" + delta += "========\n" + + if 'indigo' in total_changes and len(total_changes['indigo']) > 0: + delta += "Indigo Changes:\n" + delta += "---------------\n" + + for d in sorted(total_changes['indigo']): + delta += '* {0}\n'.format(d) + delta += "\n" + + if 'kinetic' in total_changes and len(total_changes['kinetic']) > 0: + delta += "Kinetic Changes:\n" + delta += "----------------\n" + + for d in sorted(total_changes['kinetic']): + delta += '* {0}\n'.format(d) + delta += "\n" + + if 'lunar' in total_changes and len(total_changes['lunar']) > 0: + delta += "Lunar Changes:\n" + delta += "--------------\n" + + for d in sorted(total_changes['lunar']): + delta += '* {0}\n'.format(d) + delta += "\n" + + missing_deps = '' + + if len(inst_list) > 0: + missing_deps = "Missing Dependencies:\n" + missing_deps += "=====================\n" + for pkg in sorted(inst_list): + missing_deps += " * [ ] {0}\n".format(pkg) + # Commit changes and file pull request - regen_dict = dict() - regen_dict[args.ros_distro] = args.only - overlay.regenerate_manifests(regen_dict) + overlay.regenerate_manifests(total_installers) overlay.commit_changes(args.ros_distro) - delta = "Regenerated: '%s'\n" % args.only - missing_deps = '' + if args.dry_run: info('Running in dry mode, not filing PR') title_file = open('.pr-title.tmp', 'w') @@ -169,85 +245,5 @@ def main(): sys.exit(0) file_pr(overlay, delta, missing_deps) - clean_up(args.ros_distro, args.output_repository_path) + clean_up() ok('Successfully synchronized repositories!') - sys.exit(0) - - for distro in selected_targets: - distro_installers, distro_broken, distro_changes =\ - generate_installers( - distro_name=distro, - overlay=overlay, - gen_pkg_func=regenerate_pkg, - preserve_existing=preserve_existing - ) - for key in distro_broken.keys(): - for pkg in distro_broken[key]: - total_broken.add(pkg) - - total_changes[distro] = distro_changes - total_installers[distro] = distro_installers - - num_changes = 0 - for distro_name in total_changes: - num_changes += len(total_changes[distro_name]) - - if num_changes == 0: - info('ROS distro is up to date.') - info('Exiting...') - clean_up(args.ros_distro) - sys.exit(0) - - # remove duplicates - inst_list = total_broken - - delta = "Changes:\n" - delta += "========\n" - - if 'indigo' in total_changes and len(total_changes['indigo']) > 0: - delta += "Indigo Changes:\n" - delta += "---------------\n" - - for d in sorted(total_changes['indigo']): - delta += '* {0}\n'.format(d) - delta += "\n" - - if 'kinetic' in total_changes and len(total_changes['kinetic']) > 0: - delta += "Kinetic Changes:\n" - delta += "----------------\n" - - for d in sorted(total_changes['kinetic']): - delta += '* {0}\n'.format(d) - delta += "\n" - - if 'lunar' in total_changes and len(total_changes['lunar']) > 0: - delta += "Lunar Changes:\n" - delta += "--------------\n" - - for d in sorted(total_changes['lunar']): - delta += '* {0}\n'.format(d) - delta += "\n" - - missing_deps = '' - - if len(inst_list) > 0: - missing_deps = "Missing Dependencies:\n" - missing_deps += "=====================\n" - for pkg in sorted(inst_list): - missing_deps += " * [ ] {0}\n".format(pkg) - - # Commit changes and file pull request - overlay.regenerate_manifests(total_installers) - overlay.commit_changes(args.ros_distro) - - if args.dry_run: - info('Running in dry mode, not filing PR') - title_file = open('.pr-title.tmp', 'w') - title_file.write('rosdistro sync, {0}\n'.format(time.ctime())) - pr_message_file = open('.pr-message.tmp', 'w') - pr_message_file.write('%s\n%s\n' % (delta, missing_deps)) - sys.exit(0) - file_pr(overlay, delta, missing_deps) - - clean_up(args.ros_distro, args.output_repository_path) - ok('Successfully synchronized repositories!') diff --git a/superflore/utils.py b/superflore/utils.py index 0f58180d..1204c7f1 100644 --- a/superflore/utils.py +++ b/superflore/utils.py @@ -14,11 +14,14 @@ import errno import os +import random import re +import string import sys from superflore.exceptions import UnknownLicense from superflore.exceptions import UnknownPlatform + from superflore.rosdep_support import resolve_rosdep_key from termcolor import colored @@ -73,6 +76,13 @@ def get_pkg_version(distro, pkg_name): return maj_min_patch +def rand_ascii_str(length=10): + """ + Generates a random string of ascii characters of length 'length' + """ + return ''.join(random.choice(string.ascii_letters) for x in range(length)) + + def download_yamls(): global base_yml global python_yml