Skip to content

Commit

Permalink
Add --only PKG option (ros-infrastructure#76)
Browse files Browse the repository at this point in the history 
* Creating an option to only regenerate a specified package.

* Regeneration of a single package is now working. Next is refactoring.

* Add unused installer list from regenerate function.

* Unify printing functions

* Forgot one of the print calls, and moved a variable accidently.

* Added util print functions to run.py.

* Make the exception handling for make_dir more strict.

* Linting after rebase.

* Removing duplicated logic from gen_packages function

* Moved generate installers function to its own file, outside of the gentoo directory.

* Remove all gentoo specific logic from gen_packages, and add logic to handle missing dependencies.

* Fixed issue with unresolved dependencies not being reported.

* Modify regenerate package logic to only regenerate manifests for packages with changes.

* Fix regen logic for a single package.

* Addresses comments by @tfoote.

* Added nargs='+' to the '--only' option so that multiple packages can be selected.
  • Loading branch information
@allenh1
allenh1 authored Oct 30, 2017
1 parent b037e09 commit 0d04abc
Show file tree
Hide file tree
Showing 6 changed files with 297 additions and 210 deletions.
81 changes: 81 additions & 0 deletions superflore/generate_installers.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
# Copyright 2017 Open Source Robotics Foundation, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from rosinstall_generator.distro import get_distro
from rosinstall_generator.distro import get_package_names

from superflore.utils import err
from superflore.utils import get_pkg_version
from superflore.utils import info
from superflore.utils import ok
from superflore.utils import warn


def generate_installers(
distro_name, # ros distro name
overlay, # repo instance
gen_pkg_func, # function to call for generating
preserve_existing=True # don't regenerate if installer exists
):
distro = get_distro(distro_name)
pkg_names = get_package_names(distro)
total = float(len(pkg_names[0]))
borkd_pkgs = dict()
changes = []
installers = []
bad_installers = []
succeeded = 0
failed = 0

info("Generating installers for distro '%s'" % distro_name)
for i, pkg in enumerate(sorted(pkg_names[0])):
version = get_pkg_version(distro, pkg)
percent = '%.1f' % (100 * (float(i) / total))
try:
current, bad_deps = gen_pkg_func(
overlay, pkg, distro, preserve_existing
)
if not current and bad_deps:
# we are missing dependencies
failed_msg = "{0}%: Failed to generate".format(percent)
failed_msg += " installer for package '%s'!" % pkg
err(failed_msg)
borkd_pkgs[pkg] = bad_deps
failed = failed + 1
continue
elif not current and preserve_existing:
# don't replace the installer
succeeded = succeeded + 1
continue
success_msg = 'Successfully generated installer for package'
ok('{0}%: {1} \'{2}\'.'.format(percent, success_msg, pkg))
succeeded = succeeded + 1
changes.append('*{0} --> {1}*'.format(pkg, version))
installers.append(pkg)
except KeyError:
failed_msg = 'Failed to generate installer'
err("{0}%: {1} for package {2}!".format(percent, failed_msg, pkg))
bad_installers.append(pkg)
failed = failed + 1
results = 'Generated {0} / {1}'.format(succeeded, failed + succeeded)
results += ' for distro {0}'.format(distro_name)
info("------ {0} ------\n".format(results))

if len(borkd_pkgs) > 0:
warn("Unresolved:")
for broken in borkd_pkgs.keys():
warn("{}:".format(broken))
warn(" {}".format(borkd_pkgs[broken]))

return installers, borkd_pkgs, changes
218 changes: 77 additions & 141 deletions superflore/generators/ebuild/gen_packages.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,19 +21,20 @@
from rosdistro.rosdistro import RosPackage

from rosinstall_generator.distro import _generate_rosinstall
from rosinstall_generator.distro import get_distro
from rosinstall_generator.distro import get_package_names

from termcolor import colored
from superflore.exceptions import UnresolvedDependency

import xmltodict

from .ebuild import Ebuild, UnresolvedDependency
from .metadata_xml import metadata_xml
from superflore.generators.ebuild.ebuild import Ebuild
from superflore.generators.ebuild.metadata_xml import metadata_xml

from superflore.utils import err
from superflore.utils import get_pkg_version
from superflore.utils import make_dir
from superflore.utils import ok
from superflore.utils import warn

org = "Open Source Robotics Foundation"
org_license = "BSD"
import xmltodict

# TODO(allenh1): This is a blacklist of things that
# do not yet support Python 3. This will be updated
Expand All @@ -42,145 +43,80 @@

no_python3 = ['tf']


def warn(string):
print(colored('>>>> {0}'.format(string), 'yellow'))


def ok(string):
print(colored('>>>> {0}'.format(string), 'green'))


def err(string):
print(colored('!!!! {0}'.format(string), 'red'))
org = "Open Source Robotics Foundation"
org_license = "BSD"


def make_dir(dirname):
def regenerate_pkg(overlay, pkg, distro, preserve_existing=False):
version = get_pkg_version(distro, pkg)
ebuild_name =\
'/ros-{0}/{1}/{1}-{2}.ebuild'.format(distro.name, pkg, version)
ebuild_name = overlay.repo.repo_dir + ebuild_name
patch_path = '/ros-{}/{}/files'.format(distro.name, pkg)
patch_path = overlay.repo.repo_dir + patch_path
has_patches = os.path.exists(patch_path)
pkg_names = get_package_names(distro)[0]

if pkg not in pkg_names:
raise RuntimeError("Unknown package '%s'" % (pkg))
# otherwise, remove a (potentially) existing ebuild.
existing = glob.glob(
'{0}/ros-{1}/{2}/*.ebuild'.format(
overlay.repo.repo_dir,
distro.name, pkg
)
)
if preserve_existing and existing:
ok("ebuild for package '%s' up to date, skipping..." % pkg)
return None, []
elif existing:
overlay.repo.remove_file(existing[0])
manifest_file = '{0}/ros-{1}/{2}/Manifest'.format(
overlay.repo.repo_dir, distro.name, pkg
)
overlay.repo.remove_file(manifest_file)
try:
os.makedirs(dirname)
except Exception:
pass


def get_pkg_version(distro, pkg_name):
pkg = distro.release_packages[pkg_name]
repo = distro.repositories[pkg.repository_name].release_repository
maj_min_patch, deb_inc = repo.version.split('-')
if deb_inc != '0':
return '{0}-r{1}'.format(maj_min_patch, deb_inc)
return maj_min_patch

current = gentoo_installer(distro, pkg, has_patches)
current.ebuild.name = pkg
except Exception as e:
err('Failed to generate installer for package {}!'.format(pkg))
raise e
try:
ebuild_text = current.ebuild_text()
metadata_text = current.metadata_text()
except UnresolvedDependency:
dep_err = 'Failed to resolve required dependencies for'
err("{0} package {1}!".format(dep_err, pkg))
unresolved = current.ebuild.get_unresolved()
for dep in unresolved:
err(" unresolved: \"{}\"".format(dep))
return None, current.ebuild.get_unresolved()
except KeyError as ke:
err("Failed to parse data for package {}!".format(pkg))
raise ke
make_dir(
"{}/ros-{}/{}".format(overlay.repo.repo_dir, distro.name, pkg)
)
success_msg = 'Successfully generated installer for package'
ok('{0} \'{1}\'.'.format(success_msg, pkg))

def generate_installers(distro_name, overlay, preserve_existing=True):
distro = get_distro(distro_name)
pkg_names = get_package_names(distro)
total = float(len(pkg_names[0]))
borkd_pkgs = dict()
changes = []
installers = []
bad_installers = []
succeeded = 0
failed = 0

for i, pkg in enumerate(sorted(pkg_names[0])):
version = get_pkg_version(distro, pkg)
ebuild_name =\
'/ros-{0}/{1}/{1}-{2}.ebuild'.format(distro_name, pkg, version)
ebuild_name = overlay.repo.repo_dir + ebuild_name
ebuild_exists = os.path.exists(ebuild_name)
patch_path = '/ros-{}/{}/files'.format(distro_name, pkg)
patch_path = overlay.repo.repo_dir + patch_path
has_patches = os.path.exists(patch_path)
percent = '%.1f' % (100 * (float(i) / total))

if preserve_existing and ebuild_exists:
skip_msg = 'Ebuild for package '
skip_msg += '{0} up to date, skipping...'.format(pkg)
status = '{0}%: {1}'.format(percent, skip_msg)
ok(status)
succeeded = succeeded + 1
continue
# otherwise, remove a (potentially) existing ebuild.
existing = glob.glob(
'{0}/ros-{1}/{2}/*.ebuild'.format(
overlay.repo.repo_dir,
distro_name, pkg
)
try:
ebuild_file = '{0}/ros-{1}/{2}/{2}-{3}.ebuild'.format(
overlay.repo.repo_dir,
distro.name, pkg, version
)
if existing:
overlay.repo.remove_file(existing[0])
manifest_file = '{0}/ros-{1}/{2}/Manifest'.format(
overlay.repo.repo_dir, distro_name, pkg
)
overlay.repo.remove_file(manifest_file)
try:
current = gentoo_installer(distro, pkg, has_patches)
current.ebuild.name = pkg
except Exception as e:
err('Failed to generate installer for package {}!'.format(pkg))
err(' exception: {0}'.format(e))
failed = failed + 1
continue
try:
ebuild_text = current.ebuild_text()
metadata_text = current.metadata_text()
except UnresolvedDependency:
dep_err = 'Failed to resolve required dependencies for'
err("{0} package {1}!".format(dep_err, pkg))
unresolved = current.ebuild.get_unresolved()
borkd_pkgs[pkg] = list()
for dep in unresolved:
err(" unresolved: \"{}\"".format(dep))
borkd_pkgs[pkg].append(dep)
err("Failed to generate installer for package {}!".format(pkg))
failed = failed + 1
continue # do not generate an incomplete ebuild
except KeyError:
err("Failed to parse data for package {}!".format(pkg))
unresolved = current.ebuild.get_unresolved()
err("Failed to generate installer for package {}!".format(pkg))
failed = failed + 1
continue # do not generate an incomplete ebuild
make_dir(
"{}/ros-{}/{}".format(overlay.repo.repo_dir, distro_name, pkg)
ebuild_file = open(ebuild_file, "w")
metadata_file = '{0}/ros-{1}/{2}/metadata.xml'.format(
overlay.repo.repo_dir,
distro.name, pkg
)
success_msg = 'Successfully generated installer for package'
ok('{0}%: {1} \'{2}\'.'.format(percent, success_msg, pkg))
succeeded = succeeded + 1

try:
ebuild_file = '{0}/ros-{1}/{2}/{2}-{3}.ebuild'.format(
overlay.repo.repo_dir,
distro_name, pkg, version
)
ebuild_file = open(ebuild_file, "w")
metadata_file = '{0}/ros-{1}/{2}/metadata.xml'.format(
overlay.repo.repo_dir,
distro_name, pkg
)
metadata_file = open(metadata_file, "w")
ebuild_file.write(ebuild_text)
metadata_file.write(metadata_text)
changes.append('*{0} --> {1}*'.format(pkg, version))
except Exception:
err("Failed to write ebuild/metadata to disk!")
installers.append(current)
failed_msg = 'Failed to generate installer'
err("{0}%: {1} for package {2}!".format(percent, failed_msg, pkg))
bad_installers.append(current)
failed = failed + 1
results = 'Generated {0} / {1}'.format(succeeded, failed + succeeded)
results += ' for distro {0}'.format(distro_name)
print("------ {0} ------".format(results))
print()

if len(borkd_pkgs) > 0:
warn("Unresolved:")
for broken in borkd_pkgs.keys():
warn("{}:".format(broken))
warn(" {}".format(borkd_pkgs[broken]))

return installers, borkd_pkgs, changes
metadata_file = open(metadata_file, "w")
ebuild_file.write(ebuild_text)
metadata_file.write(metadata_text)
except Exception as e:
err("Failed to write ebuild/metadata to disk!")
raise e
return current, []


def _gen_metadata_for_package(distro, pkg_name, pkg,
Expand Down
Loading

0 comments on commit 0d04abc

Please sign in to comment.