From 6165add0673c6a233b60057e20f11eee1a75feb7 Mon Sep 17 00:00:00 2001 From: Herb Kuta Date: Tue, 3 Sep 2019 10:21:38 -0700 Subject: [PATCH] Change handling of unmapped platform packages for OpenEmbedded (#223) - Use ${ROS_UNRESOLVED_PLATFORM_PKG_} as the mapped value when isn't resolved by entries in rosdep/*.yaml and add assignments of the form ROS_UNRESOLVED_PLATFORM_PKG_ = "UNRESOLVED-" to conf/ros-distro/include//generated-ros-distro.inc . These assignments are then manually overridden in conf/ros-distro/include//ros-distro.inc as the mappings are determined. This make possible rapid iteration because superflore need not be rerun until rosdep/*.yaml are updated with all of the new mappings. - Drop using the OpenEmbedded Layer Index to guess the mapping of an unresolved platform package. It would often guess wrong, requiring a rerun of superflore. This change means that yoctoRecipe.rosdep_cache is now only used to generate rosdep-resolved.yaml . - Also rename generate_rosdistro_conf() -> generate_ros_distro_inc() --- requirements.txt | 1 - setup.py | 1 - superflore/generators/bitbake/oe_query.py | 154 ------------------ superflore/generators/bitbake/run.py | 4 +- superflore/generators/bitbake/yocto_recipe.py | 84 +++++----- 5 files changed, 45 insertions(+), 199 deletions(-) delete mode 100644 superflore/generators/bitbake/oe_query.py diff --git a/requirements.txt b/requirements.txt index 5f67fc5f..a50e25ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,5 +13,4 @@ docker pyyaml pygithub catkin_pkg -bs4 rospkg diff --git a/setup.py b/setup.py index 58065a80..96ff9d27 100755 --- a/setup.py +++ b/setup.py @@ -19,7 +19,6 @@ 'pyyaml', 'pygithub', 'catkin_pkg >= 0.4.0', - 'bs4', 'rospkg >= 1.1.8', ] diff --git a/superflore/generators/bitbake/oe_query.py b/superflore/generators/bitbake/oe_query.py deleted file mode 100644 index 7555e7ac..00000000 --- a/superflore/generators/bitbake/oe_query.py +++ /dev/null @@ -1,154 +0,0 @@ -# Copyright 2019 Open Source Robotics Foundation, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from collections import OrderedDict -import urllib - -import bs4 - - -class OpenEmbeddedLayersDB(object): - def __init__(self): - # Tells if we could read recipe information - self._exists = False - # OpenEmbedded branch to be queried - self._oe_branch = 'thud' - # Valid layers in priority order to filter when searching for a recipe - self._prio_valid_layers = OrderedDict.fromkeys( - ['openembedded-core', 'meta-oe', 'meta-python', 'meta-multimedia', - 'meta-ros', 'meta-intel-realsense', 'meta-qt5', 'meta-clang', - 'meta-sca', 'meta-openstack', 'meta-virtualization']) - # All fields below come straight from OE Layer query table results - self.name = '' - self.version = '' - self.summary = '' - self.description = '' - self.section = '' - self.license = '' - self.homepage = '' - self.recipe = '' - self.layer = '' - self.inherits = '' - self.dependencies = '' - self.packageconfig = '' - - def __str__(self): - if not self._exists: - return '' - return '\n'.join( - [getattr(self, i) for i in vars(self) if not i.startswith('_')]) - - def _fill_field(self, key, value): - if key: - my_attr = '{}'.format(key.split()[0].lower()) - if hasattr(self, my_attr): - setattr(self, my_attr, value) - return True - return False - - def _get_first_on_multiple_matches(self, bs): - class QueryResult: - def __init__(self): - self.recipe_name = '' - self.link = '' - self.version = '' - self.description = '' - self.layer = '' - - def __str__(self): - return '\n'.join([self.recipe_name, self.link, self.version, - self.description, self.layer]) - - if bs.table.find('th', text='Recipe name'): - tr = bs.table.find('tr') - while tr: - td = tr.find('td') - tr = tr.findNext('tr') - if not td: - continue - qr = QueryResult() - for f in ['recipe_name', 'version', 'description', 'layer']: - if f == 'recipe_name': - a = td.find('a') - if a: - setattr(qr, 'link', str( - "https://layers.openembedded.org" - + a.get('href', '')) - ) - setattr(qr, f, str(td.text)) - td = td.find_next_sibling() - if not td: - break - if qr.link: - # Get first valid entry - self._query_url(qr.link) - return - - def _query_url(self, query_url): - try: - req = urllib.request.urlopen(query_url) - read_str = req.read() - bs = bs4.BeautifulSoup(read_str, "html.parser") - th = bs.table.find('th', text='Name') - while th: - td = th.findNext('td') - if td: - self._exists |= self._fill_field( - str(th.text), str(td.text)) - th = th.findNext('th') - except Exception: - self._exists = False - return - if not self._exists and bs: - # Didn't match fully, so search on multi-match table - self._get_first_on_multiple_matches(bs) - else: - # Confirm that the only recipe found is indeed in a valid layer - for layer in self._prio_valid_layers: - if self.layer.startswith(layer): - return - self._exists = False - - def exists(self): - return self._exists - - def query_recipe(self, recipe): - if recipe: - url_prefix = 'https://layers.openembedded.org/layerindex/branch/' - url_prefix += '{}/recipes/'.format(self._oe_branch) - url_prefix += '?q={}' - for layer in self._prio_valid_layers: - query_url = url_prefix.format( - recipe + urllib.parse.quote(' layer:') + layer) - self._query_url(query_url) - if self.exists(): - return - - -def main(): - for recipe in [ - '', 'clang', 'ament_cmake_core', 'ament-cmake-core', 'libxml2', - 'bullet', 'sdl', 'sdl-image', 'qtbase']: - print('Checking ' + recipe + '...') - oe_query = OpenEmbeddedLayersDB() - oe_query.query_recipe(recipe) - if oe_query.exists(): - print(oe_query) - else: - print("Recipe {} doesn't exist!".format(recipe)) - print() - - -if __name__ == "__main__": - main() diff --git a/superflore/generators/bitbake/run.py b/superflore/generators/bitbake/run.py index 96e94d17..93424a37 100644 --- a/superflore/generators/bitbake/run.py +++ b/superflore/generators/bitbake/run.py @@ -140,7 +140,7 @@ def main(): except KeyError: err("No package to satisfy key '%s'" % pkg) sys.exit(1) - yoctoRecipe.generate_rosdistro_conf( + yoctoRecipe.generate_ros_distro_inc( _repo, args.ros_distro, overlay.get_file_revision_logs( 'files/{0}/cache.yaml'.format(args.ros_distro)), distro.release_platforms, skip_keys) @@ -188,7 +188,7 @@ def main(): ) total_changes[adistro] = distro_changes total_installers[adistro] = distro_installers - yoctoRecipe.generate_rosdistro_conf( + yoctoRecipe.generate_ros_distro_inc( _repo, args.ros_distro, overlay.get_file_revision_logs( 'files/{0}/cache.yaml'.format(args.ros_distro)), distro.release_platforms, skip_keys) diff --git a/superflore/generators/bitbake/yocto_recipe.py b/superflore/generators/bitbake/yocto_recipe.py index 3d56c864..cb86a165 100644 --- a/superflore/generators/bitbake/yocto_recipe.py +++ b/superflore/generators/bitbake/yocto_recipe.py @@ -49,8 +49,12 @@ from superflore.utils import warn import yaml +UNRESOLVED_PLATFORM_PKG_PREFIX = 'ROS_UNRESOLVED_PLATFORM_PKG_' +UNRESOLVED_PLATFORM_PKG_REFERENCE_PREFIX = '${' + UNRESOLVED_PLATFORM_PKG_PREFIX class yoctoRecipe(object): + # This is used to generate rosdep-resolved.yaml => don't call + # convert_to_oe_name() on what's added. rosdep_cache = defaultdict(set) generated_recipes = dict() generated_components = set() @@ -148,9 +152,9 @@ def get_license_line(self): def downloadArchive(self): if os.path.exists(self.getArchiveName()): - info("using cached archive for package '%s'..." % self.name) + info("Using cached archive for package '%s'..." % self.name) else: - info("downloading archive version for package '%s' from %s..." % + info("Downloading archive version for package '%s' from %s..." % (self.name, self.src_uri)) urlretrieve(self.src_uri, self.getArchiveName()) @@ -245,8 +249,13 @@ def translate_license(self, l): return self.trim_hyphens(l.translate(conversion_table)) @staticmethod - def get_native_suffix(is_native=False): - return '-native' if is_native else '' + def modify_name_if_native(dep, is_native): + # If the name is for an unresolved platform package, move the "-native" + # inside the "}" so that it's part of the variable name. + if dep.startswith(UNRESOLVED_PLATFORM_PKG_REFERENCE_PREFIX): + return dep[0:-len('}')] + ('-native}' if is_native else '}') + else: + return dep + ('-native' if is_native else '') @staticmethod def get_spacing_prefix(): @@ -286,12 +295,14 @@ def convert_to_oe_name(cls, dep, is_native=False): dep = dep[:-len('_dev')] + '-rosdev' elif dep in ('ros1', 'ros2'): dep += '--distro-renamed' - return cls.convert_dep_except_oe_vars(dep) \ - + cls.get_native_suffix(is_native) + return cls.modify_name_if_native(cls.convert_dep_except_oe_vars(dep), + is_native) @classmethod def generate_multiline_variable(cls, var, container, sort=True, key=None): if sort: + # TODO: Have default drop trailing '}' so that + # "${..._foo-native}" sorts after "${..._foo}". container = sorted(container, key=key) assignment = '{0} = "'.format(var) expression = '"\n' @@ -326,40 +337,16 @@ def get_dependencies( yoctoRecipe.rosdep_cache[dep].add(res) info('External dependency add: ' + recipe) except UnresolvedDependency: - info('Unresolved dependency: ' + dep) - if dep in yoctoRecipe.rosdep_cache: - cached_deps = yoctoRecipe.rosdep_cache[dep] - if cached_deps == set(['null']): - system_dependencies.add(dep) - recipe = dep + self.get_native_suffix(is_native) - dependencies.add(recipe) - msg = 'Failed to resolve (cached):' - warn('{0} {1}: {2}'.format(msg, dep, recipe)) - elif cached_deps: - system_dependencies |= cached_deps - for d in cached_deps: - recipe = self.convert_to_oe_name(d, is_native) - dependencies.add(recipe) - msg = 'Resolved in OpenEmbedded (cached):' - info('{0} {1}: {2}'.format(msg, dep, recipe)) - continue - oe_query = OpenEmbeddedLayersDB() - oe_query.query_recipe(self.convert_to_oe_name(dep)) - if oe_query.exists(): - recipe = self.convert_to_oe_name(oe_query.name, is_native) - dependencies.add(recipe) - oe_name = self.convert_to_oe_name(oe_query.name) - system_dependencies.add(oe_name) - yoctoRecipe.rosdep_cache[dep].add(oe_name) - info('Resolved in OpenEmbedded: ' + dep + ' as ' + - oe_query.name + ' in ' + oe_query.layer + - ' as recipe ' + recipe) - else: - recipe = dep + self.get_native_suffix(is_native) - dependencies.add(recipe) - system_dependencies.add(dep) - yoctoRecipe.rosdep_cache[dep].add('null') - warn('Failed to resolve fully: ' + dep) + unresolved_name = UNRESOLVED_PLATFORM_PKG_REFERENCE_PREFIX\ + + dep + '}' + recipe = self.convert_to_oe_name(unresolved_name, is_native) + dependencies.add(recipe) + system_dependencies.add(recipe) + # Never add -native. + rosdep_name = self.convert_to_oe_name(unresolved_name, False) + yoctoRecipe.rosdep_cache[dep].add(rosdep_name) + info('Unresolved external dependency add: ' + recipe) + return dependencies, system_dependencies def get_recipe_text(self, distributor): @@ -534,7 +521,7 @@ def generate_superflore_datetime_inc(basepath, dist, now): raise e @staticmethod - def generate_rosdistro_conf( + def generate_ros_distro_inc( basepath, distro, version, platforms, skip_keys=[]): conf_dir = '{}/conf/ros-distro/include/{}/'.format(basepath, distro) conf_file_name = 'generated-ros-distro.inc' @@ -651,6 +638,21 @@ def generate_rosdistro_conf( yoctoRecipe.generate_multiline_variable( 'ROS_SUPERFLORE_GENERATED_RECIPES_FOR_COMPONENTS', yoctoRecipe.generated_components)) + conf_file.write( + '\n# Platform packages without a OE-RECIPE@OE-LAYER mapping' + + ' in base.yaml, python.yaml, or ruby.yaml. Until they are' + + ' added, override\n# the settings in ros-distro.inc .\n') + # Drop trailing "}" so that "..._foo-native" sorts after + # "..._foo". + unresolved = [p[0:-1] for p in yoctoRecipe.platform_deps\ + if p.startswith(UNRESOLVED_PLATFORM_PKG_REFERENCE_PREFIX)] + for p in sorted(unresolved): + # PN is last underscore-separated field. NB the trailing '}' + # has already been removed. + pn = p.split('_')[-1] + conf_file.write(UNRESOLVED_PLATFORM_PKG_PREFIX + pn + + ' = "UNRESOLVED-' + pn + '"\n') + ok('Wrote {0}'.format(conf_path)) except OSError as e: err('Failed to write conf {} to disk! {}'.format(conf_path, e))