diff --git a/requirements.txt b/requirements.txt index 5f67fc5f..a50e25ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,5 +13,4 @@ docker pyyaml pygithub catkin_pkg -bs4 rospkg diff --git a/setup.py b/setup.py index 58065a80..96ff9d27 100755 --- a/setup.py +++ b/setup.py @@ -19,7 +19,6 @@ 'pyyaml', 'pygithub', 'catkin_pkg >= 0.4.0', - 'bs4', 'rospkg >= 1.1.8', ] diff --git a/superflore/generators/bitbake/oe_query.py b/superflore/generators/bitbake/oe_query.py deleted file mode 100644 index 7555e7ac..00000000 --- a/superflore/generators/bitbake/oe_query.py +++ /dev/null @@ -1,154 +0,0 @@ -# Copyright 2019 Open Source Robotics Foundation, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from collections import OrderedDict -import urllib - -import bs4 - - -class OpenEmbeddedLayersDB(object): - def __init__(self): - # Tells if we could read recipe information - self._exists = False - # OpenEmbedded branch to be queried - self._oe_branch = 'thud' - # Valid layers in priority order to filter when searching for a recipe - self._prio_valid_layers = OrderedDict.fromkeys( - ['openembedded-core', 'meta-oe', 'meta-python', 'meta-multimedia', - 'meta-ros', 'meta-intel-realsense', 'meta-qt5', 'meta-clang', - 'meta-sca', 'meta-openstack', 'meta-virtualization']) - # All fields below come straight from OE Layer query table results - self.name = '' - self.version = '' - self.summary = '' - self.description = '' - self.section = '' - self.license = '' - self.homepage = '' - self.recipe = '' - self.layer = '' - self.inherits = '' - self.dependencies = '' - self.packageconfig = '' - - def __str__(self): - if not self._exists: - return '' - return '\n'.join( - [getattr(self, i) for i in vars(self) if not i.startswith('_')]) - - def _fill_field(self, key, value): - if key: - my_attr = '{}'.format(key.split()[0].lower()) - if hasattr(self, my_attr): - setattr(self, my_attr, value) - return True - return False - - def _get_first_on_multiple_matches(self, bs): - class QueryResult: - def __init__(self): - self.recipe_name = '' - self.link = '' - self.version = '' - self.description = '' - self.layer = '' - - def __str__(self): - return '\n'.join([self.recipe_name, self.link, self.version, - self.description, self.layer]) - - if bs.table.find('th', text='Recipe name'): - tr = bs.table.find('tr') - while tr: - td = tr.find('td') - tr = tr.findNext('tr') - if not td: - continue - qr = QueryResult() - for f in ['recipe_name', 'version', 'description', 'layer']: - if f == 'recipe_name': - a = td.find('a') - if a: - setattr(qr, 'link', str( - "https://layers.openembedded.org" - + a.get('href', '')) - ) - setattr(qr, f, str(td.text)) - td = td.find_next_sibling() - if not td: - break - if qr.link: - # Get first valid entry - self._query_url(qr.link) - return - - def _query_url(self, query_url): - try: - req = urllib.request.urlopen(query_url) - read_str = req.read() - bs = bs4.BeautifulSoup(read_str, "html.parser") - th = bs.table.find('th', text='Name') - while th: - td = th.findNext('td') - if td: - self._exists |= self._fill_field( - str(th.text), str(td.text)) - th = th.findNext('th') - except Exception: - self._exists = False - return - if not self._exists and bs: - # Didn't match fully, so search on multi-match table - self._get_first_on_multiple_matches(bs) - else: - # Confirm that the only recipe found is indeed in a valid layer - for layer in self._prio_valid_layers: - if self.layer.startswith(layer): - return - self._exists = False - - def exists(self): - return self._exists - - def query_recipe(self, recipe): - if recipe: - url_prefix = 'https://layers.openembedded.org/layerindex/branch/' - url_prefix += '{}/recipes/'.format(self._oe_branch) - url_prefix += '?q={}' - for layer in self._prio_valid_layers: - query_url = url_prefix.format( - recipe + urllib.parse.quote(' layer:') + layer) - self._query_url(query_url) - if self.exists(): - return - - -def main(): - for recipe in [ - '', 'clang', 'ament_cmake_core', 'ament-cmake-core', 'libxml2', - 'bullet', 'sdl', 'sdl-image', 'qtbase']: - print('Checking ' + recipe + '...') - oe_query = OpenEmbeddedLayersDB() - oe_query.query_recipe(recipe) - if oe_query.exists(): - print(oe_query) - else: - print("Recipe {} doesn't exist!".format(recipe)) - print() - - -if __name__ == "__main__": - main() diff --git a/superflore/generators/bitbake/run.py b/superflore/generators/bitbake/run.py index 416f98d8..f60cd415 100644 --- a/superflore/generators/bitbake/run.py +++ b/superflore/generators/bitbake/run.py @@ -147,7 +147,7 @@ def main(): except KeyError: err("No package to satisfy key '%s'" % pkg) sys.exit(1) - yoctoRecipe.generate_rosdistro_conf( + yoctoRecipe.generate_ros_distro_inc( _repo, args.ros_distro, overlay.get_file_revision_logs( 'files/{0}/cache.yaml'.format(args.ros_distro)), distro.release_platforms, skip_keys) @@ -195,7 +195,7 @@ def main(): ) total_changes[adistro] = distro_changes total_installers[adistro] = distro_installers - yoctoRecipe.generate_rosdistro_conf( + yoctoRecipe.generate_ros_distro_inc( _repo, args.ros_distro, overlay.get_file_revision_logs( 'files/{0}/cache.yaml'.format(args.ros_distro)), distro.release_platforms, skip_keys) diff --git a/superflore/generators/bitbake/yocto_recipe.py b/superflore/generators/bitbake/yocto_recipe.py index 214337d2..e8c65024 100644 --- a/superflore/generators/bitbake/yocto_recipe.py +++ b/superflore/generators/bitbake/yocto_recipe.py @@ -35,7 +35,6 @@ from rosdistro import get_distribution_cache_string, get_index, get_index_url from superflore.exceptions import NoPkgXml from superflore.exceptions import UnresolvedDependency -from superflore.generators.bitbake.oe_query import OpenEmbeddedLayersDB from superflore.PackageMetadata import PackageMetadata from superflore.utils import err from superflore.utils import get_distros @@ -46,11 +45,17 @@ from superflore.utils import make_dir from superflore.utils import ok from superflore.utils import resolve_dep -from superflore.utils import warn import yaml +UNRESOLVED_PLATFORM_PKG_PREFIX = 'ROS_UNRESOLVED_PLATFORM_PKG_' +UNRESOLVED_PLATFORM_PKG_REFERENCE_PREFIX = '${'+UNRESOLVED_PLATFORM_PKG_PREFIX + class yoctoRecipe(object): + """ + This is used to generate rosdep-resolved.yaml => don't call + convert_to_oe_name() on what's added. + """ rosdep_cache = defaultdict(set) generated_recipes = dict() generated_components = set() @@ -148,9 +153,9 @@ def get_license_line(self): def downloadArchive(self): if os.path.exists(self.getArchiveName()): - info("using cached archive for package '%s'..." % self.name) + info("Using cached archive for package '%s'..." % self.name) else: - info("downloading archive version for package '%s' from %s..." % + info("Downloading archive version for package '%s' from %s..." % (self.name, self.src_uri)) urlretrieve(self.src_uri, self.getArchiveName()) @@ -245,8 +250,15 @@ def translate_license(self, l): return self.trim_hyphens(l.translate(conversion_table)) @staticmethod - def get_native_suffix(is_native=False): - return '-native' if is_native else '' + def modify_name_if_native(dep, is_native): + """ + If the name is for an unresolved platform package, move the "-native" + inside the "}" so that it's part of the variable name. + """ + if dep.startswith(UNRESOLVED_PLATFORM_PKG_REFERENCE_PREFIX): + return dep[0:-len('}')] + ('-native}' if is_native else '}') + else: + return dep + ('-native' if is_native else '') @staticmethod def get_spacing_prefix(): @@ -286,12 +298,17 @@ def convert_to_oe_name(cls, dep, is_native=False): dep = dep[:-len('_dev')] + '-rosdev' elif dep in ('ros1', 'ros2'): dep += '--distro-renamed' - return cls.convert_dep_except_oe_vars(dep) \ - + cls.get_native_suffix(is_native) + return cls.modify_name_if_native( + cls.convert_dep_except_oe_vars(dep), + is_native) @classmethod def generate_multiline_variable(cls, var, container, sort=True, key=None): if sort: + """ + TODO(herb-kuta-lge): Have default drop trailing '}' so that + "${..._foo-native}" sorts after "${..._foo}". + """ container = sorted(container, key=key) assignment = '{0} = "'.format(var) expression = '"\n' @@ -326,40 +343,16 @@ def get_dependencies( yoctoRecipe.rosdep_cache[dep].add(res) info('External dependency add: ' + recipe) except UnresolvedDependency: - info('Unresolved dependency: ' + dep) - if dep in yoctoRecipe.rosdep_cache: - cached_deps = yoctoRecipe.rosdep_cache[dep] - if cached_deps == set(['null']): - system_dependencies.add(dep) - recipe = dep + self.get_native_suffix(is_native) - dependencies.add(recipe) - msg = 'Failed to resolve (cached):' - warn('{0} {1}: {2}'.format(msg, dep, recipe)) - elif cached_deps: - system_dependencies |= cached_deps - for d in cached_deps: - recipe = self.convert_to_oe_name(d, is_native) - dependencies.add(recipe) - msg = 'Resolved in OpenEmbedded (cached):' - info('{0} {1}: {2}'.format(msg, dep, recipe)) - continue - oe_query = OpenEmbeddedLayersDB() - oe_query.query_recipe(self.convert_to_oe_name(dep)) - if oe_query.exists(): - recipe = self.convert_to_oe_name(oe_query.name, is_native) - dependencies.add(recipe) - oe_name = self.convert_to_oe_name(oe_query.name) - system_dependencies.add(oe_name) - yoctoRecipe.rosdep_cache[dep].add(oe_name) - info('Resolved in OpenEmbedded: ' + dep + ' as ' + - oe_query.name + ' in ' + oe_query.layer + - ' as recipe ' + recipe) - else: - recipe = dep + self.get_native_suffix(is_native) - dependencies.add(recipe) - system_dependencies.add(dep) - yoctoRecipe.rosdep_cache[dep].add('null') - warn('Failed to resolve fully: ' + dep) + unresolved_name = UNRESOLVED_PLATFORM_PKG_REFERENCE_PREFIX\ + + dep + '}' + recipe = self.convert_to_oe_name(unresolved_name, is_native) + dependencies.add(recipe) + system_dependencies.add(recipe) + # Never add -native. + rosdep_name = self.convert_to_oe_name(unresolved_name, False) + yoctoRecipe.rosdep_cache[dep].add(rosdep_name) + info('Unresolved external dependency add: ' + recipe) + return dependencies, system_dependencies def get_recipe_text(self, distributor): @@ -534,7 +527,7 @@ def generate_superflore_datetime_inc(basepath, dist, now): raise e @staticmethod - def generate_rosdistro_conf( + def generate_ros_distro_inc( basepath, distro, version, platforms, skip_keys=[]): conf_dir = '{}/conf/ros-distro/include/{}/'.format(basepath, distro) conf_file_name = 'generated-ros-distro.inc' @@ -651,6 +644,28 @@ def generate_rosdistro_conf( yoctoRecipe.generate_multiline_variable( 'ROS_SUPERFLORE_GENERATED_RECIPES_FOR_COMPONENTS', yoctoRecipe.generated_components)) + conf_file.write( + '\n# Platform packages without a OE-RECIPE@OE-LAYER' + + ' mapping in base.yaml, python.yaml, or ruby.yaml. Until' + + ' they are added, override\n# the settings in' + + ' ros-distro.inc .\n') + """ + Drop trailing "}" so that "..._foo-native" sorts after + "..._foo". + """ + unresolved = [p[0:-1] for p in yoctoRecipe.platform_deps + if p.startswith( + UNRESOLVED_PLATFORM_PKG_REFERENCE_PREFIX)] + for p in sorted(unresolved): + """ + PN is last underscore-separated field. NB the trailing '}' + has already been removed. + """ + pn = p.split('_')[-1] + conf_file.write( + UNRESOLVED_PLATFORM_PKG_PREFIX + pn + ' = "UNRESOLVED-' + + pn + '"\n') + ok('Wrote {0}'.format(conf_path)) except OSError as e: err('Failed to write conf {} to disk! {}'.format(conf_path, e)) diff --git a/tests/test_oe_query.py b/tests/test_oe_query.py deleted file mode 100644 index eb54571e..00000000 --- a/tests/test_oe_query.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2019 Open Source Robotics Foundation, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from superflore.generators.bitbake.oe_query import OpenEmbeddedLayersDB -import unittest - - -class TestOpenEmbeddedQuery(unittest.TestCase): - def test_init(self): - """Test OpenEmbedded Query __init__""" - oe_query = OpenEmbeddedLayersDB() - self.assertEqual(oe_query._oe_branch, 'thud') - - def test_sample_queries(self): - """Test sample queries to OpenEmbedded layers database""" - oe_query = OpenEmbeddedLayersDB() - oe_query.query_recipe('') - self.assertEqual(oe_query.exists(), False) - self.assertIs('', str(oe_query)) - oe_query.query_recipe('clang') - self.assertEqual(oe_query.exists(), True) - self.assertIn('LLVM based C/C++ compiler', str(oe_query))