From 3ec64fd792a38879c7707b2a1fd42665d32be8cb Mon Sep 17 00:00:00 2001 From: Martin Jansa Date: Tue, 6 Aug 2019 10:01:42 +0000 Subject: [PATCH] bitbake: generate SRC_URI with git fetcher instead of unstable github archives * fixes: https://github.com/ros/meta-ros/issues/609 * github archives are regenerated from time to time with different checksums fetch them with git instead * this allows to simplify the generation quite a bit * there won't be any issues with top-level directory being named differently than ${BP}, because all recipes will use just S = ${WORKDIR}/git * instead of fetching all the archives when generating recipes we'll just run "git ls-remote" for them and cache the results (this means that if the tag actually moves, then the recipe will still point to the old SRCREV) * this is very specific to exact structure of typical ros package fetched from generated github archive, but AFAIK this change doesn't make it worse, the generation was already failing for some packages like: Tarball requested for repo 'tablet_socket_msgs', but I can't parse git URL 'https://gitlab.com/autowarefoundation/autoware.ai-ros-releases/messages-release.git'. Falling back on git clone for repo 'tablet_socket_msgs'. !!!! Failed generating installer for tablet_socket_msgs! 'tar' !!!! 84.4%: Failed to generate installer for package 'tablet_socket_msgs'! Signed-off-by: Martin Jansa --- superflore/generators/bitbake/gen_packages.py | 14 +-- superflore/generators/bitbake/run.py | 18 +-- superflore/generators/bitbake/yocto_recipe.py | 112 ++++++++++-------- 3 files changed, 74 insertions(+), 70 deletions(-) diff --git a/superflore/generators/bitbake/gen_packages.py b/superflore/generators/bitbake/gen_packages.py index 6679c214..9e4b2959 100644 --- a/superflore/generators/bitbake/gen_packages.py +++ b/superflore/generators/bitbake/gen_packages.py @@ -33,7 +33,7 @@ def regenerate_pkg( - overlay, pkg, distro, preserve_existing, tar_dir, md5_cache, sha256_cache, + overlay, pkg, distro, preserve_existing, srcrev_cache, skip_keys ): pkg_names = get_package_names(distro)[0] @@ -70,7 +70,7 @@ def regenerate_pkg( previous_version = existing[idx_version:].rstrip('.bb') try: current = oe_recipe( - distro, pkg, tar_dir, md5_cache, sha256_cache, skip_keys + distro, pkg, srcrev_cache, skip_keys ) except InvalidPackage as e: err('Invalid package: ' + str(e)) @@ -123,7 +123,7 @@ def regenerate_pkg( def _gen_recipe_for_package( distro, pkg_name, pkg, repo, ros_pkg, - pkg_rosinstall, tar_dir, md5_cache, sha256_cache, skip_keys + pkg_rosinstall, srcrev_cache, skip_keys ): pkg_names = get_package_names(distro) pkg_dep_walker = DependencyWalker(distro, @@ -151,9 +151,7 @@ def _gen_recipe_for_package( pkg_xml, distro, src_uri, - tar_dir, - md5_cache, - sha256_cache, + srcrev_cache, skip_keys, ) # add build dependencies @@ -185,7 +183,7 @@ def _gen_recipe_for_package( class oe_recipe(object): def __init__( - self, distro, pkg_name, tar_dir, md5_cache, sha256_cache, skip_keys + self, distro, pkg_name, srcrev_cache, skip_keys ): pkg = distro.release_packages[pkg_name] repo = distro.repositories[pkg.repository_name].release_repository @@ -197,7 +195,7 @@ def __init__( self.recipe = _gen_recipe_for_package( distro, pkg_name, pkg, repo, ros_pkg, pkg_rosinstall, - tar_dir, md5_cache, sha256_cache, skip_keys + srcrev_cache, skip_keys ) def recipe_text(self): diff --git a/superflore/generators/bitbake/run.py b/superflore/generators/bitbake/run.py index 1a86cff3..8bc6e77d 100644 --- a/superflore/generators/bitbake/run.py +++ b/superflore/generators/bitbake/run.py @@ -121,14 +121,10 @@ def main(): total_installers = dict() total_changes = dict() if args.tar_archive_dir: - sha256_filename = '%s/sha256_cache.pickle' % args.tar_archive_dir - md5_filename = '%s/md5_cache.pickle' % args.tar_archive_dir + srcrev_filename = '%s/srcrev_cache.pickle' % args.tar_archive_dir else: - sha256_filename = None - md5_filename = None - with TempfileManager(args.tar_archive_dir) as tar_dir,\ - CacheManager(sha256_filename) as sha256_cache,\ - CacheManager(md5_filename) as md5_cache: # noqa + srcrev_filename = None + with CacheManager(srcrev_filename) as srcrev_cache: if args.only: distro = get_distro(args.ros_distro) for pkg in args.only: @@ -143,9 +139,7 @@ def main(): pkg, distro, preserve_existing, - tar_dir, - md5_cache, - sha256_cache, + srcrev_cache, skip_keys=skip_keys, ) except KeyError: @@ -179,9 +173,7 @@ def main(): overlay, regenerate_pkg, preserve_existing, - tar_dir, - md5_cache, - sha256_cache, + srcrev_cache, skip_keys, skip_keys=skip_keys, is_oe=True, diff --git a/superflore/generators/bitbake/yocto_recipe.py b/superflore/generators/bitbake/yocto_recipe.py index 71fa5580..86ff57e9 100644 --- a/superflore/generators/bitbake/yocto_recipe.py +++ b/superflore/generators/bitbake/yocto_recipe.py @@ -68,7 +68,7 @@ class yoctoRecipe(object): def __init__( self, component_name, num_pkgs, pkg_name, pkg_xml, distro, src_uri, - tar_dir, md5_cache, sha256_cache, skip_keys + srcrev_cache, skip_keys ): self.component = component_name yoctoRecipe.max_component_name = max( @@ -114,28 +114,13 @@ def __init__( self.tdepends = set() self.tdepends_external = set() self.license_line = None - self.archive_name = None self.license_md5 = None - self.tar_dir = tar_dir - if self.getArchiveName() not in md5_cache or \ - self.getArchiveName() not in sha256_cache: - self.downloadArchive() - md5_cache[self.getArchiveName()] = hashlib.md5( - open(self.getArchiveName(), 'rb').read()).hexdigest() - sha256_cache[self.getArchiveName()] = hashlib.sha256( - open(self.getArchiveName(), 'rb').read()).hexdigest() - self.src_sha256 = sha256_cache[self.getArchiveName()] - self.src_md5 = md5_cache[self.getArchiveName()] + if self.src_uri not in srcrev_cache: + srcrev_cache[self.src_uri] = self.get_srcrev() + self.srcrev = srcrev_cache[self.src_uri] self.skip_keys = skip_keys self.multi_hyphen_re = re.compile('-{2,}') - def getArchiveName(self): - if not self.archive_name: - self.archive_name = self.tar_dir + "/" \ - + self.name.replace('-', '_') + '-' + str(self.version) \ - + '-' + self.distro + '.tar.gz' - return self.archive_name - def get_license_line(self): self.license_line = '' self.license_md5 = '' @@ -151,18 +136,61 @@ def get_license_line(self): self.license_md5 = md5.hexdigest() break - def downloadArchive(self): - if os.path.exists(self.getArchiveName()): - info("Using cached archive for package '%s'..." % self.name) - else: - info("Downloading archive version for package '%s' from %s..." % - (self.name, self.src_uri)) - urlretrieve(self.src_uri, self.getArchiveName()) + def get_repo_src_uri(self): + """ + Parse out the git repository SRC_URI out of github archive, e.g. + github.com/ros2-gbp/ament_lint-release + from + https://github.com/ros2-gbp/ament_lint-release/archive/release/bouncy/ament_cmake_copyright/0.5.2-0.tar.gz + don't include the protocol, because bitbake git fetcher will use + git://...;protocol=https + while get_srcrev will need + https://... + """ + github_start = 'https://github.com/' + structure = self.src_uri.replace(github_start, '') + dirs = structure.split('/') + return "github.com/%s/%s" % (dirs[0], dirs[1]) + + def get_repo_branch_name(self): + """ + Parse out the git branch name out of github archive SRC_URI, e.g. + release/bouncy/ament_cmake_copyright + from + https://github.com/ros2-gbp/ament_lint-release/archive/release/bouncy/ament_cmake_copyright/0.5.2-0.tar.gz + """ + github_start = 'https://github.com/' + structure = self.src_uri.replace(github_start, '') + dirs = structure.split('/') + return '{0}/{1}/{2}'.format(dirs[3], dirs[4], dirs[5]).replace('.tar.gz', '') - def extractArchive(self): - tar = tarfile.open(self.getArchiveName(), "r:gz") - tar.extractall() - tar.close() + def get_repo_tag_name(self): + """ + Parse out the git tag name out of github archive SRC_URI, e.g. + release/bouncy/ament_cmake_copyright/0.5.2-0 + from + https://github.com/ros2-gbp/ament_lint-release/archive/release/bouncy/ament_cmake_copyright/0.5.2-0.tar.gz + """ + github_start = 'https://github.com/' + structure = self.src_uri.replace(github_start, '') + dirs = structure.split('/') + return '{0}/{1}/{2}/{3}'.format(dirs[3], dirs[4], dirs[5], dirs[6]).replace('.tar.gz', '') + + def get_srcrev(self): + # e.g. git ls-remote https://github.com/ros2-gbp/ament_lint-release release/bouncy/ament_cmake_copyright/0.5.2-0 + # 48bf1aa1cb083a884fbc8520ced00523255aeaed refs/tags/release/bouncy/ament_cmake_copyright/0.5.2-0 + # from https://github.com/ros2-gbp/ament_lint-release/archive/release/bouncy/ament_cmake_copyright/0.5.2-0.tar.gz + from git.cmd import Git + + g = Git() + #warn("git ls-remote for %s\ngit ls-remote https://%s refs/tags/%s" % (self.src_uri, self.get_repo_src_uri(), self.get_repo_tag_name())) + for ref in g.execute(["git", "ls-remote", "https://%s" % self.get_repo_src_uri(), "refs/tags/%s" % self.get_repo_tag_name()]).split('\n'): + #warn(ref) + srcrev, tag = ref.split('\t') + if tag == "refs/tags/%s" % self.get_repo_tag_name(): + return srcrev + err("Cannot map refs/tags/%s to srcrev in https://%s repository with git ls-remote" % (self.get_repo_tag_name(), self.get_repo_src_uri())) + return "INVALID" def add_build_depend(self, bdepend, internal=True): if bdepend not in self.skip_keys: @@ -218,19 +246,6 @@ def add_test_depend(self, tdepend, internal=True): if tdepend not in self.tdepends: self.tdepends_external.add(tdepend) - def get_src_location(self): - """ - Parse out the folder name. - TODO(allenh1): add a case for non-GitHub packages, - after they are supported. - """ - github_start = 'https://github.com/' - structure = self.src_uri.replace(github_start, '') - dirs = structure.split('/') - return '{0}-{1}-{2}-{3}-{4}'.format(dirs[1], dirs[3], - dirs[4], dirs[5], - dirs[6]).replace('.tar.gz', '') - def get_top_inherit_line(self): ret = 'inherit ros_distro_{0}\n'.format(self.distro) ret += 'inherit ros_superflore_generated\n\n' @@ -463,12 +478,11 @@ def get_recipe_text(self, distributor): ret += '${ROS_BUILDTOOL_EXPORT_DEPENDS}"\n\n' ret += 'RDEPENDS_${PN} += "${ROS_EXEC_DEPENDS}"' + '\n\n' # SRC_URI - ret += 'SRC_URI = "' + self.src_uri + ';' - ret += 'downloadfilename=${ROS_SP}.tar.gz"\n' - ret += 'SRC_URI[md5sum] = "' + self.src_md5 + '"\n' - ret += 'SRC_URI[sha256sum] = "' + self.src_sha256 + '"\n' - ret += 'S = "${WORKDIR}/' - ret += self.get_src_location() + '"\n\n' + ret += '# matches with: ' + self.src_uri + '\n' + ret += 'ROS_BRANCH ?= "branch=' + self.get_repo_branch_name() + '"\n' + ret += 'SRC_URI = "git://' + self.get_repo_src_uri() + ';${ROS_BRANCH};protocol=https"\n' + ret += 'SRCREV = "' + self.srcrev + '"\n' + ret += 'S = "${WORKDIR}/git"\n\n' ret += 'ROS_COMPONENT_TYPE = "${@ros_distro__get_component_type(\'' ret += self.oe_component + '\', d)}"\n' ret += 'ROS_BUILD_TYPE = "' + self.build_type + '"\n'