Skip to content

Commit

Permalink
bitbake: generate SRC_URI with git fetcher instead of unstable github…
Browse files Browse the repository at this point in the history 
… archives

* fixes: ros/meta-ros#609

* github archives are regenerated from time to time with different checksums
  fetch them with git instead

* this allows to simplify the generation quite a bit
* there won't be any issues with top-level directory being named
  differently than ${BP}, because all recipes will use just
  S = ${WORKDIR}/git
* instead of fetching all the archives when generating recipes we'll
  just run "git ls-remote" for them and cache the results (this means
  that if the tag actually moves, then the recipe will still point
  to the old SRCREV)

* this is very specific to exact structure of typical ros package
  fetched from generated github archive, but AFAIK this change doesn't
  make it worse, the generation was already failing for some packages
  like:
  Tarball requested for repo 'tablet_socket_msgs', but I can't parse git URL 'https://gitlab.com/autowarefoundation/autoware.ai-ros-releases/messages-release.git'.
  Falling back on git clone for repo 'tablet_socket_msgs'.
  !!!! Failed generating installer for tablet_socket_msgs! 'tar'
  !!!! 84.4%: Failed to generate installer for package 'tablet_socket_msgs'!

Signed-off-by: Martin Jansa <[email protected]>
  • Loading branch information
@shr-project
shr-project committed Nov 19, 2019
1 parent 24ac6a8 commit 3ec64fd
Show file tree
Hide file tree
Showing 3 changed files with 74 additions and 70 deletions.
14 changes: 6 additions & 8 deletions superflore/generators/bitbake/gen_packages.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@


def regenerate_pkg(
overlay, pkg, distro, preserve_existing, tar_dir, md5_cache, sha256_cache,
overlay, pkg, distro, preserve_existing, srcrev_cache,
skip_keys
):
pkg_names = get_package_names(distro)[0]
Expand Down Expand Up @@ -70,7 +70,7 @@ def regenerate_pkg(
previous_version = existing[idx_version:].rstrip('.bb')
try:
current = oe_recipe(
distro, pkg, tar_dir, md5_cache, sha256_cache, skip_keys
distro, pkg, srcrev_cache, skip_keys
)
except InvalidPackage as e:
err('Invalid package: ' + str(e))
Expand Down Expand Up @@ -123,7 +123,7 @@ def regenerate_pkg(

def _gen_recipe_for_package(
distro, pkg_name, pkg, repo, ros_pkg,
pkg_rosinstall, tar_dir, md5_cache, sha256_cache, skip_keys
pkg_rosinstall, srcrev_cache, skip_keys
):
pkg_names = get_package_names(distro)
pkg_dep_walker = DependencyWalker(distro,
Expand Down Expand Up @@ -151,9 +151,7 @@ def _gen_recipe_for_package(
pkg_xml,
distro,
src_uri,
tar_dir,
md5_cache,
sha256_cache,
srcrev_cache,
skip_keys,
)
# add build dependencies
Expand Down Expand Up @@ -185,7 +183,7 @@ def _gen_recipe_for_package(

class oe_recipe(object):
def __init__(
self, distro, pkg_name, tar_dir, md5_cache, sha256_cache, skip_keys
self, distro, pkg_name, srcrev_cache, skip_keys
):
pkg = distro.release_packages[pkg_name]
repo = distro.repositories[pkg.repository_name].release_repository
Expand All @@ -197,7 +195,7 @@ def __init__(

self.recipe = _gen_recipe_for_package(
distro, pkg_name, pkg, repo, ros_pkg, pkg_rosinstall,
tar_dir, md5_cache, sha256_cache, skip_keys
srcrev_cache, skip_keys
)

def recipe_text(self):
Expand Down
18 changes: 5 additions & 13 deletions superflore/generators/bitbake/run.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,14 +121,10 @@ def main():
total_installers = dict()
total_changes = dict()
if args.tar_archive_dir:
sha256_filename = '%s/sha256_cache.pickle' % args.tar_archive_dir
md5_filename = '%s/md5_cache.pickle' % args.tar_archive_dir
srcrev_filename = '%s/srcrev_cache.pickle' % args.tar_archive_dir
else:
sha256_filename = None
md5_filename = None
with TempfileManager(args.tar_archive_dir) as tar_dir,\
CacheManager(sha256_filename) as sha256_cache,\
CacheManager(md5_filename) as md5_cache: # noqa
srcrev_filename = None
with CacheManager(srcrev_filename) as srcrev_cache:
if args.only:
distro = get_distro(args.ros_distro)
for pkg in args.only:
Expand All @@ -143,9 +139,7 @@ def main():
pkg,
distro,
preserve_existing,
tar_dir,
md5_cache,
sha256_cache,
srcrev_cache,
skip_keys=skip_keys,
)
except KeyError:
Expand Down Expand Up @@ -179,9 +173,7 @@ def main():
overlay,
regenerate_pkg,
preserve_existing,
tar_dir,
md5_cache,
sha256_cache,
srcrev_cache,
skip_keys,
skip_keys=skip_keys,
is_oe=True,
Expand Down
112 changes: 63 additions & 49 deletions superflore/generators/bitbake/yocto_recipe.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ class yoctoRecipe(object):

def __init__(
self, component_name, num_pkgs, pkg_name, pkg_xml, distro, src_uri,
tar_dir, md5_cache, sha256_cache, skip_keys
srcrev_cache, skip_keys
):
self.component = component_name
yoctoRecipe.max_component_name = max(
Expand Down Expand Up @@ -114,28 +114,13 @@ def __init__(
self.tdepends = set()
self.tdepends_external = set()
self.license_line = None
self.archive_name = None
self.license_md5 = None
self.tar_dir = tar_dir
if self.getArchiveName() not in md5_cache or \
self.getArchiveName() not in sha256_cache:
self.downloadArchive()
md5_cache[self.getArchiveName()] = hashlib.md5(
open(self.getArchiveName(), 'rb').read()).hexdigest()
sha256_cache[self.getArchiveName()] = hashlib.sha256(
open(self.getArchiveName(), 'rb').read()).hexdigest()
self.src_sha256 = sha256_cache[self.getArchiveName()]
self.src_md5 = md5_cache[self.getArchiveName()]
if self.src_uri not in srcrev_cache:
srcrev_cache[self.src_uri] = self.get_srcrev()
self.srcrev = srcrev_cache[self.src_uri]
self.skip_keys = skip_keys
self.multi_hyphen_re = re.compile('-{2,}')

def getArchiveName(self):
if not self.archive_name:
self.archive_name = self.tar_dir + "/" \
+ self.name.replace('-', '_') + '-' + str(self.version) \
+ '-' + self.distro + '.tar.gz'
return self.archive_name

def get_license_line(self):
self.license_line = ''
self.license_md5 = ''
Expand All @@ -151,18 +136,61 @@ def get_license_line(self):
self.license_md5 = md5.hexdigest()
break

def downloadArchive(self):
if os.path.exists(self.getArchiveName()):
info("Using cached archive for package '%s'..." % self.name)
else:
info("Downloading archive version for package '%s' from %s..." %
(self.name, self.src_uri))
urlretrieve(self.src_uri, self.getArchiveName())
def get_repo_src_uri(self):
"""
Parse out the git repository SRC_URI out of github archive, e.g.
github.com/ros2-gbp/ament_lint-release
from
https://github.com/ros2-gbp/ament_lint-release/archive/release/bouncy/ament_cmake_copyright/0.5.2-0.tar.gz
don't include the protocol, because bitbake git fetcher will use
git://...;protocol=https
while get_srcrev will need
https://...
"""
github_start = 'https://github.com/'
structure = self.src_uri.replace(github_start, '')
dirs = structure.split('/')
return "github.com/%s/%s" % (dirs[0], dirs[1])

def get_repo_branch_name(self):
"""
Parse out the git branch name out of github archive SRC_URI, e.g.
release/bouncy/ament_cmake_copyright
from
https://github.com/ros2-gbp/ament_lint-release/archive/release/bouncy/ament_cmake_copyright/0.5.2-0.tar.gz
"""
github_start = 'https://github.com/'
structure = self.src_uri.replace(github_start, '')
dirs = structure.split('/')
return '{0}/{1}/{2}'.format(dirs[3], dirs[4], dirs[5]).replace('.tar.gz', '')

def extractArchive(self):
tar = tarfile.open(self.getArchiveName(), "r:gz")
tar.extractall()
tar.close()
def get_repo_tag_name(self):
"""
Parse out the git tag name out of github archive SRC_URI, e.g.
release/bouncy/ament_cmake_copyright/0.5.2-0
from
https://github.com/ros2-gbp/ament_lint-release/archive/release/bouncy/ament_cmake_copyright/0.5.2-0.tar.gz
"""
github_start = 'https://github.com/'
structure = self.src_uri.replace(github_start, '')
dirs = structure.split('/')
return '{0}/{1}/{2}/{3}'.format(dirs[3], dirs[4], dirs[5], dirs[6]).replace('.tar.gz', '')

def get_srcrev(self):
# e.g. git ls-remote https://github.com/ros2-gbp/ament_lint-release release/bouncy/ament_cmake_copyright/0.5.2-0
# 48bf1aa1cb083a884fbc8520ced00523255aeaed refs/tags/release/bouncy/ament_cmake_copyright/0.5.2-0
# from https://github.com/ros2-gbp/ament_lint-release/archive/release/bouncy/ament_cmake_copyright/0.5.2-0.tar.gz
from git.cmd import Git

g = Git()
#warn("git ls-remote for %s\ngit ls-remote https://%s refs/tags/%s" % (self.src_uri, self.get_repo_src_uri(), self.get_repo_tag_name()))
for ref in g.execute(["git", "ls-remote", "https://%s" % self.get_repo_src_uri(), "refs/tags/%s" % self.get_repo_tag_name()]).split('\n'):
#warn(ref)
srcrev, tag = ref.split('\t')
if tag == "refs/tags/%s" % self.get_repo_tag_name():
return srcrev
err("Cannot map refs/tags/%s to srcrev in https://%s repository with git ls-remote" % (self.get_repo_tag_name(), self.get_repo_src_uri()))
return "INVALID"

def add_build_depend(self, bdepend, internal=True):
if bdepend not in self.skip_keys:
Expand Down Expand Up @@ -218,19 +246,6 @@ def add_test_depend(self, tdepend, internal=True):
if tdepend not in self.tdepends:
self.tdepends_external.add(tdepend)

def get_src_location(self):
"""
Parse out the folder name.
TODO(allenh1): add a case for non-GitHub packages,
after they are supported.
"""
github_start = 'https://github.com/'
structure = self.src_uri.replace(github_start, '')
dirs = structure.split('/')
return '{0}-{1}-{2}-{3}-{4}'.format(dirs[1], dirs[3],
dirs[4], dirs[5],
dirs[6]).replace('.tar.gz', '')

def get_top_inherit_line(self):
ret = 'inherit ros_distro_{0}\n'.format(self.distro)
ret += 'inherit ros_superflore_generated\n\n'
Expand Down Expand Up @@ -463,12 +478,11 @@ def get_recipe_text(self, distributor):
ret += '${ROS_BUILDTOOL_EXPORT_DEPENDS}"\n\n'
ret += 'RDEPENDS_${PN} += "${ROS_EXEC_DEPENDS}"' + '\n\n'
# SRC_URI
ret += 'SRC_URI = "' + self.src_uri + ';'
ret += 'downloadfilename=${ROS_SP}.tar.gz"\n'
ret += 'SRC_URI[md5sum] = "' + self.src_md5 + '"\n'
ret += 'SRC_URI[sha256sum] = "' + self.src_sha256 + '"\n'
ret += 'S = "${WORKDIR}/'
ret += self.get_src_location() + '"\n\n'
ret += '# matches with: ' + self.src_uri + '\n'
ret += 'ROS_BRANCH ?= "branch=' + self.get_repo_branch_name() + '"\n'
ret += 'SRC_URI = "git://' + self.get_repo_src_uri() + ';${ROS_BRANCH};protocol=https"\n'
ret += 'SRCREV = "' + self.srcrev + '"\n'
ret += 'S = "${WORKDIR}/git"\n\n'
ret += 'ROS_COMPONENT_TYPE = "${@ros_distro__get_component_type(\''
ret += self.oe_component + '\', d)}"\n'
ret += 'ROS_BUILD_TYPE = "' + self.build_type + '"\n'
Expand Down

0 comments on commit 3ec64fd

Please sign in to comment.