From 144e6cb65a41baafa0375645b5596596cc4db275 Mon Sep 17 00:00:00 2001 From: Yohann Cointe Date: Thu, 12 Sep 2019 17:12:16 +0200 Subject: [PATCH 1/4] build/package: Add Debian package list RedHat and Debian don't use the same packages so we need a new list of these packages. e.g: RedHat use `m2crypto` while Debian equivalent is `python-m2crypto` All kubernetes and Salt packages use the same version used on CentOS. The other packages use the last version available on official repositories. Refs: #1634 --- buildchain/buildchain/packaging.py | 6 +- buildchain/buildchain/salt_tree.py | 2 +- buildchain/buildchain/versions.py | 104 ++++++++++++++++++++++++++++- 3 files changed, 106 insertions(+), 6 deletions(-) diff --git a/buildchain/buildchain/packaging.py b/buildchain/buildchain/packaging.py index 789bec3838..5a4fe6c0c1 100644 --- a/buildchain/buildchain/packaging.py +++ b/buildchain/buildchain/packaging.py @@ -280,17 +280,17 @@ def _deb_package(name: str, sources: Path) -> targets.DEBPackage: _RPM_TO_BUILD_PKG_NAMES.append(pkg.name) # All packages not referenced in `RPM_TO_BUILD` but listed in -# `versions.PACKAGES` are supposed to be downloaded. +# `versions.RPM_PACKAGES` are supposed to be downloaded. RPM_TO_DOWNLOAD : FrozenSet[str] = frozenset( "{p.name}-{p.version}-{p.release}".format(p=package) - for package in versions.PACKAGES + for package in versions.RPM_PACKAGES if package.name not in _RPM_TO_BUILD_PKG_NAMES ) # Store these versions in a dict to use with doit.tools.config_changed _TO_DOWNLOAD_CONFIG : Dict[str, str] = { pkg.name: "{p.version}-{p.release}".format(p=pkg) - for pkg in versions.PACKAGES + for pkg in versions.RPM_PACKAGES if pkg.name not in _RPM_TO_BUILD_PKG_NAMES } diff --git a/buildchain/buildchain/salt_tree.py b/buildchain/buildchain/salt_tree.py index 4e7280e2e8..5acd4e0926 100644 --- a/buildchain/buildchain/salt_tree.py +++ b/buildchain/buildchain/salt_tree.py @@ -196,7 +196,7 @@ def _get_parts(self) -> Iterator[str]: 'kubernetes': {'version': versions.K8S_VERSION}, 'packages': { pkg.name: {'version': "{}-{}".format(pkg.version, pkg.release)} - for pkg in versions.PACKAGES + for pkg in versions.RPM_PACKAGES }, 'images': { img.name: {'version': img.version} diff --git a/buildchain/buildchain/versions.py b/buildchain/buildchain/versions.py index e76a2dd744..598e8827b8 100644 --- a/buildchain/buildchain/versions.py +++ b/buildchain/buildchain/versions.py @@ -230,7 +230,7 @@ def _version_prefix(version: str, prefix: str = 'v') -> str: # }}} # Packages {{{ -PACKAGES = ( +RPM_PACKAGES = ( # Remote packages Package( name='containerd', @@ -360,6 +360,106 @@ def _version_prefix(version: str, prefix: str = 'v') -> str: ), ) -PACKAGES_MAP = {pkg.name: pkg for pkg in PACKAGES} +PACKAGES_MAP = {pkg.name: pkg for pkg in RPM_PACKAGES} + + +DEB_PACKAGES = ( + Package( + name='containerd', + version='', + release='', + ), + Package( + name='cri-tools', + version='', + release='', + ), + Package( + name='coreutils', + version='', + release='', + ), + Package( + name='ebtables', + version='', + release='', + ), + Package( + name='ethtool', + version='', + release='', + ), + Package( + name='e2fsprogs', + version='', + release='', + ), + Package( + name='genisoimage', + version='', + release='', + ), + Package( + name='iproute2', + version='', + release='', + ), + Package( + name='iptables', + version='', + release='', + ), + Package( + name='salt-minion={}'.format(SALT_VERSION), + version='', + release='', + ), + Package( + name='kubectl={}'.format(K8S_VERSION), + version='', + release='', + ), + Package( + name='kubelet={}'.format(K8S_VERSION), + version='', + release='', + ), + Package( + name='kubernetes-cni', + version='', + release='', + ), + Package( + name='python-m2crypto', + version='', + release='', + ), + Package( + name='runc', + version='', + release='', + ), + Package( + name='socat', + version='', + release='', + ), + Package( + name='sosreport', + version='', + release='', + ), + Package( + name='util-linux', + version='', + release='', + ), + Package( + name='xfsprogs', + version='', + release='', + ), +) + # }}} From c2a1ec9efe799a029ba92b30f1de27dcc373f2bd Mon Sep 17 00:00:00 2001 From: Yohann Cointe Date: Thu, 12 Sep 2019 17:28:32 +0200 Subject: [PATCH 2/4] build/package: Add the task to download Debian packages Add a new task to call the script which will download all the Debian packages required. This task relies a list of pre-defined packages, DEB_TO_DOWNLOAD. Refs: #1634 --- buildchain/buildchain/packaging.py | 51 +++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/buildchain/buildchain/packaging.py b/buildchain/buildchain/packaging.py index 5a4fe6c0c1..75f43b22ff 100644 --- a/buildchain/buildchain/packaging.py +++ b/buildchain/buildchain/packaging.py @@ -54,6 +54,7 @@ def task_packaging() -> types.TaskDict: '_build_rpm_packages:*', '_build_rpm_repositories:*', '_build_deb_packages:*', + '_download_deb_packages', ], } @@ -134,7 +135,7 @@ def clean() -> None: run_config=docker_command.RPM_BASE_CONFIG ) return { - 'title': utils.title_with_target1('GET PKGS'), + 'title': utils.title_with_target1('GET RPM PKGS'), 'actions': [dl_packages_callable], 'targets': [constants.PKG_RPM_ROOT/'var'], 'task_dep': [ @@ -148,6 +149,40 @@ def clean() -> None: 'verbosity': 0, } + +def task__download_deb_packages() -> types.TaskDict: + """Download Debian packages locally.""" + # TODO: Clean the repository + mounts = [ + utils.bind_ro_mount( + source=constants.ROOT/'packages'/'debian'/'download_packages.py', + target=Path('/download_packages.py'), + ), + utils.bind_mount( + source=constants.REPO_DEB_ROOT, + target=Path('/repositories') + ), + ] + dl_packages_callable = docker_command.DockerRun( + command=['/download_packages.py', *DEB_TO_DOWNLOAD], + builder=DEB_BUILDER, + mounts=mounts, + environment={'SALT_VERSION': versions.SALT_VERSION}, + run_config=docker_command.DEB_BASE_CONFIG + ) + return { + 'title': utils.title_with_target1('GET DEB PKGS'), + 'actions': [dl_packages_callable], + 'targets': [constants.REPO_DEB_ROOT/'.witness'], + 'task_dep': [ + '_package_mkdir_deb_root', + '_package_mkdir_deb_iso_root', + '_build_deb_container' + ], + 'uptodate': [config_changed(_TO_DOWNLOAD_DEB_CONFIG)], + } + + def task__build_rpm_packages() -> Iterator[types.TaskDict]: """Build a RPM package.""" for repo_pkgs in RPM_TO_BUILD.values(): @@ -274,6 +309,7 @@ def _deb_package(name: str, sources: Path) -> targets.DEBPackage: } _RPM_TO_BUILD_PKG_NAMES : List[str] = [] +_DEB_TO_BUILD_PKG_NAMES : List[str] = [] for pkgs in RPM_TO_BUILD.values(): for pkg in pkgs: @@ -287,6 +323,7 @@ def _deb_package(name: str, sources: Path) -> targets.DEBPackage: if package.name not in _RPM_TO_BUILD_PKG_NAMES ) + # Store these versions in a dict to use with doit.tools.config_changed _TO_DOWNLOAD_CONFIG : Dict[str, str] = { pkg.name: "{p.version}-{p.release}".format(p=pkg) @@ -294,6 +331,13 @@ def _deb_package(name: str, sources: Path) -> targets.DEBPackage: if pkg.name not in _RPM_TO_BUILD_PKG_NAMES } +_TO_DOWNLOAD_DEB_CONFIG : Dict[str, str] = { + pkg.name: "{p.version}-{p.release}".format(p=pkg) + for pkg in versions.DEB_PACKAGES + if pkg.name not in _DEB_TO_BUILD_PKG_NAMES +} + + SCALITY_RPM_REPOSITORY = targets.RPMRepository( basename='_build_rpm_repositories', name='scality', @@ -356,5 +400,10 @@ def _deb_package(name: str, sources: Path) -> targets.DEBPackage: ) } +DEB_TO_DOWNLOAD : FrozenSet[str] = frozenset( + "{p.name}".format(p=package) + for package in versions.DEB_PACKAGES + if package.name not in DEB_TO_BUILD +) __all__ = utils.export_only_tasks(__name__) From ef6ee02a68d979bc415608ee9702c6aebfa1862d Mon Sep 17 00:00:00 2001 From: Yohann Cointe Date: Thu, 12 Sep 2019 17:38:41 +0200 Subject: [PATCH 3/4] package/debian: chown back the downloaded packages parent directory So packages folders have the same environment UID/GID. Refs: #1634 --- packages/debian/download_packages.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/debian/download_packages.py b/packages/debian/download_packages.py index 26ef741002..510dd5d4fb 100755 --- a/packages/debian/download_packages.py +++ b/packages/debian/download_packages.py @@ -180,12 +180,16 @@ def main(packages: Sequence[str], env: Mapping[str, str]) -> None: apt_pkg.init() cache = apt.cache.Cache() to_download = {} + dest = pathlib.Path('/repositories') for package in packages: deps = get_package_deps(package, cache) to_download.update(deps) for pkg in to_download.values(): filepath = download_package(pkg) os.chown(filepath, int(env['TARGET_UID']), int(env['TARGET_GID'])) + # TODO: need to be done by the build chain + for directory in dest.iterdir(): + os.chown(directory, int(env['TARGET_UID']), int(env['TARGET_GID'])) if __name__ == '__main__': From dd08f78980a07244ac7f9e206026b3461b1aa37c Mon Sep 17 00:00:00 2001 From: Yohann Cointe Date: Mon, 16 Sep 2019 12:02:53 +0200 Subject: [PATCH 4/4] build/package: add a witness to verify the well task running We add a .witness file creation in the scipt `download_packages.py` to verify that the script is well executed. So we add this file as a target in `packaging.py` Refs: #1634 --- packages/debian/download_packages.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/debian/download_packages.py b/packages/debian/download_packages.py index 510dd5d4fb..43aa05eef8 100755 --- a/packages/debian/download_packages.py +++ b/packages/debian/download_packages.py @@ -176,6 +176,11 @@ def download_package(package: apt.package.Version) -> pathlib.Path: def main(packages: Sequence[str], env: Mapping[str, str]) -> None: """Download the packages specified on the command-line.""" + witness_file = pathlib.Path('/repositories/.witness') + try: + witness_file.unlink() + except FileNotFoundError: + pass add_external_repositories(env['SALT_VERSION']) apt_pkg.init() cache = apt.cache.Cache() @@ -190,7 +195,8 @@ def main(packages: Sequence[str], env: Mapping[str, str]) -> None: # TODO: need to be done by the build chain for directory in dest.iterdir(): os.chown(directory, int(env['TARGET_UID']), int(env['TARGET_GID'])) - + witness_file.touch() + os.chown(witness_file, int(env['TARGET_UID']), int(env['TARGET_GID'])) if __name__ == '__main__': _, *PACKAGES = sys.argv