diff --git a/buildchain/buildchain/packaging.py b/buildchain/buildchain/packaging.py index 789bec3838..75f43b22ff 100644 --- a/buildchain/buildchain/packaging.py +++ b/buildchain/buildchain/packaging.py @@ -54,6 +54,7 @@ def task_packaging() -> types.TaskDict: '_build_rpm_packages:*', '_build_rpm_repositories:*', '_build_deb_packages:*', + '_download_deb_packages', ], } @@ -134,7 +135,7 @@ def clean() -> None: run_config=docker_command.RPM_BASE_CONFIG ) return { - 'title': utils.title_with_target1('GET PKGS'), + 'title': utils.title_with_target1('GET RPM PKGS'), 'actions': [dl_packages_callable], 'targets': [constants.PKG_RPM_ROOT/'var'], 'task_dep': [ @@ -148,6 +149,40 @@ def clean() -> None: 'verbosity': 0, } + +def task__download_deb_packages() -> types.TaskDict: + """Download Debian packages locally.""" + # TODO: Clean the repository + mounts = [ + utils.bind_ro_mount( + source=constants.ROOT/'packages'/'debian'/'download_packages.py', + target=Path('/download_packages.py'), + ), + utils.bind_mount( + source=constants.REPO_DEB_ROOT, + target=Path('/repositories') + ), + ] + dl_packages_callable = docker_command.DockerRun( + command=['/download_packages.py', *DEB_TO_DOWNLOAD], + builder=DEB_BUILDER, + mounts=mounts, + environment={'SALT_VERSION': versions.SALT_VERSION}, + run_config=docker_command.DEB_BASE_CONFIG + ) + return { + 'title': utils.title_with_target1('GET DEB PKGS'), + 'actions': [dl_packages_callable], + 'targets': [constants.REPO_DEB_ROOT/'.witness'], + 'task_dep': [ + '_package_mkdir_deb_root', + '_package_mkdir_deb_iso_root', + '_build_deb_container' + ], + 'uptodate': [config_changed(_TO_DOWNLOAD_DEB_CONFIG)], + } + + def task__build_rpm_packages() -> Iterator[types.TaskDict]: """Build a RPM package.""" for repo_pkgs in RPM_TO_BUILD.values(): @@ -274,26 +309,35 @@ def _deb_package(name: str, sources: Path) -> targets.DEBPackage: } _RPM_TO_BUILD_PKG_NAMES : List[str] = [] +_DEB_TO_BUILD_PKG_NAMES : List[str] = [] for pkgs in RPM_TO_BUILD.values(): for pkg in pkgs: _RPM_TO_BUILD_PKG_NAMES.append(pkg.name) # All packages not referenced in `RPM_TO_BUILD` but listed in -# `versions.PACKAGES` are supposed to be downloaded. +# `versions.RPM_PACKAGES` are supposed to be downloaded. RPM_TO_DOWNLOAD : FrozenSet[str] = frozenset( "{p.name}-{p.version}-{p.release}".format(p=package) - for package in versions.PACKAGES + for package in versions.RPM_PACKAGES if package.name not in _RPM_TO_BUILD_PKG_NAMES ) + # Store these versions in a dict to use with doit.tools.config_changed _TO_DOWNLOAD_CONFIG : Dict[str, str] = { pkg.name: "{p.version}-{p.release}".format(p=pkg) - for pkg in versions.PACKAGES + for pkg in versions.RPM_PACKAGES if pkg.name not in _RPM_TO_BUILD_PKG_NAMES } +_TO_DOWNLOAD_DEB_CONFIG : Dict[str, str] = { + pkg.name: "{p.version}-{p.release}".format(p=pkg) + for pkg in versions.DEB_PACKAGES + if pkg.name not in _DEB_TO_BUILD_PKG_NAMES +} + + SCALITY_RPM_REPOSITORY = targets.RPMRepository( basename='_build_rpm_repositories', name='scality', @@ -356,5 +400,10 @@ def _deb_package(name: str, sources: Path) -> targets.DEBPackage: ) } +DEB_TO_DOWNLOAD : FrozenSet[str] = frozenset( + "{p.name}".format(p=package) + for package in versions.DEB_PACKAGES + if package.name not in DEB_TO_BUILD +) __all__ = utils.export_only_tasks(__name__) diff --git a/buildchain/buildchain/salt_tree.py b/buildchain/buildchain/salt_tree.py index e682240d19..53edb2ef84 100644 --- a/buildchain/buildchain/salt_tree.py +++ b/buildchain/buildchain/salt_tree.py @@ -196,7 +196,7 @@ def _get_parts(self) -> Iterator[str]: 'kubernetes': {'version': versions.K8S_VERSION}, 'packages': { pkg.name: {'version': "{}-{}".format(pkg.version, pkg.release)} - for pkg in versions.PACKAGES + for pkg in versions.RPM_PACKAGES }, 'images': { img.name: {'version': img.version} diff --git a/buildchain/buildchain/versions.py b/buildchain/buildchain/versions.py index 676ca11245..ab95a436f3 100644 --- a/buildchain/buildchain/versions.py +++ b/buildchain/buildchain/versions.py @@ -230,7 +230,7 @@ def _version_prefix(version: str, prefix: str = 'v') -> str: # }}} # Packages {{{ -PACKAGES = ( +RPM_PACKAGES = ( # Remote packages Package( name='containerd', @@ -360,6 +360,106 @@ def _version_prefix(version: str, prefix: str = 'v') -> str: ), ) -PACKAGES_MAP = {pkg.name: pkg for pkg in PACKAGES} +PACKAGES_MAP = {pkg.name: pkg for pkg in RPM_PACKAGES} + + +DEB_PACKAGES = ( + Package( + name='containerd', + version='', + release='', + ), + Package( + name='cri-tools', + version='', + release='', + ), + Package( + name='coreutils', + version='', + release='', + ), + Package( + name='ebtables', + version='', + release='', + ), + Package( + name='ethtool', + version='', + release='', + ), + Package( + name='e2fsprogs', + version='', + release='', + ), + Package( + name='genisoimage', + version='', + release='', + ), + Package( + name='iproute2', + version='', + release='', + ), + Package( + name='iptables', + version='', + release='', + ), + Package( + name='salt-minion={}'.format(SALT_VERSION), + version='', + release='', + ), + Package( + name='kubectl={}'.format(K8S_VERSION), + version='', + release='', + ), + Package( + name='kubelet={}'.format(K8S_VERSION), + version='', + release='', + ), + Package( + name='kubernetes-cni', + version='', + release='', + ), + Package( + name='python-m2crypto', + version='', + release='', + ), + Package( + name='runc', + version='', + release='', + ), + Package( + name='socat', + version='', + release='', + ), + Package( + name='sosreport', + version='', + release='', + ), + Package( + name='util-linux', + version='', + release='', + ), + Package( + name='xfsprogs', + version='', + release='', + ), +) + # }}} diff --git a/packages/debian/download_packages.py b/packages/debian/download_packages.py index 26ef741002..43aa05eef8 100755 --- a/packages/debian/download_packages.py +++ b/packages/debian/download_packages.py @@ -176,17 +176,27 @@ def download_package(package: apt.package.Version) -> pathlib.Path: def main(packages: Sequence[str], env: Mapping[str, str]) -> None: """Download the packages specified on the command-line.""" + witness_file = pathlib.Path('/repositories/.witness') + try: + witness_file.unlink() + except FileNotFoundError: + pass add_external_repositories(env['SALT_VERSION']) apt_pkg.init() cache = apt.cache.Cache() to_download = {} + dest = pathlib.Path('/repositories') for package in packages: deps = get_package_deps(package, cache) to_download.update(deps) for pkg in to_download.values(): filepath = download_package(pkg) os.chown(filepath, int(env['TARGET_UID']), int(env['TARGET_GID'])) - + # TODO: need to be done by the build chain + for directory in dest.iterdir(): + os.chown(directory, int(env['TARGET_UID']), int(env['TARGET_GID'])) + witness_file.touch() + os.chown(witness_file, int(env['TARGET_UID']), int(env['TARGET_GID'])) if __name__ == '__main__': _, *PACKAGES = sys.argv