Merge branch 'improvement/1634-implement-download-packages' into q/2.4

buildchain/buildchain/packaging.py

@@ -54,6 +54,7 @@ def task_packaging() -> types.TaskDict:
             '_build_rpm_packages:*',
             '_build_rpm_repositories:*',
             '_build_deb_packages:*',
+            '_download_deb_packages',
         ],
     }
 
@@ -134,7 +135,7 @@ def clean() -> None:
         run_config=docker_command.RPM_BASE_CONFIG
     )
     return {
-        'title': utils.title_with_target1('GET PKGS'),
+        'title': utils.title_with_target1('GET RPM PKGS'),
         'actions': [dl_packages_callable],
         'targets': [constants.PKG_RPM_ROOT/'var'],
         'task_dep': [
@@ -148,6 +149,40 @@ def clean() -> None:
         'verbosity': 0,
     }
 
+
+def task__download_deb_packages() -> types.TaskDict:
+    """Download Debian packages locally."""
+    # TODO: Clean the repository
+    mounts = [
+        utils.bind_ro_mount(
+            source=constants.ROOT/'packages'/'debian'/'download_packages.py',
+            target=Path('/download_packages.py'),
+        ),
+        utils.bind_mount(
+            source=constants.REPO_DEB_ROOT,
+            target=Path('/repositories')
+        ),
+    ]
+    dl_packages_callable = docker_command.DockerRun(
+        command=['/download_packages.py', *DEB_TO_DOWNLOAD],
+        builder=DEB_BUILDER,
+        mounts=mounts,
+        environment={'SALT_VERSION': versions.SALT_VERSION},
+        run_config=docker_command.DEB_BASE_CONFIG
+    )
+    return {
+        'title': utils.title_with_target1('GET DEB PKGS'),
+        'actions': [dl_packages_callable],
+        'targets': [constants.REPO_DEB_ROOT/'.witness'],
+        'task_dep': [
+            '_package_mkdir_deb_root',
+            '_package_mkdir_deb_iso_root',
+            '_build_deb_container'
+        ],
+        'uptodate': [config_changed(_TO_DOWNLOAD_DEB_CONFIG)],
+    }
+
+
 def task__build_rpm_packages() -> Iterator[types.TaskDict]:
     """Build a RPM package."""
     for repo_pkgs in RPM_TO_BUILD.values():
@@ -274,26 +309,35 @@ def _deb_package(name: str, sources: Path) -> targets.DEBPackage:
 }
 
 _RPM_TO_BUILD_PKG_NAMES : List[str] = []
+_DEB_TO_BUILD_PKG_NAMES : List[str] = []
 
 for pkgs in RPM_TO_BUILD.values():
     for pkg in pkgs:
         _RPM_TO_BUILD_PKG_NAMES.append(pkg.name)
 
 # All packages not referenced in `RPM_TO_BUILD` but listed in
-# `versions.PACKAGES` are supposed to be downloaded.
+# `versions.RPM_PACKAGES` are supposed to be downloaded.
 RPM_TO_DOWNLOAD : FrozenSet[str] = frozenset(
     "{p.name}-{p.version}-{p.release}".format(p=package)
-    for package in versions.PACKAGES
+    for package in versions.RPM_PACKAGES
     if package.name not in _RPM_TO_BUILD_PKG_NAMES
 )
 
+
 # Store these versions in a dict to use with doit.tools.config_changed
 _TO_DOWNLOAD_CONFIG : Dict[str, str] = {
     pkg.name: "{p.version}-{p.release}".format(p=pkg)
-    for pkg in versions.PACKAGES
+    for pkg in versions.RPM_PACKAGES
     if pkg.name not in _RPM_TO_BUILD_PKG_NAMES
 }
 
+_TO_DOWNLOAD_DEB_CONFIG : Dict[str, str] = {
+    pkg.name: "{p.version}-{p.release}".format(p=pkg)
+    for pkg in versions.DEB_PACKAGES
+    if pkg.name not in _DEB_TO_BUILD_PKG_NAMES
+}
+
+
 SCALITY_RPM_REPOSITORY = targets.RPMRepository(
     basename='_build_rpm_repositories',
     name='scality',
@@ -356,5 +400,10 @@ def _deb_package(name: str, sources: Path) -> targets.DEBPackage:
     )
 }
 
+DEB_TO_DOWNLOAD : FrozenSet[str] = frozenset(
+        "{p.name}".format(p=package)
+        for package in versions.DEB_PACKAGES
+        if package.name not in DEB_TO_BUILD
+)
 
 __all__ = utils.export_only_tasks(__name__)

buildchain/buildchain/salt_tree.py

@@ -196,7 +196,7 @@ def _get_parts(self) -> Iterator[str]:
             'kubernetes': {'version': versions.K8S_VERSION},
             'packages': {
                 pkg.name: {'version': "{}-{}".format(pkg.version, pkg.release)}
-                for pkg in versions.PACKAGES
+                for pkg in versions.RPM_PACKAGES
             },
             'images': {
                 img.name: {'version': img.version}

buildchain/buildchain/versions.py

@@ -230,7 +230,7 @@ def _version_prefix(version: str, prefix: str = 'v') -> str:
 # }}}
 # Packages {{{
 
-PACKAGES = (
+RPM_PACKAGES = (
     # Remote packages
     Package(
         name='containerd',
@@ -360,6 +360,106 @@ def _version_prefix(version: str, prefix: str = 'v') -> str:
     ),
 )
 
-PACKAGES_MAP = {pkg.name: pkg for pkg in PACKAGES}
+PACKAGES_MAP = {pkg.name: pkg for pkg in RPM_PACKAGES}
+
+
+DEB_PACKAGES = (
+    Package(
+        name='containerd',
+        version='',
+        release='',
+    ),
+    Package(
+        name='cri-tools',
+        version='',
+        release='',
+    ),
+    Package(
+        name='coreutils',
+        version='',
+        release='',
+    ),
+    Package(
+        name='ebtables',
+        version='',
+        release='',
+    ),
+    Package(
+        name='ethtool',
+        version='',
+        release='',
+    ),
+    Package(
+        name='e2fsprogs',
+        version='',
+        release='',
+    ),
+    Package(
+        name='genisoimage',
+        version='',
+        release='',
+    ),
+    Package(
+        name='iproute2',
+        version='',
+        release='',
+    ),
+    Package(
+        name='iptables',
+        version='',
+        release='',
+    ),
+    Package(
+        name='salt-minion={}'.format(SALT_VERSION),
+        version='',
+        release='',
+    ),
+    Package(
+        name='kubectl={}'.format(K8S_VERSION),
+        version='',
+        release='',
+    ),
+    Package(
+        name='kubelet={}'.format(K8S_VERSION),
+        version='',
+        release='',
+    ),
+    Package(
+        name='kubernetes-cni',
+        version='',
+        release='',
+    ),
+    Package(
+        name='python-m2crypto',
+        version='',
+        release='',
+    ),
+    Package(
+        name='runc',
+        version='',
+        release='',
+    ),
+    Package(
+        name='socat',
+        version='',
+        release='',
+    ),
+    Package(
+        name='sosreport',
+        version='',
+        release='',
+    ),
+    Package(
+        name='util-linux',
+        version='',
+        release='',
+    ),
+    Package(
+        name='xfsprogs',
+        version='',
+        release='',
+    ),
+)
+
 
 # }}}

packages/debian/download_packages.py

@@ -176,17 +176,27 @@ def download_package(package: apt.package.Version) -> pathlib.Path:
 
 def main(packages: Sequence[str], env: Mapping[str, str]) -> None:
     """Download the packages specified on the command-line."""
+    witness_file = pathlib.Path('/repositories/.witness')
+    try:
+        witness_file.unlink()
+    except FileNotFoundError:
+        pass
     add_external_repositories(env['SALT_VERSION'])
     apt_pkg.init()
     cache = apt.cache.Cache()
     to_download = {}
+    dest = pathlib.Path('/repositories')
     for package in packages:
         deps = get_package_deps(package, cache)
         to_download.update(deps)
     for pkg in to_download.values():
         filepath = download_package(pkg)
         os.chown(filepath, int(env['TARGET_UID']), int(env['TARGET_GID']))
-
+    # TODO: need to be done by the build chain
+    for directory in dest.iterdir():
+        os.chown(directory, int(env['TARGET_UID']), int(env['TARGET_GID']))
+    witness_file.touch()
+    os.chown(witness_file, int(env['TARGET_UID']), int(env['TARGET_GID']))
 
 if __name__ == '__main__':
     _, *PACKAGES = sys.argv