Skip to content

Commit

Permalink
versions: introduce common packages and extras/overrides
Browse files Browse the repository at this point in the history
The support of an additional OS requires a 'compatibility' layer for
package names and versions.

We define a common core of packages whose names do not diverge across OS
families and whose versions are either latest or fixed across OS families.

We introduce overrides in package versions, such that a package for a specific
version may override the package referenced in the common core.

We unpin most packages in the RPM listing, where we only really need to pin
K8s core components as well as some of the 'unstable' Python APIs we rely
on.
  • Loading branch information
jbertran committed Sep 18, 2019
1 parent 81e5493 commit eb2afe7
Show file tree
Hide file tree
Showing 4 changed files with 113 additions and 195 deletions.
5 changes: 4 additions & 1 deletion .pylint-dict
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ buildrepo
buildrpm
buildsrpm
cli
cni
cmd
cp
checksum
Expand All @@ -22,7 +23,9 @@ gofmt
filename
init
io
Kubernetes
kubernetes
kubectl
kubelet
linter
metadata
mkdir
Expand Down
2 changes: 1 addition & 1 deletion buildchain/buildchain/packaging.py
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,7 @@ def task__build_rpm_repositories() -> Iterator[types.TaskDict]:
# Packages to build, per repository.
def _rpm_package(name: str, sources: List[Path]) -> targets.RPMPackage:
try:
pkg_info = versions.PACKAGES_MAP[name]
pkg_info = versions.RPM_PACKAGES_MAP[name]
except KeyError:
raise ValueError(
'Missing version for package "{}"'.format(name)
Expand Down
2 changes: 1 addition & 1 deletion buildchain/buildchain/salt_tree.py
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ def _get_parts(self) -> Iterator[str]:
'kubernetes': {'version': versions.K8S_VERSION},
'packages': {
pkg.name: {'version': pkg.full_version}
for pkg in versions.PACKAGES
for pkg in versions.RPM_PACKAGES
},
'images': {
img.name: {'version': img.version}
Expand Down
299 changes: 107 additions & 192 deletions buildchain/buildchain/versions.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

from collections import namedtuple
from pathlib import Path
from typing import cast, Optional, Tuple
from typing import cast, Dict, Optional, Tuple


Image = namedtuple('Image', ('name', 'version', 'digest'))
Expand Down Expand Up @@ -243,7 +243,8 @@ def __init__(
self,
name: str,
version: Optional[str] = None,
release: Optional[str] = None
release: Optional[str] = None,
override: Optional[str] = None
):
"""Initializes a package version.
Expand All @@ -255,10 +256,12 @@ def __init__(
self._name = name
self._version = version
self._release = release
self._override = override

name = property(operator.attrgetter('_name'))
version = property(operator.attrgetter('_version'))
release = property(operator.attrgetter('_release'))
override = property(operator.attrgetter('_override'))

@property
def full_version(self) -> Optional[str]:
Expand All @@ -285,201 +288,113 @@ def deb_full_name(self) -> str:
return cast(str, self.name)


RPM_PACKAGES = (
# Remote packages
PackageVersion(
name='containerd',
version='1.2.4',
release='1.el7',
),
PackageVersion(
name='cri-tools',
version='1.13.0',
release='0',
),
PackageVersion(
name='container-selinux',
version='2.99',
release='1.el7_6',
),
PackageVersion(
name='coreutils',
version='8.22',
release='23.el7',
),
PackageVersion(
name='ebtables',
version='2.0.10',
release='16.el7',
),
PackageVersion(
name='ethtool',
version='4.8',
release='9.el7',
),
PackageVersion(
name='e2fsprogs',
version='1.42.9',
release='13.el7',
),
PackageVersion(
name='genisoimage',
version='1.1.11',
release='25.el7',
),
PackageVersion(
name='iproute',
version='4.11.0',
release='14.el7_6.2',
),
PackageVersion(
name='iptables',
version='1.4.21',
release='28.el7',
),
PackageVersion(
name='kubectl',
version=K8S_VERSION,
release='0',
),
PackageVersion(
name='kubelet',
version=K8S_VERSION,
release='0',
),
PackageVersion(
name='kubernetes-cni',
version='0.7.5',
release='0',
),
PackageVersion(
name='m2crypto',
version='0.31.0',
release='3.el7',
),
PackageVersion(
name='python2-kubernetes',
version='8.0.1',
release='1.el7',
),
PackageVersion(
name='runc',
version='1.0.0',
release='59.dev.git2abd837.el7.centos',
),
PackageVersion(
name='salt-minion',
version=SALT_VERSION,
release='1.el7',
),
PackageVersion(
name='skopeo',
version='0.1.35',
release='2.git404c5bd.el7.centos',
),
PackageVersion(
name='socat',
version='1.7.3.2',
release='2.el7',
),
PackageVersion(
name='sos',
version='3.6',
release='17.el7.centos',
),
PackageVersion(
name='util-linux',
version='2.23.2',
release='59.el7_6.1',
),
PackageVersion(
name='xfsprogs',
version='4.5.0',
release='18.el7',
),
PackageVersion(
name='yum-plugin-versionlock',
version='1.1.31',
release='50.el7',
),
# Local packages
PackageVersion(
name='metalk8s-sosreport',
version=SHORT_VERSION,
release='1.el7',
# The authoritative list of packages required.
#
# Common packages are packages for which we need not care about OS-specific
# divergences.
#
# In this case, either:
# * the _latest_ version is good enough, and will be the one
# selected by the package managers (so far: apt and yum).
# * we have strict version requirements that span OS families, and the
# version schemes _and_ package names do not diverge
#
# Strict version requirements are notably:
# * kubelet and kubectl which _make_ the K8s version of the cluster
# * salt-minion which _makes_ the Salt version of the cluster
#
# These common packages may be overridden by OS-specific packages if package
# names or version conventions diverge.
#
# Packages that we build ourselves require a version and release as part of
# their build process.
PACKAGES: Dict[str, Tuple[PackageVersion, ...]] = {
'common': (
# Pinned packages
PackageVersion(name='kubectl', version=K8S_VERSION),
PackageVersion(name='kubelet', version=K8S_VERSION),
# Latest packages
PackageVersion(name='containerd'),
PackageVersion(name='coreutils'),
PackageVersion(name='cri-tools'),
PackageVersion(name='e2fsprogs'),
PackageVersion(name='ebtables'),
PackageVersion(name='ethtool'),
PackageVersion(name='genisoimage'),
PackageVersion(name='iproute'),
PackageVersion(name='iptables'),
PackageVersion(name='kubernetes-cni'),
PackageVersion(name='m2crypto'),
PackageVersion(name='runc'),
PackageVersion(name='salt-minion', version=SALT_VERSION),
PackageVersion(name='socat'),
PackageVersion(name='sos'), # TODO download built package dependencies
PackageVersion(name='util-linux'),
PackageVersion(name='xfsprogs'),
),
'redhat': (
PackageVersion(
name='calico-cni-plugin',
version=CALICO_VERSION,
release='1.el7'
),
PackageVersion(name='container-selinux'), # TODO #1710
PackageVersion(
name='metalk8s-sosreport',
version=SHORT_VERSION,
release='1.el7'
),
PackageVersion(name='yum-plugin-versionlock'),
),
PackageVersion(
name='calico-cni-plugin',
version=CALICO_VERSION,
release='1.el7',
'debian': (
PackageVersion(
name='calico-cni-plugin',
version=CALICO_VERSION,
release='0'
),
PackageVersion(name='iproute2', override='iproute'),
PackageVersion(
name='metalk8s-sosreport',
version=SHORT_VERSION,
release='0'
),
PackageVersion(name='python-m2crypto', override='m2crypto'),
PackageVersion(name='sosreport', override='sos'),
),
)
}

PACKAGES_MAP = {pkg.name: pkg for pkg in RPM_PACKAGES}

def _list_pkgs_for_os_family(os_family: str) -> Tuple[PackageVersion, ...]:
"""List downloaded packages for a given OS family.
DEB_PACKAGES = (
PackageVersion(
name='containerd',
),
PackageVersion(
name='cri-tools',
),
PackageVersion(
name='coreutils',
),
PackageVersion(
name='ebtables',
),
PackageVersion(
name='ethtool',
),
PackageVersion(
name='e2fsprogs',
),
PackageVersion(
name='genisoimage',
),
PackageVersion(
name='iproute2',
),
PackageVersion(
name='iptables',
),
PackageVersion(
name='salt-minion',
version=SALT_VERSION,
),
PackageVersion(
name='kubectl',
version=K8S_VERSION,
),
PackageVersion(
name='kubelet',
version=K8S_VERSION,
),
PackageVersion(
name='kubernetes-cni',
),
PackageVersion(
name='python-m2crypto',
),
PackageVersion(
name='runc',
),
PackageVersion(
name='socat',
),
PackageVersion(
name='sosreport',
),
PackageVersion(
name='util-linux',
),
PackageVersion(
name='xfsprogs',
),
)
Arguments:
os_family: OS_family for which to list packages
"""
common_pkgs = PACKAGES['common']
os_family_pkgs = PACKAGES.get(os_family)

if os_family_pkgs is None:
raise Exception('No packages for OS family: {}'.format(os_family))

os_override_names = [
pkg.override for pkg in os_family_pkgs
if pkg.override is not None
]

overridden = filter(
lambda item: item.name not in os_override_names,
common_pkgs
)

return tuple(overridden) + os_family_pkgs


RPM_PACKAGES = _list_pkgs_for_os_family('redhat')

RPM_PACKAGES_MAP = {pkg.name: pkg for pkg in RPM_PACKAGES}

DEB_PACKAGES = _list_pkgs_for_os_family('debian')

DEB_PACKAGES_MAP = {pkg.name: pkg for pkg in DEB_PACKAGES}

# }}}

0 comments on commit eb2afe7

Please sign in to comment.