Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update to F36 CI VM Images + Testing netavark/aardvark-dns #13376

Merged
merged 6 commits into from
Apr 27, 2022
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 15 additions & 60 deletions .cirrus.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,6 @@ env:
####
# Name of the ultimate destination branch for this CI run, PR or post-merge.
DEST_BRANCH: "main"
# Netavark branch to use when TEST_ENVIRON=host-netavark
NETAVARK_BRANCH: "main"
# Aardvark branch to use
AARDVARK_BRANCH: "main"
# Overrides default location (/tmp/cirrus) for repo clone
GOPATH: &gopath "/var/tmp/go"
GOBIN: "${GOPATH}/bin"
Expand All @@ -24,21 +20,16 @@ env:
# Runner statistics log file path/name
STATS_LOGFILE_SFX: 'runner_stats.log'
STATS_LOGFILE: '$GOSRC/${CIRRUS_TASK_NAME}-${STATS_LOGFILE_SFX}'
# Netavark/aardvark location/options when TEST_ENVIRON=host-netavark
NETAVARK_URL: "https://api.cirrus-ci.com/v1/artifact/github/containers/netavark/success/binary.zip?branch=${NETAVARK_BRANCH}"
NETAVARK_DEBUG: 0 # set non-zero to use the debug-mode binary
AARDVARK_URL: "https://api.cirrus-ci.com/v1/artifact/github/containers/aardvark-dns/success/binary.zip?branch=${AARDVARK_BRANCH}"
AARDVARK_DEBUG: 0 # set non-zero to use the debug-mode binary

####
#### Cache-image names to test with (double-quotes around names are critical)
####
FEDORA_NAME: "fedora-35"
PRIOR_FEDORA_NAME: "fedora-34"
FEDORA_NAME: "fedora-36"
PRIOR_FEDORA_NAME: "fedora-35"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not for this PR, but it would be nice to be able to figure out programmatically what these values should be rather than having to hard code them. That said, I don't have a good suggestion on how to pull that off.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This file is the source of definitions for automation, so it really can't be done using imperial logic. Cirrus-CI does have support for using a scripting-language instead of YAML, but looking up the values would be complex at best, and error-prone at worst. So hard-coding the values is the least-worst solution that makes the most sense at the moment.

UBUNTU_NAME: "ubuntu-2110"

# Google-cloud VM Images
IMAGE_SUFFIX: "c4831699639992320"
IMAGE_SUFFIX: "c4955393725038592"
FEDORA_CACHE_IMAGE_NAME: "fedora-${IMAGE_SUFFIX}"
PRIOR_FEDORA_CACHE_IMAGE_NAME: "prior-fedora-${IMAGE_SUFFIX}"
UBUNTU_CACHE_IMAGE_NAME: "ubuntu-${IMAGE_SUFFIX}"
Expand All @@ -53,7 +44,7 @@ env:
#### N/B: Required ALL of these are set for every single task.
####
TEST_FLAVOR: # int, sys, ext_svc, validate, automation, etc.
TEST_ENVIRON: host # 'host', 'host-netavark', or 'container'
TEST_ENVIRON: host # 'host', or 'container'
PODBIN_NAME: podman # 'podman' or 'remote'
PRIV_NAME: root # 'root' or 'rootless'
DISTRO_NV: # any {PRIOR_,}{FEDORA,UBUNTU}_NAME value
Expand Down Expand Up @@ -161,11 +152,11 @@ build_task:
CTR_FQIN: ${FEDORA_CONTAINER_FQIN}
# ID for re-use of build output
_BUILD_CACHE_HANDLE: ${FEDORA_NAME}-build-${CIRRUS_BUILD_ID}
#- env: &priorfedora_envvars
# DISTRO_NV: ${PRIOR_FEDORA_NAME}
# VM_IMAGE_NAME: ${PRIOR_FEDORA_CACHE_IMAGE_NAME}
# CTR_FQIN: ${PRIOR_FEDORA_CONTAINER_FQIN}
# _BUILD_CACHE_HANDLE: ${PRIOR_FEDORA_NAME}-build-${CIRRUS_BUILD_ID}
- env: &priorfedora_envvars
DISTRO_NV: ${PRIOR_FEDORA_NAME}
VM_IMAGE_NAME: ${PRIOR_FEDORA_CACHE_IMAGE_NAME}
CTR_FQIN: ${PRIOR_FEDORA_CONTAINER_FQIN}
_BUILD_CACHE_HANDLE: ${PRIOR_FEDORA_NAME}-build-${CIRRUS_BUILD_ID}
- env: &ubuntu_envvars
DISTRO_NV: ${UBUNTU_NAME}
VM_IMAGE_NAME: ${UBUNTU_CACHE_IMAGE_NAME}
Expand Down Expand Up @@ -394,7 +385,7 @@ unit_test_task:
- validate
matrix:
- env: *stdenvars
#- env: *priorfedora_envvars
- env: *priorfedora_envvars
- env: *ubuntu_envvars
# Special-case: Rootless on latest Fedora (standard) VM
- name: "Rootless unit on $DISTRO_NV"
Expand Down Expand Up @@ -515,11 +506,11 @@ container_integration_test_task:
_BUILD_CACHE_HANDLE: ${FEDORA_NAME}-build-${CIRRUS_BUILD_ID}
VM_IMAGE_NAME: ${FEDORA_CACHE_IMAGE_NAME}
CTR_FQIN: ${FEDORA_CONTAINER_FQIN}
#- env:
# DISTRO_NV: ${PRIOR_FEDORA_NAME}
# _BUILD_CACHE_HANDLE: ${PRIOR_FEDORA_NAME}-build-${CIRRUS_BUILD_ID}
# VM_IMAGE_NAME: ${PRIOR_FEDORA_CACHE_IMAGE_NAME}
# CTR_FQIN: ${PRIOR_FEDORA_CONTAINER_FQIN}
- env:
DISTRO_NV: ${PRIOR_FEDORA_NAME}
_BUILD_CACHE_HANDLE: ${PRIOR_FEDORA_NAME}-build-${CIRRUS_BUILD_ID}
VM_IMAGE_NAME: ${PRIOR_FEDORA_CACHE_IMAGE_NAME}
CTR_FQIN: ${PRIOR_FEDORA_CONTAINER_FQIN}
gce_instance: *standardvm
timeout_in: 90m
env:
Expand Down Expand Up @@ -553,41 +544,6 @@ rootless_integration_test_task:
always: *int_logs_artifacts


# Run various scenarios using upstream netavark/aardvark-dns binaries
netavark_task:
name: "Netavark $TEST_FLAVOR $PODBIN_NAME $PRIV_NAME"
alias: netavark
only_if: *not_build
skip: *branches_and_tags
depends_on:
- unit_test
gce_instance: *standardvm
matrix:
- env: &nenv
DISTRO_NV: ${FEDORA_NAME}
_BUILD_CACHE_HANDLE: ${FEDORA_NAME}-build-${CIRRUS_BUILD_ID}
VM_IMAGE_NAME: ${FEDORA_CACHE_IMAGE_NAME}
CTR_FQIN: ${FEDORA_CONTAINER_FQIN}
TEST_FLAVOR: int
TEST_ENVIRON: host-netavark
- env:
<<: *nenv
TEST_FLAVOR: int
PRIV_NAME: rootless
- env:
<<: *nenv
TEST_FLAVOR: sys
- env:
<<: *nenv
TEST_FLAVOR: sys
PRIV_NAME: rootless
clone_script: *noop # Comes from cache
gopath_cache: *ro_gopath_cache
setup_script: *setup
main_script: *main
always: *int_logs_artifacts


# Always run subsequent to integration tests. While parallelism is lost
# with runtime, debugging system-test failures can be more challenging
# for some golang developers. Otherwise the following tasks run across
Expand Down Expand Up @@ -841,7 +797,6 @@ success_task:
- remote_integration_test
- container_integration_test
- rootless_integration_test
- netavark
- local_system_test
- remote_system_test
- rootless_system_test
Expand Down
49 changes: 34 additions & 15 deletions contrib/cirrus/lib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ EPOCH_TEST_COMMIT="$CIRRUS_BASE_SHA"
# testing operations on all platforms and versions. This is necessary
# to avoid needlessly passing through global/system values across
# contexts, such as host->container or root->rootless user
PASSTHROUGH_ENV_RE='(^CI.*)|(^CIRRUS)|(^DISTRO_NV)|(^GOPATH)|(^GOCACHE)|(^GOSRC)|(^SCRIPT_BASE)|(CGROUP_MANAGER)|(OCI_RUNTIME)|(^TEST.*)|(^PODBIN_NAME)|(^PRIV_NAME)|(^ALT_NAME)|(^ROOTLESS_USER)|(SKIP_USERNS)|(.*_NAME)|(.*_FQIN)'
PASSTHROUGH_ENV_RE='(^CI.*)|(^CIRRUS)|(^DISTRO_NV)|(^GOPATH)|(^GOCACHE)|(^GOSRC)|(^SCRIPT_BASE)|(CGROUP_MANAGER)|(OCI_RUNTIME)|(^TEST.*)|(^PODBIN_NAME)|(^PRIV_NAME)|(^ALT_NAME)|(^ROOTLESS_USER)|(SKIP_USERNS)|(.*_NAME)|(.*_FQIN)|(NETWORK_BACKEND)'
# Unsafe env. vars for display
SECRET_ENV_RE='(ACCOUNT)|(GC[EP]..+)|(SSH)|(PASSWORD)|(TOKEN)'

Expand Down Expand Up @@ -216,20 +216,39 @@ setup_rootless() {
install_test_configs() {
msg "Installing ./test/registries.conf system-wide."
install -v -D -m 644 ./test/registries.conf /etc/containers/
if [[ "$TEST_ENVIRON" =~ netavark ]]; then
# belt-and-suspenders: any pre-existing CNI config. will spoil
# default use tof netavark (when both are installed).
rm -rf /etc/cni/net.d/*
else
echo "Installing cni config, policy and registry config"
req_env_vars GOSRC SCRIPT_BASE
cd $GOSRC || exit 1
install -v -D -m 644 ./cni/87-podman-bridge.conflist /etc/cni/net.d/
# This config must always sort last in the list of networks (podman picks first one
# as the default). This config prevents allocation of network address space used
# by default in google cloud. https://cloud.google.com/vpc/docs/vpc#ip-ranges
install -v -D -m 644 $SCRIPT_BASE/99-do-not-use-google-subnets.conflist /etc/cni/net.d/
fi
}

use_cni() {
msg "Unsetting NETWORK_BACKEND for all subsequent environments."
echo "export -n NETWORK_BACKEND" >> /etc/ci_environment
echo "unset NETWORK_BACKEND" >> /etc/ci_environment
export -n NETWORK_BACKEND
unset NETWORK_BACKEND
msg "Installing default CNI configuration"
cd $GOSRC || exit 1
rm -rvf /etc/cni/net.d
mkdir -p /etc/cni/net.d
install -v -D -m 644 ./cni/87-podman-bridge.conflist \
/etc/cni/net.d/
# This config must always sort last in the list of networks (podman picks
# first one as the default). This config prevents allocation of network
# address space used by default in google cloud.
# https://cloud.google.com/vpc/docs/vpc#ip-ranges
install -v -D -m 644 $SCRIPT_BASE/99-do-not-use-google-subnets.conflist \
/etc/cni/net.d/
}

use_netavark() {
msg "Forcing NETWORK_BACKEND=netavark for all subsequent environments."
echo "NETWORK_BACKEND=netavark" >> /etc/ci_environment
export NETWORK_BACKEND=netavark # needed for install_test_configs()
msg "Removing any/all CNI configuration"
rm -rvf /etc/cni/net.d/*

# TODO: Remove this when netavark/aardvark-dns development slows down
warn "Updating netavark/aardvark-dns to avoid frequent VM image rebuilds"
# N/B: This is coming from updates-testing repo in F36
lilto dnf update -y netavark aardvark-dns
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should these be installed from updates-testing?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The comment suggests that they are, but the string updates.test does not appear anywhere else (relevantly) in this repo, nor in containers-automation, nor in containers-image, so I have this question too

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So updates-testing repo is enabled.

}

# Remove all files provided by the distro version of podman.
Expand Down
43 changes: 16 additions & 27 deletions contrib/cirrus/logcollector.sh
Original file line number Diff line number Diff line change
Expand Up @@ -40,32 +40,34 @@ case $1 in
packages)
# These names are common to Fedora and Ubuntu
PKG_NAMES=(\
conmon \
containernetworking-plugins \
containers-common \
criu \
crun \
golang \
podman \
runc \
skopeo \
slirp4netns \
conmon
containernetworking-plugins
containers-common
criu
crun
golang
podman
runc
skopeo
slirp4netns
)
case $OS_RELEASE_ID in
fedora)
cat /etc/fedora-release
PKG_LST_CMD='rpm -q --qf=%{N}-%{V}-%{R}-%{ARCH}\n'
PKG_NAMES+=(\
container-selinux \
libseccomp \
aardvark
container-selinux
libseccomp
netavark
)
;;
ubuntu)
cat /etc/issue
PKG_LST_CMD='dpkg-query --show --showformat=${Package}-${Version}-${Architecture}\n'
PKG_NAMES+=(\
cri-o-runc \
libseccomp2 \
cri-o-runc
libseccomp2
)
;;
*) bad_os_id_ver ;;
Expand All @@ -74,19 +76,6 @@ case $1 in
echo "Cgroups: " $(stat -f -c %T /sys/fs/cgroup)
# Any not-present packages will be listed as such
$PKG_LST_CMD "${PKG_NAMES[@]}" | sort -u

# TODO: Remove this once netavark/aardvark-dns packages are used
if [[ "$TEST_ENVIRON" =~ netavark ]]; then
_npath=/usr/local/libexec/podman/
for name in netavark aardvark-dns; do
echo "$name binary details:"
if [[ -r "$_npath/${name}.info" ]]; then
cat "$_npath/${name}.info"
else
echo "WARNING: $_npath/${name}.info not found."
fi
done
fi
;;
time)
# Assumed to be empty/undefined outside of Cirrus-CI (.cirrus.yml)
Expand Down
2 changes: 1 addition & 1 deletion contrib/cirrus/runner.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ set -eo pipefail
# most notably:
#
# PODBIN_NAME : "podman" (i.e. local) or "remote"
# TEST_ENVIRON : 'host', 'host-netavark', or 'container'; desired environment in which to run
# TEST_ENVIRON : 'host', or 'container'; desired environment in which to run
# CONTAINER : 1 if *currently* running inside a container, 0 if host
#

Expand Down
69 changes: 14 additions & 55 deletions contrib/cirrus/setup_environment.sh
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,6 @@ done

cp hack/podman-registry /bin

# Make sure cni network plugins directory exists
mkdir -p /etc/cni/net.d

# Some test operations & checks require a git "identity"
_gc='git config --file /root/.gitconfig'
$_gc user.email "[email protected]"
Expand Down Expand Up @@ -84,13 +81,6 @@ case "$CG_FS_TYPE" in
else
echo "OCI_RUNTIME=runc" >> /etc/ci_environment
fi

# As a general policy CGv1 + runc should coincide with the "older"
# VM Images in CI. Verify this is the case.
if [[ -n "$VM_IMAGE_NAME" ]] && [[ ! "$VM_IMAGE_NAME" =~ prior ]]
then
die "Most recent distro. version should never run with CGv1"
fi
fi
;;
cgroup2fs)
Expand All @@ -99,13 +89,6 @@ case "$CG_FS_TYPE" in
# which uses runc as the default.
warn "Forcing testing with crun instead of runc"
echo "OCI_RUNTIME=crun" >> /etc/ci_environment

# As a general policy CGv2 + crun should coincide with the "newer"
# VM Images in CI. Verify this is the case.
if [[ -n "$VM_IMAGE_NAME" ]] && [[ "$VM_IMAGE_NAME" =~ prior ]]
then
die "Least recent distro. version should never run with CGv2"
fi
fi
;;
*) die_unknown CG_FS_TYPE
Expand All @@ -130,14 +113,27 @@ case "$OS_RELEASE_ID" in
msg "Enabling container_manage_cgroup"
setsebool container_manage_cgroup true
fi

# For release 36 and later, netavark/aardvark is the default
# networking stack for podman. All previous releases only have
# CNI networking available. Upgrading from one to the other is
# not supported at this time. Support execution of the upgrade
# tests in F36 and later, by disabling Netavark and enabling CNI.
if [[ "$OS_RELEASE_VER" -ge 36 ]] && \
cevich marked this conversation as resolved.
Show resolved Hide resolved
[[ "$TEST_FLAVOR" != "upgrade_test" ]];
then
use_netavark
else # Fedora < 36, or upgrade testing.
use_cni
fi
;;
*) die_unknown OS_RELEASE_ID
esac

# Required to be defined by caller: The environment where primary testing happens
# shellcheck disable=SC2154
case "$TEST_ENVIRON" in
host*)
host)
# The e2e tests wrongly guess `--cgroup-manager` option
# shellcheck disable=SC2154
if [[ "$CG_FS_TYPE" == "cgroup2fs" ]] || [[ "$PRIV_NAME" == "root" ]]
Expand All @@ -148,43 +144,6 @@ case "$TEST_ENVIRON" in
warn "Forcing CGROUP_MANAGER=cgroupfs"
echo "CGROUP_MANAGER=cgroupfs" >> /etc/ci_environment
fi
# TODO: For the foreseeable future, need to support running tests
# with and without the latest netavark/aardvark. Once they're more
# stable and widely supported in Fedora, they can be pre-installed
# from its RPM at VM image build-time.
if [[ "$TEST_ENVIRON" =~ netavark ]]; then
for info in "netavark $NETAVARK_BRANCH $NETAVARK_URL $NETAVARK_DEBUG" \
"aardvark-dns $AARDVARK_BRANCH $AARDVARK_URL $AARDVARK_DEBUG"; do

read _name _branch _url _debug <<<"$info"
req_env_vars _name _branch _url _debug
msg "Downloading latest $_name from upstream branch '$_branch'"
# Use identifiable archive filename in of a get_ci_env.sh environment
curl --fail --location -o /tmp/$_name.zip "$_url"

# Needs to be in a specific location
# ref: https://github.com/containers/common/blob/main/pkg/config/config_linux.go#L39
_pdir=/usr/local/libexec/podman
mkdir -p $_pdir
cd $_pdir
msg "$PWD"
unzip /tmp/$_name.zip
if ((_debug)); then
warn "Using debug $_name binary"
mv $_name.debug $_name
else
rm $_name.debug
fi
chmod 0755 $_pdir/$_name
cd -
done

restorecon -F -v $_nvdir
# This is critical, it signals to all tests that netavark
# use is expected.
msg "Forcing NETWORK_BACKEND=netavark in all subsequent environments."
echo "NETWORK_BACKEND=netavark" >> /etc/ci_environment
fi
;;
container)
if ((CONTAINER==0)); then # not yet inside a container
Expand Down