Skip to content

Commit

Permalink
Merge "vpp-hst: add VPP HostStack Test framework jobs"
Browse files Browse the repository at this point in the history
  • Loading branch information
vvalderrv authored and Gerrit Code Review committed May 31, 2024
2 parents 7302086 + 80a84dd commit cde8cd6
Show file tree
Hide file tree
Showing 18 changed files with 358 additions and 20 deletions.
4 changes: 2 additions & 2 deletions docker/scripts/build_executor_docker_image.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#! /bin/bash

# Copyright (c) 2021 Cisco and/or its affiliates.
# Copyright (c) 2024 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
Expand Down Expand Up @@ -105,7 +105,7 @@ docker_build_setup_ciman
docker_build_setup_vpp
docker_build_setup_csit
for executor_os_name in $os_names ; do
docker_from_image="$(echo $executor_os_name | sed -e 's/-/:/')"
docker_from_image="${executor_os_name/-/:}"
# Remove '-' and '.' from executor_os_name in Docker Hub repo name
os_name="${executor_os_name//-}"
repository="fdiotools/${EXECUTOR_CLASS}-${os_name//.}"
Expand Down
19 changes: 12 additions & 7 deletions docker/scripts/dbld_vpp_install_packages.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#! /bin/bash

# Copyright (c) 2021 Cisco and/or its affiliates.
# Copyright (c) 2024 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
Expand All @@ -15,26 +15,31 @@

set -euxo pipefail

export CIMAN_DOCKER_SCRIPTS=${CIMAN_DOCKER_SCRIPTS:-"$(dirname $BASH_SOURCE)"}
export CIMAN_DOCKER_SCRIPTS=${CIMAN_DOCKER_SCRIPTS:-"$(dirname "${BASH_SOURCE[0]}")"}
. "$CIMAN_DOCKER_SCRIPTS/lib_vpp.sh"
. "$CIMAN_DOCKER_SCRIPTS/lib_apt.sh"

must_be_run_in_docker_build

echo_log
if ! vpp_supported_executor_class "$FDIOTOOLS_EXECUTOR_CLASS" ; then
echo_log "VPP is not supported on executor class '$FDIOTOOLS_EXECUTOR_CLASS'. Skipping $(basename $0)..."
echo_log "VPP is not supported on executor class '$FDIOTOOLS_EXECUTOR_CLASS'. Skipping $(basename "$0")..."
exit 0
else
echo_log "Starting $(basename $0)"
echo_log "Starting $(basename "$0")"
fi

do_git_config vpp
for branch in ${VPP_BRANCHES[$OS_NAME]} ; do
do_git_branch "$branch"

# Install hs-test depndencies
if [ "$OS_NAME" = "ubuntu-22.04" ] && [ "$branch" = "master" ] ; then
install_hst_deps "$branch"
fi

# Install OS packages
make_vpp "install-dep" "$branch"
make_vpp "install-deps" "$branch"

# Download, build, and cache external deps packages
make_vpp "install-ext-deps" "$branch"
Expand All @@ -47,7 +52,7 @@ for branch in ${VPP_BRANCHES[$OS_NAME]} ; do
exit 1
fi
if [ -f "$vpp_ext_deps_pkg" ] ; then
cp -f $vpp_ext_deps_pkg $DOCKER_DOWNLOADS_DIR
cp -f "$vpp_ext_deps_pkg" $DOCKER_DOWNLOADS_DIR
else
echo "ERROR: Missing VPP external deps package: '$vpp_ext_deps_pkg'"
exit 1
Expand Down Expand Up @@ -76,4 +81,4 @@ for branch in ${VPP_BRANCHES[$OS_NAME]} ; do
esac
done

echo_log -e "Completed $(basename $0)!\n\n=========="
echo_log -e "Completed $(basename "$0")!\n\n=========="
52 changes: 48 additions & 4 deletions docker/scripts/lib_apt.sh
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# lib_apt.sh - Docker build script apt library.
# For import only.

# Copyright (c) 2023 Cisco and/or its affiliates.
# Copyright (c) 2024 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
Expand All @@ -20,6 +20,9 @@ if [ -n "$(alias lib_apt_imported 2> /dev/null)" ] ; then
fi
alias lib_apt_imported=true

DIND_FROM_IMAGE="cruizba/ubuntu-dind:latest"
HST_FROM_IMAGE="ubuntu:22.04"

export CIMAN_DOCKER_SCRIPTS=${CIMAN_DOCKER_SCRIPTS:-"$(dirname $BASH_SOURCE)"}
. "$CIMAN_DOCKER_SCRIPTS/lib_common.sh"
. "$CIMAN_DOCKER_SCRIPTS/lib_csit.sh"
Expand Down Expand Up @@ -163,6 +166,7 @@ RUN wget https://releases.hashicorp.com/terraform/1.7.3/terraform_1.7.3_linux_$d
RUN apt-get update -qq \\
&& dbld_vpp_install_packages.sh \\
&& dbld_csit_install_packages.sh \\
&& apt-get install -y pkg-config \\
&& rm -r /var/lib/apt/lists/*
EOF
}
Expand All @@ -172,19 +176,44 @@ generate_apt_dockerfile_clean() {
# Clean up copy-in build tree
RUN dbld_dump_build_logs.sh \\
&& apt-get -y autoremove \\
&& rm -rf "/tmp/*" "$DOCKER_BUILD_FILES_DIR" "/root/.ccache"
EOF
}

generate_apt_dockerfile_install_golang() {
local GO_VERSION="$1"

cat <<EOF >>"$DOCKERFILE"
# Install golang for HostStack Test (HST) jobs
#
ENV GOPATH /go
ENV GOROOT /usr/local/go
ENV PATH \$GOPATH/bin:/usr/local/go/bin:\$PATH
RUN rm -rf /usr/local/go /usr/bin/go \\
&& wget -P /tmp "https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz" \\
&& tar -C /usr/local -xzf "/tmp/go${GO_VERSION}.linux-amd64.tar.gz" \\
&& rm "/tmp/go${GO_VERSION}.linux-amd64.tar.gz" \\
&& ln -s /usr/local/go/bin/go /usr/bin/go \\
&& echo -n "\nGOPATH=\$GOPATH\nGOROOT=\$GOROOT" | tee -a /etc/environment \\
&& mkdir -p "\$GOPATH/src" "\$GOPATH/bin" && chmod -R 777 "\$GOPATH"
EOF
}

# Generate 'builder' class apt dockerfile
builder_generate_apt_dockerfile() {
local executor_class="$1"
local executor_os_name="$2"
local executor_image="$3"
local install_golang="$4"
local vpp_install_skip_sysctl_envvar="";

generate_apt_dockerfile_common $executor_class $executor_image
generate_apt_dockerfile_common "$executor_class" "$executor_image"
csit_builder_generate_docker_build_files
if [ "$install_golang" = "true" ] ; then
generate_apt_dockerfile_install_golang "1.21.9"
fi
cat <<EOF >>"$DOCKERFILE"
# Install LF-IT requirements
Expand Down Expand Up @@ -336,6 +365,14 @@ generate_apt_dockerfile() {
local executor_os_name="$2"
local from_image="$3"
local executor_image="$4"
local install_golang="false"
local is_dind_image="false"

if [ "$from_image" = "$HST_FROM_IMAGE" ] ; then
from_image="$DIND_FROM_IMAGE"
install_golang="true"
is_dind_image="true"
fi

cat <<EOF >"$DOCKERIGNOREFILE"
**/__pycache__
Expand All @@ -347,6 +384,13 @@ LABEL Description="FD.io CI '$executor_class' executor docker image for $executo
LABEL Vendor="fd.io"
LABEL Version="$DOCKER_TAG"
EOF
${executor_class}_generate_apt_dockerfile "$executor_class" \
"$executor_os_name" "$executor_image"
if [ "$is_dind_image" = "true" ] ; then
cat <<EOF >>"$DOCKERFILE"
# Remove extraneous environment variable(s) created in "$from_image"
ENV DEBUG=
EOF
fi

"${executor_class}"_generate_apt_dockerfile "$executor_class" \
"$executor_os_name" "$executor_image" "$install_golang"
}
25 changes: 19 additions & 6 deletions docker/scripts/lib_vpp.sh
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# lib_vpp.sh - Docker build script VPP library.
# For import only.

# Copyright (c) 2023 Cisco and/or its affiliates.
# Copyright (c) 2024 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
Expand All @@ -20,8 +20,8 @@ if [ -n "$(alias lib_vpp_imported 2> /dev/null)" ] ; then
fi
alias lib_vpp_imported=true

export CIMAN_DOCKER_SCRIPTS=${CIMAN_DOCKER_SCRIPTS:-"$(dirname $BASH_SOURCE)"}
. $CIMAN_DOCKER_SCRIPTS/lib_common.sh
export CIMAN_DOCKER_SCRIPTS=${CIMAN_DOCKER_SCRIPTS:-"$(dirname "${BASH_SOURCE[0]}")"}
. "$CIMAN_DOCKER_SCRIPTS"/lib_common.sh


VPP_SUPPORTED_EXECUTOR_CLASSES="builder"
Expand All @@ -32,10 +32,23 @@ vpp_supported_executor_class() {
return 0
}

install_hst_deps() {
local branch=${1:-"master"}
local branchname=${branch/\//_}
local hst_dir="./extras/hs-test"
local bld_log="$DOCKER_BUILD_LOG_DIR"
bld_log="${bld_log}/$FDIOTOOLS_IMAGENAME-$branchname"
bld_log="${bld_log}-install_hst_deps_bld.log"

if [ -d "$hst_dir" ] ; then
make -C "$hst_dir" install-deps 2>&1 | tee -a "$bld_log"
fi
}

make_vpp() {
local target=$1
local branch=${2:-"master"}
local branchname="$(echo $branch | sed -e 's,/,_,')"
local branchname=${branch/\//_}
local bld_log="$DOCKER_BUILD_LOG_DIR"
bld_log="${bld_log}/$FDIOTOOLS_IMAGENAME-$branchname"
bld_log="${bld_log}-make_vpp_${target}-bld.log"
Expand All @@ -48,15 +61,15 @@ make_vpp() {
git clean -qfdx
description="'make UNATTENDED=yes $target' in $(pwd) ($branch)"
echo_log -e " Starting $description..."
make UNATTENDED=yes $target 2>&1 | tee -a "$bld_log"
make UNATTENDED=yes "$target" 2>&1 | tee -a "$bld_log"
git checkout -q -- .
echo_log " Completed $description!"
}

make_vpp_test() {
local target=$1
local branch=${2:-"master"}
local branchname="$(echo $branch | sed -e 's,/,_,')"
local branchname=${branch/\//_}
local bld_log="$DOCKER_BUILD_LOG_DIR"
bld_log="${bld_log}/$FDIOTOOLS_IMAGENAME-$branchname"
bld_log="${bld_log}-make_vpp_test_${target}-bld.log"
Expand Down
2 changes: 1 addition & 1 deletion jenkins-config/clouds/nomad/FDIONOMAD/defaults.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
---
templates:
- idleTerminationInMinutes: 10
- idleTerminationInMinutes: 1
numExecutors: 1
remoteFs: "/w"
reusable: false
Expand Down
Empty file modified jjb/scripts/vpp/api-checkstyle.sh
100644 → 100755
Empty file.
Empty file modified jjb/scripts/vpp/build.sh
100644 → 100755
Empty file.
Empty file modified jjb/scripts/vpp/commitmsg.sh
100644 → 100755
Empty file.
Empty file modified jjb/scripts/vpp/copy_archives.sh
100644 → 100755
Empty file.
Empty file modified jjb/scripts/vpp/cov-build.sh
100644 → 100755
Empty file.
Empty file modified jjb/scripts/vpp/csit-bisect.sh
100644 → 100755
Empty file.
Empty file modified jjb/scripts/vpp/debug-build.sh
100644 → 100755
Empty file.
62 changes: 62 additions & 0 deletions jjb/scripts/vpp/debug-hst.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
#!/bin/bash

# Copyright (c) 2024 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

echo "---> jjb/scripts/vpp/debug-hst.sh"

set -euxo pipefail

line="*************************************************************************"
OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
OS_ARCH=$(uname -m)
DRYRUN="${DRYRUN:-}"
BUILD_RESULT="SUCCESSFULLY COMPLETED"
BUILD_ERROR=""
RETVAL="0"
HST_DIR="./extras/hs-test"

vpp_make_build_debug() {
if ! make UNATTENDED=yes install-dep ; then
BUILD_ERROR="FAILED 'make install-dep'"
return
fi
if ! make UNATTENDED=yes install-ext-deps ; then
BUILD_ERROR="FAILED 'make install-ext-deps'"
return
fi
if ! make UNATTENDED=yes build ; then
BUILD_ERROR="FAILED 'make build'"
return
fi
}

hst_test_debug() {
if ! make VERBOSE=true VPPSRC="$(pwd)" -C "$HST_DIR" build-debug test ; then
BUILD_ERROR="FAILED 'make -C $HST_DIR test' (debug)"
return
fi
}

if [ "${DRYRUN,,}" != "true" ] ; then
vpp_make_build_debug
hst_test_debug
fi
if [ -n "$BUILD_ERROR" ] ; then
BUILD_RESULT="$BUILD_ERROR"
RETVAL="1"
fi
echo -e "\n$line\n* VPP ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^}" \
"BUILD $BUILD_RESULT\n$line\n"
exit $RETVAL
Empty file modified jjb/scripts/vpp/dpdk-rdma-version-mismatch.sh
100644 → 100755
Empty file.
Empty file modified jjb/scripts/vpp/gcc-build.sh
100644 → 100755
Empty file.
66 changes: 66 additions & 0 deletions jjb/scripts/vpp/hst.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
#!/bin/bash

# Copyright (c) 2024 Cisco and/or its affiliates.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

echo "---> jjb/scripts/vpp/hst.sh"

set -euxo pipefail

line="*************************************************************************"
OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
OS_VERSION_ID=$(grep '^VERSION_ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
OS_ARCH=$(uname -m)
DRYRUN="${DRYRUN:-}"
BUILD_RESULT="SUCCESSFULLY COMPLETED"
BUILD_ERROR=""
RETVAL="0"
HST_DIR="./extras/hs-test"

vpp_make_build() {
if ! make UNATTENDED=yes install-deps ; then
BUILD_ERROR="FAILED 'make install-dep'"
return
fi
if ! make UNATTENDED=yes install-ext-deps ; then
BUILD_ERROR="FAILED 'make install-ext-deps'"
return
fi
if ! make build-release; then
BUILD_ERROR="FAILED 'make build-release'"
return
fi
if ! make -C "$HST_DIR" install-deps build ; then
BUILD_ERROR="FAILED 'make -C $HST_DIR install-deps build'"
return
fi
}

hst_test() {
if ! make VERBOSE=true VPPSRC="$(pwd)" -C "$HST_DIR" build test ; then
BUILD_ERROR="FAILED 'make -C $HST_DIR test'"
return
fi
}

if [ "${DRYRUN,,}" != "true" ] ; then
vpp_make_build
hst_test
fi
if [ -n "$BUILD_ERROR" ] ; then
BUILD_RESULT="$BUILD_ERROR"
RETVAL="1"
fi
echo -e "\n$line\n* VPP ${OS_ID^^}-${OS_VERSION_ID}-${OS_ARCH^^}" \
"BUILD $BUILD_RESULT\n$line\n"
exit $RETVAL
Empty file modified jjb/scripts/vpp/test-checkstyle.sh
100644 → 100755
Empty file.
Loading

0 comments on commit cde8cd6

Please sign in to comment.