From 100842786ce78818a848373c79a40b91b8335f4c Mon Sep 17 00:00:00 2001 From: Starnop Date: Tue, 4 Sep 2018 19:26:33 +0800 Subject: [PATCH] add node e2e test in CI Signed-off-by: Starnop --- .travis.yml | 11 +- Makefile | 8 +- hack/install/install_cni.sh | 72 +++++++++++ hack/install/install_critest.sh | 11 +- hack/install/install_etcd.sh | 50 ++++++++ hack/install/install_ginkgo.sh | 29 +++++ hack/testing/run_daemon_cri_e2e.sh | 142 +++++++++++++++++++++ hack/testing/run_daemon_cri_integration.sh | 53 +------- 8 files changed, 312 insertions(+), 64 deletions(-) create mode 100755 hack/install/install_cni.sh create mode 100755 hack/install/install_etcd.sh create mode 100755 hack/install/install_ginkgo.sh create mode 100755 hack/testing/run_daemon_cri_e2e.sh diff --git a/.travis.yml b/.travis.yml index cc091fafd..f28bc3d13 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,7 @@ env: - TEST_SUITE=integrationtest - TEST_SUITE=criv1alpha1test - TEST_SUITE=criv1alpha2test + - TEST_SUITE=nodee2etest script: | set -e # fast fail @@ -43,7 +44,7 @@ script: | sudo env "PATH=$PATH" make download-dependencies sudo env "PATH=$PATH" make cri-v1alpha1-test make coverage - else + elif [[ "${TEST_SUITE}" = "criv1alpha2test" ]]; then make build TEST_FLAGS= make build-daemon-integration sudo env "PATH=$PATH" make install @@ -51,6 +52,14 @@ script: | sudo env "PATH=$PATH" make download-dependencies sudo env "PATH=$PATH" make cri-v1alpha2-test make coverage + else + make build + TEST_FLAGS= make build-daemon-integration + sudo env "PATH=$PATH" make install + + sudo env "PATH=$PATH" make download-dependencies + sudo env "PATH=$PATH" make cri-e2e-test + make coverage fi after_success: diff --git a/Makefile b/Makefile index c679618f8..4e5eb491f 100644 --- a/Makefile +++ b/Makefile @@ -157,8 +157,14 @@ cri-v1alpha2-test: ## run v1 alpha2 cri-v1alpha2-test @mkdir -p coverage ./hack/testing/run_daemon_cri_integration.sh v1alpha2 +.PHONY: cri-e2e-test +cri-e2e-test: ## run cri-e2e-test + @echo $@ + @mkdir -p coverage + ./hack/testing/run_daemon_cri_e2e.sh v1alpha2 + .PHONY: test -test: unit-test integration-test cri-v1alpha1-test cri-v1alpha2-test ## run the unit-test, integration-test , cri-v1alpha1-test and cri-v1alpha2-test +test: unit-test integration-test cri-v1alpha1-test cri-v1alpha2-test cri-e2e-test ## run the unit-test, integration-test , cri-v1alpha1-test , cri-v1alpha2-test and cri-e2e-test .PHONY: coverage coverage: ## combine coverage after test diff --git a/hack/install/install_cni.sh b/hack/install/install_cni.sh new file mode 100755 index 000000000..024e3ff76 --- /dev/null +++ b/hack/install/install_cni.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# keep the first one only +GOPATH="${GOPATH%%:*}" + +# add bin folder into PATH. +export PATH="${GOPATH}/bin:${PATH}" + + +# cni::install_cni installs cni plugins. +cni::install_cni() { + echo "install cni..." + + local workdir pkg + + # for multiple GOPATHs, keep the first one only + pkg="github.com/containernetworking/plugins" + workdir="${GOPATH}/src/${pkg}" + + # downloads github.com/containernetworking/plugins + go get -u -d "${pkg}"/... + + # build and copy into /opt/cni/bin + "${workdir}"/build.sh + mkdir -p /etc/cni/net.d /opt/cni/bin + cp "${workdir}"/bin/* /opt/cni/bin + + # setup the config + sh -c 'cat >/etc/cni/net.d/10-mynet.conflist <<-EOF +{ + "cniVersion": "0.3.1", + "name": "mynet", + "plugins": [ + { + "type": "bridge", + "bridge": "cni0", + "isGateway": true, + "ipMasq": true, + "ipam": { + "type": "host-local", + "subnet": "10.30.0.0/16", + "routes": [ + { "dst": "0.0.0.0/0" } + ] + } + }, + { + "type": "portmap", + "capabilities": {"portMappings": true}, + "snat": true + } + ] +} +EOF' + + sh -c 'cat >/etc/cni/net.d/99-loopback.conf <<-EOF +{ + "cniVersion": "0.3.1", + "type": "loopback" +} +EOF' + + echo +} + +main() { + cni::install_cni +} + +main "$@" diff --git a/hack/install/install_critest.sh b/hack/install/install_critest.sh index c31f7d0e7..271f1c9ea 100755 --- a/hack/install/install_critest.sh +++ b/hack/install/install_critest.sh @@ -46,15 +46,7 @@ critest::install() { # critest::install_ginkgo installs ginkgo if missing. critest::install_ginkgo() { - local has_installed pkg - - pkg="github.com/onsi/ginkgo/ginkgo" - has_installed="$(command -v ginkgo || echo false)" - if [[ "${has_installed}" = "false" ]]; then - go get -u "${pkg}" - fi - - command -v ginkgo > /dev/null + hack/install/install_ginkgo.sh } main() { @@ -79,7 +71,6 @@ main() { critest::install command -v critest > /dev/null - command -v ginkgo > /dev/null } main "$@" diff --git a/hack/install/install_etcd.sh b/hack/install/install_etcd.sh new file mode 100755 index 000000000..6d64a30e3 --- /dev/null +++ b/hack/install/install_etcd.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ETCD_VERSION=v3.3.5 +ARCH=amd64 + +# keep the first one only +GOPATH="${GOPATH%%:*}" + +# add bin folder into PATH. +export PATH="${GOPATH}/bin:${PATH}" + +# etcd::check_version checks the command and the version. +etcd::check_version() { + local has_installed + + has_installed="$(command -v etcd || echo false)" + if [[ "${has_installed}" = "false" ]]; then + echo false + exit 0 + fi + + echo true +} + +# etcd::install downloads the package and build. +etcd::install() { + wget --quiet "https://github.com/coreos/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-${ARCH}.tar.gz" + tar -xf "etcd-${ETCD_VERSION}-linux-${ARCH}.tar.gz" -C "/usr/local" + rm etcd-${ETCD_VERSION}-linux-${ARCH}.tar.gz + export PATH="/usr/local/etcd-${ETCD_VERSION}-linux-${ARCH}:${PATH}" +} + +main() { + local has_installed + + has_installed="$(etcd::check_version)" + if [[ "${has_installed}" = "true" ]]; then + echo "etcd-${ETCD_VERSION} has been installed." + exit 0 + fi + + echo ">>>> install etcd-${ETCD_VERSION} <<<<" + etcd::install + + command -v etcd > /dev/null +} + +main "$@" diff --git a/hack/install/install_ginkgo.sh b/hack/install/install_ginkgo.sh new file mode 100755 index 000000000..7da83a901 --- /dev/null +++ b/hack/install/install_ginkgo.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# keep the first one only +GOPATH="${GOPATH%%:*}" + +# add bin folder into PATH. +export PATH="${GOPATH}/bin:${PATH}" + + +# ginkgo::install_ginkgo installs ginkgo if missing. +ginkgo::install_ginkgo() { + local has_installed pkg + + pkg="github.com/onsi/ginkgo/ginkgo" + has_installed="$(command -v ginkgo || echo false)" + if [[ "${has_installed}" = "false" ]]; then + go get -u "${pkg}" + fi + + command -v ginkgo > /dev/null +} + +main() { + ginkgo::install_ginkgo +} + +main "$@" diff --git a/hack/testing/run_daemon_cri_e2e.sh b/hack/testing/run_daemon_cri_e2e.sh new file mode 100755 index 000000000..550c5002f --- /dev/null +++ b/hack/testing/run_daemon_cri_e2e.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash + +set -euo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")" +source utils.sh + +cd ../../ +readonly REPO_BASE="$(pwd -P)" + +# keep the first one only +GOPATH="${GOPATH%%:*}" + +# add bin folder into PATH so that pouch-e2e is available. +export PATH="${REPO_BASE}/bin:${PATH}" + +# add bin folder into PATH. +export PATH="${GOPATH}/bin:${PATH}" + +# CRI_SKIP skips the test to skip. +DEFAULT_SKIP="\[Flaky\]|\[Slow\]|\[Serial\]" +DEFAULT_SKIP+="|querying\s\/stats\/summary" +DEFAULT_SKIP+="|should execute prestop exec hook properly" +DEFAULT_SKIP+="|should execute poststart exec hook properly" +DEFAULT_SKIP+="|should function for intra-pod communication: http*" +DEFAULT_SKIP+="|should function for intra-pod communication: udp*" +DEFAULT_SKIP+="|should project all components that make up the projection API*" +export SKIP=${SKIP:-${DEFAULT_SKIP}} + +# FOCUS focuses the test to run. +export FOCUS=${FOCUS:-} + +POUCH_SOCK="/var/run/pouchcri.sock" + +# tmplog_dir stores the background job log data +tmplog_dir="$(mktemp -d /tmp/e2e-daemon-cri-testing-XXXXX)" +pouchd_log="${tmplog_dir}/pouchd.log" +local_persist_log="${tmplog_dir}/local_persist.log" +trap 'rm -rf /tmp/e2e-daemon-cri-testing-*' EXIT + +# integration::install_cni installs cni plugins. +integration::install_cni() { + hack/install/install_cni.sh +} + +# integration::install_ginkgo installs ginkgo. +integration::install_ginkgo() { + hack/install/install_ginkgo.sh +} + +# integration::install_etcd installs etcd. +integration::install_etcd() { + hack/install/install_etcd.sh +} + +# integration::run_daemon_cri_test_e2e_cases runs CRI e2e test cases. +integration::run_daemon_cri_test_e2e_cases() { + local cri_runtime code KUBERNETES_VERSION + cri_runtime=$1 + + if [[ "${cri_runtime}" == "v1alpha1" ]]; then + KUBERNETES_VERSION="release-1.9" + else + KUBERNETES_VERSION="release-1.10" + fi + + KUBERNETES_REPO="github.com/kubernetes/kubernetes" + KUBERNETES_PATH="${GOPATH}/src/k8s.io/kubernetes" + if [ ! -d "${KUBERNETES_PATH}" ]; then + mkdir -p "${KUBERNETES_PATH}" + cd "${KUBERNETES_PATH}" + git clone https://${KUBERNETES_REPO} . + fi + cd "${KUBERNETES_PATH}" + git fetch --all + git checkout ${KUBERNETES_VERSION} + + echo "start pouch daemon cri-${cri_runtime} e2e test..." + set +e + + make test-e2e-node \ + RUNTIME=remote \ + CONTAINER_RUNTIME_ENDPOINT=unix://${POUCH_SOCK} \ + SKIP="${SKIP}" \ + FOCUS="${FOCUS}" \ + PARALLELISM=8 + + code=$? + + integration::stop_local_persist + integration::stop_pouchd + set -e + + if [[ "${code}" != "0" ]]; then + echo "failed to pass e2e cases!" + echo "there is daemon logs..." + cat "${pouchd_log}" + exit ${code} + fi + + # sleep for pouchd stop and got the coverage + sleep 5 +} + +integration::run_cri_e2e_test(){ + local cri_runtime cmd flags coverage_profile + cri_runtime=$1 + + # daemon cri integration coverage profile + coverage_profile="${REPO_BASE}/coverage/e2e_daemon_cri_${cri_runtime}_profile.out" + rm -rf "${coverage_profile}" + + cmd="pouchd-integration" + flags=" -test.coverprofile=${coverage_profile} DEVEL" + flags="${flags} --enable-cri --cri-version ${cri_runtime} --sandbox-image=gcr.io/google_containers/pause-amd64:3.0" + + + integration::stop_local_persist + integration::run_local_persist_background "${local_persist_log}" + integration::stop_pouchd + integration::run_pouchd_background "${cmd}" "${flags}" "${pouchd_log}" + + set +e; integration::ping_pouchd; code=$?; set -e + if [[ "${code}" != "0" ]]; then + echo "there is daemon logs..." + cat "${pouchd_log}" + exit ${code} + fi + integration::run_daemon_cri_test_e2e_cases "${cri_runtime}" +} + +main() { + local cri_runtime + cri_runtime=$1 + + integration::install_cni + integration::install_ginkgo + integration::install_etcd + integration::run_cri_e2e_test "${cri_runtime}" +} + +main "$@" diff --git a/hack/testing/run_daemon_cri_integration.sh b/hack/testing/run_daemon_cri_integration.sh index 5af0b9ff9..d440255ad 100755 --- a/hack/testing/run_daemon_cri_integration.sh +++ b/hack/testing/run_daemon_cri_integration.sh @@ -44,58 +44,7 @@ integration::install_critest() { # integration::install_cni installs cni plugins. integration::install_cni() { - echo "install cni..." - - local workdir pkg - - # for multiple GOPATHs, keep the first one only - pkg="github.com/containernetworking/plugins" - workdir="${GOPATH}/src/${pkg}" - - # downloads github.com/containernetworking/plugins - go get -u -d "${pkg}"/... - - # build and copy into /opt/cni/bin - "${workdir}"/build.sh - mkdir -p /etc/cni/net.d /opt/cni/bin - cp "${workdir}"/bin/* /opt/cni/bin - - # setup the config - sh -c 'cat >/etc/cni/net.d/10-mynet.conflist <<-EOF -{ - "cniVersion": "0.3.1", - "name": "mynet", - "plugins": [ - { - "type": "bridge", - "bridge": "cni0", - "isGateway": true, - "ipMasq": true, - "ipam": { - "type": "host-local", - "subnet": "10.30.0.0/16", - "routes": [ - { "dst": "0.0.0.0/0" } - ] - } - }, - { - "type": "portmap", - "capabilities": {"portMappings": true}, - "snat": true - } - ] -} -EOF' - - sh -c 'cat >/etc/cni/net.d/99-loopback.conf <<-EOF -{ - "cniVersion": "0.3.1", - "type": "loopback" -} -EOF' - - echo + hack/install/install_cni.sh }