Skip to content

Commit

Permalink
Merge pull request #694 from wongma7/hacke2e
Browse files Browse the repository at this point in the history
Refactor e2e testing scripts to be more reusable and use them instead…
  • Loading branch information
k8s-ci-robot authored Jan 13, 2021
2 parents 842fb1b + fb0e9ae commit 1d66536
Show file tree
Hide file tree
Showing 16 changed files with 375 additions and 626 deletions.
29 changes: 19 additions & 10 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -73,22 +73,31 @@ test-sanity:
#go test -v ./tests/sanity/...
echo "succeed"

bin/k8s-e2e-tester: | bin
go get github.com/aws/aws-k8s-tester/e2e/tester/cmd/k8s-e2e-tester@master

.PHONY: test-e2e-single-az
test-e2e-single-az: bin/k8s-e2e-tester
TESTCONFIG=./tester/single-az-config.yaml ${GOBIN}/k8s-e2e-tester
test-e2e-single-az:
AWS_REGION=us-west-2 \
AWS_AVAILABILITY_ZONES=us-west-2a \
TEST_PATH=./tests/e2e/... \
GINKGO_FOCUS="\[ebs-csi-e2e\] \[single-az\]" \
GINKGO_SKIP="\"sc1\"|\"st1\"" \
./hack/e2e/run.sh

.PHONY: test-e2e-multi-az
test-e2e-multi-az: bin/k8s-e2e-tester
TESTCONFIG=./tester/multi-az-config.yaml ${GOBIN}/k8s-e2e-tester
test-e2e-multi-az:
AWS_REGION=us-west-2 \
AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b,us-west-2c \
TEST_PATH=./tests/e2e/... \
GINKGO_FOCUS="\[ebs-csi-e2e\] \[multi-az\]" \
./hack/e2e/run.sh

.PHONY: test-e2e-migration
test-e2e-migration:
AWS_REGION=us-west-2 AWS_AVAILABILITY_ZONES=us-west-2a GINKGO_FOCUS="\[ebs-csi-migration\]" ./hack/run-e2e-test
# TODO: enable migration test to use new framework
#TESTCONFIG=./tester/migration-test-config.yaml go run tester/cmd/main.go
AWS_REGION=us-west-2 \
AWS_AVAILABILITY_ZONES=us-west-2a \
TEST_PATH=./tests/e2e-migration/... \
GINKGO_FOCUS="\[ebs-csi-migration\]" \
EBS_CHECK_MIGRATION=true \
./hack/e2e/run.sh

.PHONY: image-release
image-release:
Expand Down
1 change: 1 addition & 0 deletions hack/e2e/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
/csi-test-artifacts
26 changes: 26 additions & 0 deletions hack/e2e/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
# Usage

run.sh will build and push a driver image, create a kops cluster, helm install the driver pointing to the built image, run ginkgo tests, then clean everything up.

See below for an example.

KOPS_STATE_FILE is an S3 bucket you have write access to.

TEST_ID is a token used for idempotency.

For more details, see the script itself.

For more examples, see the top-level Makefile.

```
TEST_PATH=./tests/e2e-migration/... \
EBS_CHECK_MIGRATION=true \
TEST_ID=18512 \
CLEAN=false \
KOPS_STATE_FILE=s3://mattwon \
AWS_REGION=us-west-2 \
AWS_AVAILABILITY_ZONES=us-west-2a \
GINKGO_FOCUS=Dynamic.\*xfs.\*should.store.data \
GINKGO_NODES=1 \
./hack/e2e/run.sh
```
61 changes: 61 additions & 0 deletions hack/e2e/ebs.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
#!/bin/bash

set -uo pipefail

BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
source "${BASE_DIR}"/util.sh

function ebs_check_migration() {
loudecho "Checking migration"
# There should have been no calls to the in-tree driver kubernetes.io/aws-ebs but many calls to ebs.csi.aws.com
# Find the controller-manager log and read its metrics to verify
NODE=$(kubectl get node -l kubernetes.io/role=master -o json | jq -r ".items[].metadata.name")
kubectl port-forward kube-controller-manager-"${NODE}" 10252:10252 -n kube-system &

# Ensure port forwarding succeeded
n=0
until [ "$n" -ge 30 ]; do
set +e
HEALTHZ=$(curl -s 127.0.0.1:10252/healthz)
set -e
if [[ ${HEALTHZ} == "ok" ]]; then
loudecho "Port forwarding succeeded"
break
else
loudecho "Port forwarding is not yet ready"
fi
n=$((n + 1))
sleep 1
done
if [[ "$n" -eq 30 ]]; then
loudecho "Timed out waiting for port forward"
for PROC in $(jobs -p); do
kill "${PROC}"
done
return 1
fi

set +e
curl 127.0.0.1:10252/metrics -s | grep -a 'volume_operation_total_seconds_bucket{operation_name="provision",plugin_name="ebs.csi.aws.com"'
CSI_CALLED=${PIPESTATUS[1]}
set -e

set +e
curl 127.0.0.1:10252/metrics -s | grep -a 'volume_operation_total_seconds_bucket{operation_name="provision",plugin_name="kubernetes.io/aws-ebs"'
INTREE_CALLED=${PIPESTATUS[1]}
set -e

for PROC in $(jobs -p); do
kill "${PROC}"
done

loudecho "CSI_CALLED: ${CSI_CALLED}"
loudecho "INTREE_CALLED: ${INTREE_CALLED}"

# if CSI was called and In-tree was not called, return 0/true/success
if [ "${CSI_CALLED}" == 0 ] && [ "${INTREE_CALLED}" == 1 ]; then
echo 0
else
echo 1
fi
}
24 changes: 24 additions & 0 deletions hack/e2e/ecr.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
#!/bin/bash

set -uo pipefail

BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
source "${BASE_DIR}"/util.sh

function ecr_build_and_push() {
REGION=${1}
AWS_ACCOUNT_ID=${2}
IMAGE_NAME=${3}
IMAGE_TAG=${4}
set +e
if docker images | grep "${IMAGE_NAME}" | grep "${IMAGE_TAG}"; then
set -e
loudecho "Assuming ${IMAGE_NAME}:${IMAGE_TAG} has been built and pushed"
else
set -e
loudecho "Building and pushing test driver image to ${IMAGE_NAME}:${IMAGE_TAG}"
aws ecr get-login-password --region "${REGION}" | docker login --username AWS --password-stdin "${AWS_ACCOUNT_ID}".dkr.ecr."${REGION}".amazonaws.com
docker build -t "${IMAGE_NAME}":"${IMAGE_TAG}" .
docker push "${IMAGE_NAME}":"${IMAGE_TAG}"
fi
}
2 changes: 1 addition & 1 deletion hack/utils/helm.sh → hack/e2e/helm.sh
100755 → 100644
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

set -uo pipefail

helm::install() {
function helm_install() {
INSTALL_PATH=${1}
if [[ ! -e ${INSTALL_PATH}/helm ]]; then
curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
Expand Down
71 changes: 71 additions & 0 deletions hack/e2e/kops.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
#!/bin/bash

set -uo pipefail

OS_ARCH=$(go env GOOS)-amd64

BASE_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
source "${BASE_DIR}"/util.sh

function kops_install() {
INSTALL_PATH=${1}
KOPS_VERSION=${2}
if [[ ! -e ${INSTALL_PATH}/kops ]]; then
KOPS_DOWNLOAD_URL=https://github.com/kubernetes/kops/releases/download/v${KOPS_VERSION}/kops-${OS_ARCH}
curl -L -X GET "${KOPS_DOWNLOAD_URL}" -o "${INSTALL_PATH}"/kops
chmod +x "${INSTALL_PATH}"/kops
fi
}

function kops_create_cluster() {
SSH_KEY_PATH=${1}
KOPS_STATE_FILE=${2}
CLUSTER_NAME=${3}
KOPS_BIN=${4}
ZONES=${5}
INSTANCE_TYPE=${6}
K8S_VERSION=${7}
TEST_DIR=${8}
KOPS_FEATURE_GATES_FILE=${10}
KOPS_ADDITIONAL_POLICIES_FILE=${11}

loudecho "Generating SSH key $SSH_KEY_PATH"
if [[ ! -e ${SSH_KEY_PATH} ]]; then
ssh-keygen -P csi-e2e -f "${SSH_KEY_PATH}"
fi

set +e
if ${KOPS_BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}"; then
set -e
loudecho "Updating cluster $CLUSTER_NAME"
else
set -e
loudecho "Creating cluster $CLUSTER_NAME"
${KOPS_BIN} create cluster --state "${KOPS_STATE_FILE}" \
--zones "${ZONES}" \
--node-count=3 \
--node-size="${INSTANCE_TYPE}" \
--kubernetes-version="${K8S_VERSION}" \
--ssh-public-key="${SSH_KEY_PATH}".pub \
"${CLUSTER_NAME}"
fi

CLUSTER_YAML_PATH=${TEST_DIR}/${CLUSTER_NAME}.yaml
${KOPS_BIN} get cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" -o yaml > "${CLUSTER_YAML_PATH}"
cat "${KOPS_FEATURE_GATES_FILE}" >> "${CLUSTER_YAML_PATH}"
cat "${KOPS_ADDITIONAL_POLICIES_FILE}" >> "${CLUSTER_YAML_PATH}"
${KOPS_BIN} replace --state "${KOPS_STATE_FILE}" -f "${CLUSTER_YAML_PATH}"
${KOPS_BIN} update cluster --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --yes

loudecho "Validating cluster $CLUSTER_NAME"
${KOPS_BIN} validate cluster --state "${KOPS_STATE_FILE}" --wait 10m
return $?
}

function kops_delete_cluster() {
KOPS_BIN=${1}
CLUSTER_NAME=${2}
KOPS_STATE_FILE=${3}
loudecho "Deleting cluster ${CLUSTER_NAME}"
${KOPS_BIN} delete cluster --name "${CLUSTER_NAME}" --state "${KOPS_STATE_FILE}" --yes
}
Loading

0 comments on commit 1d66536

Please sign in to comment.