Add run-integration-tests.sh script (#698)

* Add run-integration-tests.sh script - Runs integration tests - Uses aws-k8s-tester - Adapted from CSI test script, there may still be some stuff in here that won't apply * Add node ping test and make ginkgo verbose * Remove kubectl exec test code * Fix test runner script - Fix env vars - Fix Deprovision command - Split test and cluster config into separate /tmp directories - combine cluster functions into lib/cluster.sh
aws · Nov 12, 2019 · 3f3a2a9 · 3f3a2a9
1 parent 7eab704
commit 3f3a2a9
Show file tree

Hide file tree

Showing 6 changed files with 746 additions and 2 deletions.
diff --git a/go.mod b/go.mod
@@ -25,8 +25,8 @@ require (
 	github.com/json-iterator/go v1.1.5 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect
-	github.com/onsi/ginkgo v1.8.0 // indirect
-	github.com/onsi/gomega v1.5.0 // indirect
+	github.com/onsi/ginkgo v1.8.0
+	github.com/onsi/gomega v1.5.0
 	github.com/operator-framework/operator-sdk v0.0.7
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pkg/errors v0.8.0

diff --git a/scripts/lib/cluster.sh b/scripts/lib/cluster.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+function down-test-cluster() {
+    echo "Deleting cluster $CLUSTER_NAME"
+    $TESTER_PATH eks delete cluster --path $CLUSTER_CONFIG
+}
+
+function up-test-cluster() {
+    echo "Creating cluster $CLUSTER_NAME"
+    ssh-keygen -P cni-test -f $SSH_KEY_PATH
+
+    $TESTER_PATH eks create config --path $CLUSTER_CONFIG
+
+    AWS_K8S_TESTER_EKS_CLUSTER_NAME=$CLUSTER_NAME \
+        AWS_K8S_TESTER_EKS_KUBECONFIG_PATH=$KUBECONFIG_PATH \
+        AWS_K8S_TESTER_EKS_KUBERNETES_VERSION=${K8S_VERSION%.*} \
+        AWS_K8S_TESTER_EKS_ENABLE_WORKER_NODE_PRIVILEGED_PORT_ACCESS=true \
+        AWS_K8S_TESTER_EKS_WORKER_NODE_ASG_MIN=3 \
+        AWS_K8S_TESTER_EKS_WORKER_NODE_ASG_MAX=3 \
+        AWS_K8S_TESTER_EKS_WORKER_NODE_ASG_DESIRED_CAPACITY=3 \
+        AWS_K8S_TESTER_EKS_WORKER_NODE_PRIVATE_KEY_PATH=$SSH_KEY_PATH \
+        AWS_K8S_TESTER_EKS_WORKER_NODE_INSTANCE_TYPE=m3.xlarge \
+        AWS_K8S_TESTER_EKS_AWS_K8S_TESTER_PATH=$TESTER_PATH \
+        AWS_K8S_TESTER_EKS_AWS_IAM_AUTHENTICATOR_PATH=$AUTHENTICATOR_PATH \
+        AWS_K8S_TESTER_EKS_KUBECTL_PATH=$KUBECTL_PATH \
+        $TESTER_PATH eks create cluster --path $CLUSTER_CONFIG 1>&2
+}
diff --git a/scripts/run-integration-tests.sh b/scripts/run-integration-tests.sh
@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+DIR=$(cd "$(dirname "$0")"; pwd)
+source $DIR/lib/cluster.sh
+
+OS=$(go env GOOS)
+ARCH=amd64
+REGION=${AWS_REGION:-us-west-2}
+K8S_VERSION=${K8S_VERSION:-1.14.6}
+PROVISION=${PROVISION:-true}
+DEPROVISION=${DEPROVISION:-true}
+BUILD=${BUILD:-true}
+
+# test specific config, results location
+TEST_ID=${TEST_ID:-$RANDOM}
+TEST_DIR=/tmp/cni-test/$(date "+%Y%M%d%H%M%S")-$TEST_ID
+REPORT_DIR=${TEST_DIR}/report
+
+# test cluster config location
+# Pass in CLUSTER_ID to reuse a test cluster
+CLUSTER_ID=${CLUSTER_ID:-$RANDOM}
+CLUSTER_NAME=cni-test-$CLUSTER_ID
+TEST_CLUSTER_DIR=/tmp/cni-test/cluster-$CLUSTER_NAME
+CLUSTER_CONFIG=${CLUSTER_CONFIG:-${TEST_CLUSTER_DIR}/${CLUSTER_NAME}.yaml}
+SSH_KEY_PATH=${SSH_KEY_PATH:-${TEST_CLUSTER_DIR}/id_rsa}
+KUBECONFIG_PATH=${KUBECONFIG_PATH:-${TEST_CLUSTER_DIR}/kubeconfig}
+
+# shared binaries
+TESTER_DOWNLOAD_URL=https://github.com/aws/aws-k8s-tester/releases/download/v0.4.3/aws-k8s-tester-v0.4.3-$OS-$ARCH
+TESTER_PATH=${TESTER_PATH:-/tmp/aws-k8s-tester/aws-k8s-tester}
+AUTHENTICATOR_PATH=${AUTHENTICATOR_PATH:-/tmp/aws-k8s-tester/aws-iam-authenticator}
+KUBECTL_PATH=${KUBECTL_PATH:-/tmp/aws-k8s-tester/kubectl}
+
+# The version substituted in ./config/X/aws-k8s-cni.yaml
+CNI_TEMPLATE_VERSION=${CNI_TEMPLATE_VERSION:-v1.5}
+
+echo "Running $TEST_ID on $CLUSTER_NAME in $REGION"
+echo "+ Cluster config dir: $TEST_CLUSTER_DIR"
+echo "+ Result dir:         $TEST_DIR"
+echo "+ Tester:             $TESTER_PATH"
+echo "+ Kubeconfig:         $KUBECONFIG_PATH"
+echo "+ Node SSH key:       $SSH_KEY_PATH"
+echo "+ Cluster config:     $CLUSTER_CONFIG"
+echo ""
+
+mkdir -p $TEST_DIR
+mkdir -p $REPORT_DIR
+mkdir -p $TEST_CLUSTER_DIR
+
+# Download aws-k8s-tester if not yet
+if [[ ! -e $TESTER_PATH ]]; then
+  echo "Downloading aws-k8s-tester from $TESTER_DOWNLOAD_URL to $TESTER_PATH"
+  curl -L -X GET $TESTER_DOWNLOAD_URL -o $TESTER_PATH
+  chmod +x $TESTER_PATH
+fi
+
+if [[ "$PROVISION" = true ]]; then
+    up-test-cluster
+fi
+
+if [[ "$BUILD" = true ]]; then
+    # Push test image
+    eval $(aws ecr get-login --region $REGION --no-include-email)
+    AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
+    IMAGE_NAME="$AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/amazon/amazon-k8s-cni"
+    IMAGE_VERSION=$(git describe --tags --always --dirty)
+
+    make docker IMAGE=$IMAGE_NAME VERSION=$IMAGE_VERSION
+    docker push $IMAGE_NAME:$IMAGE_VERSION
+
+    echo "Using ./config/$CNI_TEMPLATE_VERSION/aws-k8s-cni.yaml as a template"
+    if [[ ! -f "./config/$CNI_TEMPLATE_VERSION/aws-k8s-cni.yaml" ]]; then
+        echo "./config/$CNI_TEMPLATE_VERSION/aws-k8s-cni.yaml DOES NOT exist. Set \$CNI_TEMPLATE_VERSION to an existing directory in ./config/"
+        exit
+    fi
+
+    sed -i'.bak' "s,602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon-k8s-cni,$IMAGE_NAME," ./config/$CNI_TEMPLATE_VERSION/aws-k8s-cni.yaml
+    sed -i'.bak' "s,v1.5.3,$IMAGE_VERSION," ./config/$CNI_TEMPLATE_VERSION/aws-k8s-cni.yaml
+fi
+
+echo "Deploying CNI"
+export KUBECONFIG=$KUBECONFIG_PATH
+kubectl apply -f ./config/$CNI_TEMPLATE_VERSION/aws-k8s-cni.yaml
+
+# Run the test
+pushd ./test/integration
+go test -v -timeout 0 ./... --kubeconfig=$KUBECONFIG --ginkgo.focus="\[cni-integration\]" --ginkgo.skip="\[Disruptive\]" \
+    --assets=./assets
+TEST_PASS=$?
+popd
+
+if [[ "$DEPROVISION" = true ]]; then
+    down-test-cluster
+fi
+
+if [[ $TEST_PASS -ne 0 ]]; then
+  exit 1
+fi
diff --git a/test/integration/go.mod b/test/integration/go.mod
@@ -0,0 +1,60 @@
+module github.com/aws/amazon-vpc-cni-k8s/test/integration
+
+go 1.12
+
+replace k8s.io/api => k8s.io/api v0.0.0-20190819141258-3544db3b9e44
+
+replace k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.0.0-20190819143637-0dbe462fe92d
+
+replace k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20190817020851-f2f3a405f61d
+
+replace k8s.io/apiserver => k8s.io/apiserver v0.0.0-20190819142446-92cc630367d0
+
+replace k8s.io/cli-runtime => k8s.io/cli-runtime v0.0.0-20190819144027-541433d7ce35
+
+replace k8s.io/client-go => k8s.io/client-go v0.0.0-20190819141724-e14f31a72a77
+
+replace k8s.io/cloud-provider => k8s.io/cloud-provider v0.0.0-20190819145148-d91c85d212d5
+
+replace k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.0.0-20190819145008-029dd04813af
+
+replace k8s.io/code-generator => k8s.io/code-generator v0.0.0-20190612205613-18da4a14b22b
+
+replace k8s.io/component-base => k8s.io/component-base v0.0.0-20190819141909-f0f7c184477d
+
+replace k8s.io/cri-api => k8s.io/cri-api v0.0.0-20190817025403-3ae76f584e79
+
+replace k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.0.0-20190819145328-4831a4ced492
+
+replace k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.0.0-20190819142756-13daafd3604f
+
+replace k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.0.0-20190819144832-f53437941eef
+
+replace k8s.io/kube-proxy => k8s.io/kube-proxy v0.0.0-20190819144346-2e47de1df0f0
+
+replace k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.0.0-20190819144657-d1a724e0828e
+
+replace k8s.io/kubectl => k8s.io/kubectl v0.0.0-20190602132728-7075c07e78bf
+
+replace k8s.io/kubelet => k8s.io/kubelet v0.0.0-20190819144524-827174bad5e8
+
+replace k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.0.0-20190819145509-592c9a46fd00
+
+replace k8s.io/metrics => k8s.io/metrics v0.0.0-20190819143841-305e1cef1ab1
+
+replace k8s.io/node-api => k8s.io/node-api v0.0.0-20190819145652-b61681edbd0a
+
+replace k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.0.0-20190819143045-c84c31c165c4
+
+replace k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.0.0-20190819144209-f9ca4b649af0
+
+replace k8s.io/sample-controller => k8s.io/sample-controller v0.0.0-20190819143301-7c475f5e1313
+
+require (
+	github.com/onsi/ginkgo v1.8.0
+	github.com/onsi/gomega v1.5.0
+	k8s.io/api v0.0.0
+	k8s.io/apimachinery v0.0.0
+	k8s.io/client-go v0.0.0
+	k8s.io/kubernetes v1.15.3
+)