Add script for running k8s conformance E2E #471
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,62 @@ | ||
| #!/bin/bash | ||
| # Copyright 2020 The Kubernetes Authors. | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| # Verify the required Environment Variables are present. | ||
| : "${AZURE_SUBSCRIPTION_ID:?Environment variable empty or not defined.}" | ||
| : "${AZURE_TENANT_ID:?Environment variable empty or not defined.}" | ||
| : "${AZURE_CLIENT_ID:?Environment variable empty or not defined.}" | ||
| : "${AZURE_CLIENT_SECRET:?Environment variable empty or not defined.}" | ||
| : "${REGISTRY:?Environment variable empty or not defined.}" | ||
|
|
||
| # Cluster settings. | ||
| export CLUSTER_NAME="${CLUSTER_NAME:-capz-test}" | ||
| export AZURE_VNET_NAME=${CLUSTER_NAME}-vnet | ||
|
|
||
| # Azure settings. | ||
| export AZURE_LOCATION="${AZURE_LOCATION:-southcentralus}" | ||
| export AZURE_RESOURCE_GROUP=${CLUSTER_NAME} | ||
| export AZURE_SUBSCRIPTION_ID_B64="$(echo -n "$AZURE_SUBSCRIPTION_ID" | base64 | tr -d '\n')" | ||
| export AZURE_TENANT_ID_B64="$(echo -n "$AZURE_TENANT_ID" | base64 | tr -d '\n')" | ||
| export AZURE_CLIENT_ID_B64="$(echo -n "$AZURE_CLIENT_ID" | base64 | tr -d '\n')" | ||
| export AZURE_CLIENT_SECRET_B64="$(echo -n "$AZURE_CLIENT_SECRET" | base64 | tr -d '\n')" | ||
|
|
||
| # Machine settings. | ||
| export CONTROL_PLANE_MACHINE_COUNT=${CONTROL_PLANE_MACHINE_COUNT:-3} | ||
| export AZURE_CONTROL_PLANE_MACHINE_TYPE="${CONTROL_PLANE_MACHINE_TYPE:-Standard_D2s_v3}" | ||
| export AZURE_NODE_MACHINE_TYPE="${NODE_MACHINE_TYPE:-Standard_D2s_v3}" | ||
| export WORKER_MACHINE_COUNT=${WORKER_MACHINE_COUNT:-2} | ||
| export KUBERNETES_VERSION="${KUBERNETES_VERSION:-v1.17.3}" | ||
|
|
||
| # Generate SSH key. | ||
| SSH_KEY_FILE=${SSH_KEY_FILE:-""} | ||
| if ! [ -n "$SSH_KEY_FILE" ]; then | ||
| SSH_KEY_FILE=.sshkey | ||
| rm -f "${SSH_KEY_FILE}" 2>/dev/null | ||
| ssh-keygen -t rsa -b 2048 -f "${SSH_KEY_FILE}" -N '' 1>/dev/null | ||
| echo "Machine SSH key generated in ${SSH_KEY_FILE}" | ||
| fi | ||
| export AZURE_SSH_PUBLIC_KEY=$(cat "${SSH_KEY_FILE}.pub" | base64 | tr -d '\r\n') | ||
|
|
||
| echo "================ DOCKER BUILD ===============" | ||
| PULL_POLICY=IfNotPresent make modules docker-build | ||
|
|
||
| echo "================ MAKE CLEAN ===============" | ||
| make clean | ||
|
|
||
| echo "================ KIND RESET ===============" | ||
| make kind-reset | ||
|
|
||
| echo "================ CREATE CLUSTER ===============" | ||
| make create-cluster |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,152 @@ | ||
| #!/bin/bash | ||
|
There was a problem hiding this comment. A lot of this script was taken directly or inspired from https://github.com/kubernetes-sigs/cluster-api-provider-aws/blob/master/hack/ci/e2e-conformance.sh |
||
|
|
||
| # Copyright 2020 The Kubernetes Authors. | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| ############################################################################### | ||
|
|
||
| # To run locally, set AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_SUBSCRIPTION_ID, AZURE_TENANT_ID | ||
|
|
||
| set -o nounset | ||
| set -o pipefail | ||
|
|
||
| REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. | ||
| cd "${REPO_ROOT}" || exit 1 | ||
|
|
||
| # shellcheck source=../hack/ensure-go.sh | ||
| source "${REPO_ROOT}/hack/ensure-go.sh" | ||
| # shellcheck source=../hack/ensure-kind.sh | ||
| source "${REPO_ROOT}/hack/ensure-kind.sh" | ||
| # shellcheck source=../hack/ensure-kubectl.sh | ||
| source "${REPO_ROOT}/hack/ensure-kubectl.sh" | ||
| # shellcheck source=../hack/ensure-kustomize.sh | ||
| source "${REPO_ROOT}/hack/ensure-kustomize.sh" | ||
|
|
||
| random-string() { | ||
| cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w ${1:-32} | head -n 1 | ||
| } | ||
|
|
||
| # generate manifests needed for creating the Azure cluster to run the tests | ||
| add_kustomize_patch() { | ||
| # Enable the bits to inject a script that can pull newer versions of kubernetes | ||
| if ! grep -i -wq "patchesStrategicMerge" "templates/kustomization.yaml"; then | ||
| echo "patchesStrategicMerge:" >> "templates/kustomization.yaml" | ||
| fi | ||
| if ! grep -i -wq "kustomizeversions" "templates/kustomization.yaml"; then | ||
| echo "- kustomizeversions.yaml" >> "templates/kustomization.yaml" | ||
| fi | ||
| } | ||
|
|
||
| # build Kubernetes E2E binaries | ||
| build_k8s() { | ||
| # possibly enable bazel build caching before building kubernetes | ||
| if [[ "${BAZEL_REMOTE_CACHE_ENABLED:-false}" == "true" ]]; then | ||
| create_bazel_cache_rcs.sh || true | ||
| fi | ||
|
|
||
| pushd "$(go env GOPATH)/src/k8s.io/kubernetes" | ||
|
|
||
| # make sure we have e2e requirements | ||
| bazel build //cmd/kubectl //test/e2e:e2e.test //vendor/github.com/onsi/ginkgo/ginkgo | ||
|
|
||
| # ensure the e2e script will find our binaries ... | ||
| mkdir -p "${PWD}/_output/bin/" | ||
| cp -f "${PWD}/bazel-bin/test/e2e/e2e.test" "${PWD}/_output/bin/e2e.test" | ||
| cp -f "${PWD}/bazel-bin/vendor/github.com/onsi/ginkgo/ginkgo/darwin_amd64_stripped/ginkgo" "${PWD}/_output/bin/ginkgo" | ||
| export KUBECTL_PATH="$(dirname "$(find "${PWD}/bazel-bin/" -name kubectl -type f)")/kubectl" | ||
| PATH="${KUBECTL_PATH}:${PATH}" | ||
| export PATH | ||
|
|
||
| # attempt to release some memory after building | ||
| sync || true | ||
| echo 1 > /proc/sys/vm/drop_caches || true | ||
|
|
||
| popd | ||
| } | ||
|
|
||
| create_cluster() { | ||
CecileRobertMichon marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| export CLUSTER_NAME="capz-conformance-$(head /dev/urandom | LC_ALL=C tr -dc a-z0-9 | head -c 6 ; echo '')" | ||
| export CI_VERSION=${CI_VERSION:-$(curl -sSL https://dl.k8s.io/ci/k8s-master.txt)} | ||
| export REGISTRY=conformance | ||
| ${REPO_ROOT}/hack/create-dev-cluster.sh | ||
| } | ||
|
|
||
| run_tests() { | ||
| # export the target cluster KUBECONFIG if not already set | ||
| export KUBECONFIG="${KUBECONFIG:-${PWD}/kubeconfig}" | ||
| # ginkgo regexes | ||
| SKIP="${SKIP:-}" | ||
| FOCUS="${FOCUS:-"\\[Conformance\\]"}" | ||
| # if we set PARALLEL=true, skip serial tests set --ginkgo-parallel | ||
| if [[ "${PARALLEL:-false}" == "true" ]]; then | ||
| export GINKGO_PARALLEL=y | ||
| if [[ -z "${SKIP}" ]]; then | ||
| SKIP="\\[Serial\\]" | ||
| else | ||
| SKIP="\\[Serial\\]|${SKIP}" | ||
| fi | ||
| fi | ||
|
|
||
| # get the number of worker nodes | ||
| NUM_NODES="$(kubectl get nodes --kubeconfig="$KUBECONFIG" \ | ||
| -o=jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.taints}{"\n"}{end}' \ | ||
| | grep -cv "node-role.kubernetes.io/master" )" | ||
|
|
||
| # wait for all the nodes to be ready | ||
| kubectl wait --for=condition=Ready node --kubeconfig="$KUBECONFIG" --all || true | ||
|
|
||
| ARTIFACTS="${ARTIFACTS:-${PWD}/_artifacts}" | ||
|
|
||
| # setting this env prevents ginkg e2e from trying to run provider setup | ||
| export KUBERNETES_CONFORMANCE_TEST="y" | ||
| # run the tests | ||
| (cd "$(go env GOPATH)/src/k8s.io/kubernetes" && ./hack/ginkgo-e2e.sh \ | ||
| '--provider=skeleton' "--num-nodes=${NUM_NODES}" \ | ||
| "--ginkgo.focus=${FOCUS}" "--ginkgo.skip=${SKIP}" \ | ||
| "--report-dir=${ARTIFACTS}" '--disable-log-dump=true') | ||
|
|
||
| unset KUBECONFIG | ||
| unset KUBERNETES_CONFORMANCE_TEST | ||
| } | ||
|
|
||
| # cleanup all resources we use | ||
| cleanup() { | ||
| timeout 600 kubectl \ | ||
| delete cluster "${CLUSTER_NAME}" || true | ||
| timeout 600 kubectl \ | ||
| wait --for=delete cluster/"${CLUSTER_NAME}" || true | ||
| make kind-reset || true | ||
| # clean up e2e.test symlink | ||
| (cd "$(go env GOPATH)/src/k8s.io/kubernetes" && rm -f _output/bin/e2e.test) || true | ||
| } | ||
|
|
||
| # create cluster | ||
| SKIP_CREATE_CLUSTER=${SKIP_CREATE_CLUSTER:-""} | ||
| if [[ -z "${SKIP_CREATE_CLUSTER}" ]]; then | ||
| add_kustomize_patch | ||
| create_cluster | ||
| fi | ||
|
|
||
| # build k8s binaries and run conformance tests | ||
| SKIP_TESTS=${SKIP_TESTS:-""} | ||
| if [[ -z "${SKIP_TESTS}" ]]; then | ||
| build_k8s | ||
| run_tests | ||
| fi | ||
|
|
||
| # cleanup | ||
| SKIP_CLEANUP=${SKIP_CLEANUP:-""} | ||
| if [[ -z "${SKIP_CLEANUP}" ]]; then | ||
| cleanup | ||
| fi | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
👍
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
thank you for this one :)