diff --git a/docs/development.md b/docs/development.md index 5aaba208bdb..426653ffed4 100644 --- a/docs/development.md +++ b/docs/development.md @@ -140,7 +140,7 @@ To build a kind cluster and start Tilt, just run: ```shell make tilt-up ``` -By default, the Cluster API components deployed by Tilt have experimental features turned off. +By default, the Cluster API components deployed by Tilt have experimental features turned off. If you would like to enable these features, add `extra_args` as specified in [The Cluster API Book](https://cluster-api.sigs.k8s.io/developer/tilt.html#create-a-tilt-settingsjson-file). Once your kind management cluster is up and running, you can [deploy a workload cluster](#deploying-a-workload-cluster). @@ -350,26 +350,33 @@ You can optionally set `AZURE_SSH_PUBLIC_KEY_FILE` to use your own SSH key. To run Conformance locally, set `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET`, `AZURE_SUBSCRIPTION_ID`, `AZURE_TENANT_ID` and run: ```bash -./scripts/ci-conformance.sh +./scripts/ci-entrypoint.sh ``` You can optionally set the following variables: -| Variable | Description | -| --------------------------- | -------------------------------------------------------------------------------------------------------------- | -| `AZURE_SSH_PUBLIC_KEY_FILE` | Use your own SSH key. | -| `SKIP_CREATE_CLUSTER` | Skip cluster creation. | -| `SKIP_TESTS` | Skip running Kubernetes E2E tests. | +| Variable | Description | +|-----------------------------|---------------------------------------------------------------------------------------------------------------| +| `AZURE_SSH_PUBLIC_KEY_FILE` | Use your own SSH key. | +| `SKIP_CREATE_CLUSTER` | Skip cluster creation. | +| `SKIP_UPSTREAM_E2E_TESTS` | Skip running upstream Kubernetes E2E tests. | | `SKIP_CLEANUP` | Skip deleting the cluster after the tests finish running. | -| `KUBECONFIG` | Provide your existing cluster kubeconfig filepath. If no kubeconfig is provided, `./kubeconfig` will be used. | -| `SKIP` | Regexp for test cases to skip. | -| `FOCUS` | Regexp for which test cases to run. | +| `KUBECONFIG` | Provide your existing cluster kubeconfig filepath. If no kubeconfig is provided, `./kubeconfig` will be used. | +| `SKIP` | Regexp for test cases to skip. | +| `FOCUS` | Regexp for which test cases to run. | | `PARALLEL` | Skip serial tests and set --ginkgo-parallel. | -| `USE_CI_ARTIFACTS` | Use a CI version of Kubernetes, ie. not a released version (eg. `v1.19.0-alpha.1.426+0926c9c47677e9`) | -| `CI_VERSION` | Provide a custom CI version of Kubernetes. By default, the latest master commit will be used. | +| `USE_CI_ARTIFACTS` | Use a CI version of Kubernetes, ie. not a released version (eg. `v1.19.0-alpha.1.426+0926c9c47677e9`) | +| `CI_VERSION` | Provide a custom CI version of Kubernetes. By default, the latest master commit will be used. | You can also customize the configuration of the CAPZ cluster (assuming that `SKIP_CREATE_CLUSTER` is not set). See [Customizing the cluster deployment](#customizing-the-cluster-deployment) for more details. +In addition to upstream E2E, you can append custom commands to `./scripts/ci-entrypoint.sh` to run E2E from other projects against a CAPZ cluster: + +```bash +export SKIP_UPSTREAM_E2E_TESTS="false" +./scripts/ci-entrypoint.sh bash -c "cd ${GOPATH}/src/github.com/my-org/my-project && make e2e" +``` + [go]: https://golang.org/doc/install diff --git a/scripts/ci-entrypoint.sh b/scripts/ci-entrypoint.sh new file mode 100755 index 00000000000..ea8bbc6e02d --- /dev/null +++ b/scripts/ci-entrypoint.sh @@ -0,0 +1,177 @@ +#!/bin/bash + +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +############################################################################### + +# To run locally, set AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_SUBSCRIPTION_ID, AZURE_TENANT_ID + +set -o errexit +set -o nounset +set -o pipefail + +REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. +cd "${REPO_ROOT}" || exit 1 + +# shellcheck source=../hack/ensure-go.sh +source "${REPO_ROOT}/hack/ensure-go.sh" +# shellcheck source=../hack/ensure-kind.sh +source "${REPO_ROOT}/hack/ensure-kind.sh" +# shellcheck source=../hack/ensure-kubectl.sh +source "${REPO_ROOT}/hack/ensure-kubectl.sh" +# shellcheck source=../hack/ensure-kustomize.sh +source "${REPO_ROOT}/hack/ensure-kustomize.sh" +# shellcheck source=../hack/parse-prow-creds.sh +source "${REPO_ROOT}/hack/parse-prow-creds.sh" + +random-string() { + cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w ${1:-32} | head -n 1 +} + +# build Kubernetes E2E binaries +build_k8s() { + # possibly enable bazel build caching before building kubernetes + if [[ "${BAZEL_REMOTE_CACHE_ENABLED:-false}" == "true" ]]; then + create_bazel_cache_rcs.sh || true + fi + + pushd "$(go env GOPATH)/src/k8s.io/kubernetes" + + # make sure we have e2e requirements + bazel build //cmd/kubectl //test/e2e:e2e.test //vendor/github.com/onsi/ginkgo/ginkgo + + # ensure the e2e script will find our binaries ... + mkdir -p "${PWD}/_output/bin/" + cp -f "${PWD}/bazel-bin/test/e2e/e2e.test" "${PWD}/_output/bin/e2e.test" + + PATH="$(dirname "$(find "${PWD}/bazel-bin/" -name kubectl -type f)"):${PATH}" + export PATH + + # attempt to release some memory after building + sync || true + (echo 1 > /proc/sys/vm/drop_caches) 2>/dev/null || true + + popd +} + +create_cluster() { + # export cluster template which contains the manifests needed for creating the Azure cluster to run the tests + if [[ -n ${CI_VERSION:-} || -n ${USE_CI_ARTIFACTS:-} ]]; then + KUBERNETES_BRANCH="$(cd $(go env GOPATH)/src/k8s.io/kubernetes && git rev-parse --abbrev-ref HEAD)" + if [[ "${KUBERNETES_BRANCH:-}" =~ "release-" ]]; then + CI_VERSION_URL="https://dl.k8s.io/ci/latest-${KUBERNETES_BRANCH/release-}.txt" + else + CI_VERSION_URL="https://dl.k8s.io/ci/k8s-master.txt" + fi + export CLUSTER_TEMPLATE="test/cluster-template-conformance-ci-version.yaml" + export CI_VERSION="${CI_VERSION:-$(curl -sSL ${CI_VERSION_URL})}" + export KUBERNETES_VERSION="${CI_VERSION}" + else + export CLUSTER_TEMPLATE="test/cluster-template-conformance.yaml" + fi + + export CLUSTER_NAME="capz-$(head /dev/urandom | LC_ALL=C tr -dc a-z0-9 | head -c 6 ; echo '')" + # Need a cluster with at least 2 nodes + export CONTROL_PLANE_MACHINE_COUNT=${CONTROL_PLANE_MACHINE_COUNT:-1} + export WORKER_MACHINE_COUNT=${WORKER_MACHINE_COUNT:-2} + export REGISTRY=capz + export JOB_NAME="${JOB_NAME:-"cluster-api-provider-azure-conformance"}" + # timestamp is in RFC-3339 format to match kubetest + export TIMESTAMP=$(date -u '+%Y-%m-%dT%H:%M:%SZ') + ${REPO_ROOT}/hack/create-dev-cluster.sh +} + +run_upstream_e2e_tests() { + # export the target cluster KUBECONFIG if not already set + export KUBECONFIG="${KUBECONFIG:-${PWD}/kubeconfig}" + # ginkgo regexes + SKIP="${SKIP:-}" + FOCUS="${FOCUS:-"\\[Conformance\\]"}" + # if we set PARALLEL=true, skip serial tests set --ginkgo-parallel + if [[ "${PARALLEL:-false}" == "true" ]]; then + export GINKGO_PARALLEL=y + if [[ -z "${SKIP}" ]]; then + SKIP="\\[Serial\\]" + else + SKIP="\\[Serial\\]|${SKIP}" + fi + fi + + # get the number of worker nodes + NUM_NODES="$(kubectl get nodes --kubeconfig="$KUBECONFIG" \ + -o=jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.taints}{"\n"}{end}' \ + | grep -cv "node-role.kubernetes.io/master" )" + + # wait for all the nodes to be ready + kubectl wait --for=condition=Ready node --kubeconfig="$KUBECONFIG" --all || true + + # setting this env prevents ginkg e2e from trying to run provider setup + export KUBERNETES_CONFORMANCE_TEST="y" + # run the tests + (cd "$(go env GOPATH)/src/k8s.io/kubernetes" && ./hack/ginkgo-e2e.sh \ + '--provider=skeleton' "--num-nodes=${NUM_NODES}" \ + "--ginkgo.focus=${FOCUS}" "--ginkgo.skip=${SKIP}" \ + "--report-dir=${ARTIFACTS}" '--disable-log-dump=true') + + unset KUBECONFIG + unset KUBERNETES_CONFORMANCE_TEST +} + +get_logs() { + kubectl logs deploy/capz-controller-manager -n capz-system manager > "${ARTIFACTS}/logs/capz-manager.log" || true +} + +# cleanup all resources we use +cleanup() { + timeout 600 kubectl \ + delete cluster "${CLUSTER_NAME}" || true + timeout 600 kubectl \ + wait --for=delete cluster/"${CLUSTER_NAME}" || true + make kind-reset || true + # clean up e2e.test symlink + (cd "$(go env GOPATH)/src/k8s.io/kubernetes" && rm -f _output/bin/e2e.test) || true +} + +on_exit() { + unset KUBECONFIG + get_logs + # cleanup + if [[ -z "${SKIP_CLEANUP:-}" ]]; then + cleanup + fi +} + +trap on_exit EXIT +ARTIFACTS="${ARTIFACTS:-${PWD}/_artifacts}" +mkdir -p "${ARTIFACTS}/logs" + +# create cluster +if [[ -z "${SKIP_CREATE_CLUSTER:-}" ]]; then + create_cluster +fi + +# build k8s binaries and run upstream e2e tests +if [[ -z "${SKIP_UPSTREAM_E2E_TESTS:-}" ]]; then + build_k8s + run_upstream_e2e_tests +fi + +if [[ "${#}" -gt 0 ]]; then + # disable error exit so we can run post-command cleanup + set +o errexit + "${@}" + EXIT_VALUE="${?}" + exit ${EXIT_VALUE} +fi diff --git a/templates/addons/storageclass-azure-disk.yaml b/templates/addons/storageclass-azure-disk.yaml new file mode 100644 index 00000000000..81b17b36494 --- /dev/null +++ b/templates/addons/storageclass-azure-disk.yaml @@ -0,0 +1,39 @@ +apiVersion: storage.k8s.io/v1beta1 +kind: StorageClass +metadata: + name: default + annotations: + storageclass.beta.kubernetes.io/is-default-class: "true" + labels: + kubernetes.io/cluster-service: "true" +provisioner: kubernetes.io/azure-disk +parameters: + kind: Managed + storageaccounttype: Standard_LRS + cachingmode: ReadOnly +--- +apiVersion: storage.k8s.io/v1beta1 +kind: StorageClass +metadata: + name: managed-premium + annotations: + labels: + kubernetes.io/cluster-service: "true" +provisioner: kubernetes.io/azure-disk +parameters: + kind: Managed + storageaccounttype: Premium_LRS + cachingmode: ReadOnly +--- +apiVersion: storage.k8s.io/v1beta1 +kind: StorageClass +metadata: + name: managed-standard + annotations: + labels: + kubernetes.io/cluster-service: "true" +provisioner: kubernetes.io/azure-disk +parameters: + kind: Managed + storageaccounttype: Standard_LRS + cachingmode: ReadOnly diff --git a/templates/addons/storageclass-azure-file.yaml b/templates/addons/storageclass-azure-file.yaml new file mode 100644 index 00000000000..69943454370 --- /dev/null +++ b/templates/addons/storageclass-azure-file.yaml @@ -0,0 +1,10 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: azurefile + annotations: + labels: + kubernetes.io/cluster-service: "true" +provisioner: kubernetes.io/azure-file +parameters: + skuName: Standard_LRS