🏃extend ci-conformance.sh testing capability

docs/development.md

@@ -140,7 +140,7 @@ To build a kind cluster and start Tilt, just run:
 ```shell
 make tilt-up
 ```
-By default, the Cluster API components deployed by Tilt have experimental features turned off. 
+By default, the Cluster API components deployed by Tilt have experimental features turned off.
 If you would like to enable these features, add `extra_args` as specified in [The Cluster API Book](https://cluster-api.sigs.k8s.io/developer/tilt.html#create-a-tilt-settingsjson-file).
 
 Once your kind management cluster is up and running, you can [deploy a workload cluster](#deploying-a-workload-cluster).
@@ -355,21 +355,28 @@ To run Conformance locally, set `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET`, `AZURE
 
 You can optionally set the following variables:
 
-| Variable                    | Description                                                                                                    |
-| --------------------------- | -------------------------------------------------------------------------------------------------------------- |
-| `AZURE_SSH_PUBLIC_KEY_FILE` | Use your own SSH key.                                                                                          |
-| `SKIP_CREATE_CLUSTER`       | Skip cluster creation.                                                                                         |
-| `SKIP_TESTS`                | Skip running Kubernetes E2E tests.                                                                             |
+| Variable                    | Description                                                                                                   |
+|-----------------------------|---------------------------------------------------------------------------------------------------------------|
+| `AZURE_SSH_PUBLIC_KEY_FILE` | Use your own SSH key.                                                                                         |
+| `SKIP_CREATE_CLUSTER`       | Skip cluster creation.                                                                                        |
+| `SKIP_UPSTREAM_E2E_TESTS`   | Skip running upstream Kubernetes E2E tests.                                                                   |
 | `SKIP_CLEANUP`              | Skip deleting the cluster after the tests finish running.                                                      |
-| `KUBECONFIG`                | Provide your existing cluster kubeconfig filepath. If no kubeconfig is provided, `./kubeconfig` will be used.  |
-| `SKIP`                      | Regexp for test cases to skip.                                                                                 |
-| `FOCUS`                     | Regexp for which test cases to run.                                                                            |
+| `KUBECONFIG`                | Provide your existing cluster kubeconfig filepath. If no kubeconfig is provided, `./kubeconfig` will be used.     |
+| `SKIP`                      | Regexp for test cases to skip.                                                                                |
+| `FOCUS`                     | Regexp for which test cases to run.                                                                           |
 | `PARALLEL`                  | Skip serial tests and set --ginkgo-parallel.                                                                  |
-| `USE_CI_ARTIFACTS`          | Use a CI version of Kubernetes, ie. not a released version (eg. `v1.19.0-alpha.1.426+0926c9c47677e9`)          |
-| `CI_VERSION`                | Provide a custom CI version of Kubernetes. By default, the latest master commit will be used.                  |
+| `USE_CI_ARTIFACTS`          | Use a CI version of Kubernetes, ie. not a released version (eg. `v1.19.0-alpha.1.426+0926c9c47677e9`)         |
+| `CI_VERSION`                | Provide a custom CI version of Kubernetes. By default, the latest master commit will be used.                 |
 
 You can also customize the configuration of the CAPZ cluster (assuming that `SKIP_CREATE_CLUSTER` is not set). See [Customizing the cluster deployment](#customizing-the-cluster-deployment) for more details.
 
+In addition to upstream E2E, you can append custom commands to `./scripts/ci-entrypoint.sh` to run E2E from other projects against a CAPZ cluster:
+
+```bash
+export SKIP_UPSTREAM_E2E_TESTS="false"
+./scripts/ci-entrypoint.sh bash -c "cd ${GOPATH}/src/github.com/my-org/my-project && make e2e"
+```
+
 <!-- References -->
 
 [go]: https://golang.org/doc/install

scripts/ci-entrypoint.sh

@@ -0,0 +1,177 @@
+#!/bin/bash
+
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+###############################################################################
+
+# To run locally, set AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_SUBSCRIPTION_ID, AZURE_TENANT_ID
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+cd "${REPO_ROOT}" || exit 1
+
+# shellcheck source=../hack/ensure-go.sh
+source "${REPO_ROOT}/hack/ensure-go.sh"
+# shellcheck source=../hack/ensure-kind.sh
+source "${REPO_ROOT}/hack/ensure-kind.sh"
+# shellcheck source=../hack/ensure-kubectl.sh
+source "${REPO_ROOT}/hack/ensure-kubectl.sh"
+# shellcheck source=../hack/ensure-kustomize.sh
+source "${REPO_ROOT}/hack/ensure-kustomize.sh"
+# shellcheck source=../hack/parse-prow-creds.sh
+source "${REPO_ROOT}/hack/parse-prow-creds.sh"
+
+random-string() {
+    cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w ${1:-32} | head -n 1
+}
+
+# build Kubernetes E2E binaries
+build_k8s() {
+    # possibly enable bazel build caching before building kubernetes
+    if [[ "${BAZEL_REMOTE_CACHE_ENABLED:-false}" == "true" ]]; then
+        create_bazel_cache_rcs.sh || true
+    fi
+
+    pushd "$(go env GOPATH)/src/k8s.io/kubernetes"
+
+    # make sure we have e2e requirements
+    bazel build //cmd/kubectl //test/e2e:e2e.test //vendor/github.com/onsi/ginkgo/ginkgo
+
+    # ensure the e2e script will find our binaries ...
+    mkdir -p "${PWD}/_output/bin/"
+    cp -f "${PWD}/bazel-bin/test/e2e/e2e.test" "${PWD}/_output/bin/e2e.test"
+
+    PATH="$(dirname "$(find "${PWD}/bazel-bin/" -name kubectl -type f)"):${PATH}"
+    export PATH
+
+    # attempt to release some memory after building
+    sync || true
+    (echo 1 > /proc/sys/vm/drop_caches) 2>/dev/null || true
+
+    popd
+}
+
+create_cluster() {
+    # export cluster template which contains the manifests needed for creating the Azure cluster to run the tests
+    if [[ -n ${CI_VERSION:-} || -n ${USE_CI_ARTIFACTS:-} ]]; then
+        KUBERNETES_BRANCH="$(cd $(go env GOPATH)/src/k8s.io/kubernetes && git rev-parse --abbrev-ref HEAD)"
+        if [[ "${KUBERNETES_BRANCH:-}" =~ "release-" ]]; then
+            CI_VERSION_URL="https://dl.k8s.io/ci/latest-${KUBERNETES_BRANCH/release-}.txt"
+        else
+            CI_VERSION_URL="https://dl.k8s.io/ci/k8s-master.txt"
+        fi
+        export CLUSTER_TEMPLATE="test/cluster-template-conformance-ci-version.yaml"
+        export CI_VERSION="${CI_VERSION:-$(curl -sSL ${CI_VERSION_URL})}"
+        export KUBERNETES_VERSION="${CI_VERSION}"
+    else
+        export CLUSTER_TEMPLATE="test/cluster-template-conformance.yaml"
+    fi
+
+    export CLUSTER_NAME="capz-$(head /dev/urandom | LC_ALL=C tr -dc a-z0-9 | head -c 6 ; echo '')"
+    # Need a cluster with at least 2 nodes
+    export CONTROL_PLANE_MACHINE_COUNT=${CONTROL_PLANE_MACHINE_COUNT:-1}
+    export WORKER_MACHINE_COUNT=${WORKER_MACHINE_COUNT:-2}
+    export REGISTRY=capz
+    export JOB_NAME="${JOB_NAME:-"cluster-api-provider-azure-conformance"}"
+    # timestamp is in RFC-3339 format to match kubetest
+    export TIMESTAMP=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
+    ${REPO_ROOT}/hack/create-dev-cluster.sh
+}
+
+run_upstream_e2e_tests() {
+    # export the target cluster KUBECONFIG if not already set
+    export KUBECONFIG="${KUBECONFIG:-${PWD}/kubeconfig}"
+    # ginkgo regexes
+    SKIP="${SKIP:-}"
+    FOCUS="${FOCUS:-"\\[Conformance\\]"}"
+    # if we set PARALLEL=true, skip serial tests set --ginkgo-parallel
+    if [[ "${PARALLEL:-false}" == "true" ]]; then
+        export GINKGO_PARALLEL=y
+        if [[ -z "${SKIP}" ]]; then
+            SKIP="\\[Serial\\]"
+        else
+            SKIP="\\[Serial\\]|${SKIP}"
+        fi
+    fi
+
+    # get the number of worker nodes
+    NUM_NODES="$(kubectl get nodes --kubeconfig="$KUBECONFIG" \
+    -o=jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.taints}{"\n"}{end}' \
+    | grep -cv "node-role.kubernetes.io/master" )"
+
+    # wait for all the nodes to be ready
+    kubectl wait --for=condition=Ready node --kubeconfig="$KUBECONFIG" --all || true
+
+    # setting this env prevents ginkg e2e from trying to run provider setup
+    export KUBERNETES_CONFORMANCE_TEST="y"
+    # run the tests
+    (cd "$(go env GOPATH)/src/k8s.io/kubernetes" && ./hack/ginkgo-e2e.sh \
+    '--provider=skeleton' "--num-nodes=${NUM_NODES}" \
+    "--ginkgo.focus=${FOCUS}" "--ginkgo.skip=${SKIP}" \
+    "--report-dir=${ARTIFACTS}" '--disable-log-dump=true')
+
+    unset KUBECONFIG
+    unset KUBERNETES_CONFORMANCE_TEST
+}
+
+get_logs() {
+    kubectl logs deploy/capz-controller-manager -n capz-system manager > "${ARTIFACTS}/logs/capz-manager.log" || true
+}
+
+# cleanup all resources we use
+cleanup() {
+    timeout 600 kubectl \
+        delete cluster "${CLUSTER_NAME}" || true
+        timeout 600 kubectl \
+        wait --for=delete cluster/"${CLUSTER_NAME}" || true
+    make kind-reset || true
+    # clean up e2e.test symlink
+    (cd "$(go env GOPATH)/src/k8s.io/kubernetes" && rm -f _output/bin/e2e.test) || true
+}
+
+on_exit() {
+    unset KUBECONFIG
+    get_logs
+    # cleanup
+    if [[ -z "${SKIP_CLEANUP:-}" ]]; then
+        cleanup
+    fi
+}
+
+trap on_exit EXIT
+ARTIFACTS="${ARTIFACTS:-${PWD}/_artifacts}"
+mkdir -p "${ARTIFACTS}/logs"
+
+# create cluster
+if [[ -z "${SKIP_CREATE_CLUSTER:-}" ]]; then
+    create_cluster
+fi
+
+# build k8s binaries and run upstream e2e tests
+if [[ -z "${SKIP_UPSTREAM_E2E_TESTS:-}" ]]; then
+    build_k8s
+    run_upstream_e2e_tests
+fi
+
+if [[ "${#}" -gt 0 ]]; then
+    # disable error exit so we can run post-command cleanup
+    set +o errexit
+    "${@}"
+    EXIT_VALUE="${?}"
+    exit ${EXIT_VALUE}
+fi

templates/addons/storageclass-azure-disk.yaml

@@ -0,0 +1,39 @@
+apiVersion: storage.k8s.io/v1beta1
+kind: StorageClass
+metadata:
+  name: default
+  annotations:
+    storageclass.beta.kubernetes.io/is-default-class: "true"
+  labels:
+    kubernetes.io/cluster-service: "true"
+provisioner: kubernetes.io/azure-disk
+parameters:
+  kind: Managed
+  storageaccounttype: Standard_LRS
+  cachingmode: ReadOnly
+---
+apiVersion: storage.k8s.io/v1beta1
+kind: StorageClass
+metadata:
+  name: managed-premium
+  annotations:
+  labels:
+    kubernetes.io/cluster-service: "true"
+provisioner: kubernetes.io/azure-disk
+parameters:
+  kind: Managed
+  storageaccounttype: Premium_LRS
+  cachingmode: ReadOnly
+---
+apiVersion: storage.k8s.io/v1beta1
+kind: StorageClass
+metadata:
+  name: managed-standard
+  annotations:
+  labels:
+    kubernetes.io/cluster-service: "true"
+provisioner: kubernetes.io/azure-disk
+parameters:
+  kind: Managed
+  storageaccounttype: Standard_LRS
+  cachingmode: ReadOnly

templates/addons/storageclass-azure-file.yaml

@@ -0,0 +1,10 @@
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: azurefile
+  annotations:
+  labels:
+    kubernetes.io/cluster-service: "true"
+provisioner: kubernetes.io/azure-file
+parameters:
+  skuName: Standard_LRS