From 7f1f6a368f294d0676c4a0549814dced340a364a Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Thu, 10 Jun 2021 12:57:49 -0700 Subject: [PATCH] Resolve Repo list for Windows --- Makefile | 10 +++--- test/e2e/conformance_test.go | 10 ++++-- test/e2e/data/kubetest/repo-list-k8sprow.yaml | 8 +++++ test/e2e/data/kubetest/repo-list.yaml | 8 +++-- test/e2e/e2e_suite_test.go | 4 ++- test/e2e/helpers.go | 34 +++++++++++++++++++ 6 files changed, 65 insertions(+), 9 deletions(-) create mode 100644 test/e2e/data/kubetest/repo-list-k8sprow.yaml diff --git a/Makefile b/Makefile index aee65b29c96..fd69dc742ad 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,7 @@ GO_INSTALL = ./scripts/go_install.sh E2E_DATA_DIR ?= $(ROOT_DIR)/test/e2e/data KUBETEST_CONF_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/conformance.yaml) KUBETEST_WINDOWS_CONF_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/upstream-windows.yaml) -KUBETEST_REPO_LIST_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/repo-list.yaml) +KUBETEST_REPO_LIST_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/) AZURE_TEMPLATES := $(E2E_DATA_DIR)/infrastructure-azure # set --output-base used for conversion-gen which needs to be different for in GOPATH and outside GOPATH dev @@ -130,7 +130,7 @@ E2E_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/azure-dev.yaml E2E_CONF_FILE_ENVSUBST := $(ROOT_DIR)/test/e2e/config/azure-dev-envsubst.yaml SKIP_CLEANUP ?= false SKIP_CREATE_MGMT_CLUSTER ?= false -WIN_REPO_LIST ?= https://raw.githubusercontent.com/kubernetes-sigs/windows-testing/master/images/image-repo-list +WIN_REPO_URL ?= # Build time versioning details. LDFLAGS := $(shell hack/version.sh) @@ -202,8 +202,10 @@ test-conformance-fast: ## Run conformance test on workload cluster using a subse .PHONY: test-windows-upstream test-windows-upstream: ## Run windows upstream tests on workload cluster. - curl --retry $(CURL_RETRIES) $(WIN_REPO_LIST) -o $(KUBETEST_REPO_LIST_PATH) - $(MAKE) test-conformance CONFORMANCE_E2E_ARGS="-kubetest.config-file=$(KUBETEST_WINDOWS_CONF_PATH) -kubetest.repo-list-file=$(KUBETEST_REPO_LIST_PATH) $(E2E_ARGS)" +ifneq ($(WIN_REPO_URL), ) + curl --retry $(CURL_RETRIES) $(WIN_REPO_URL) -o $(KUBETEST_REPO_LIST_PATH)/custom-repo-list.yaml +endif + $(MAKE) test-conformance CONFORMANCE_E2E_ARGS="-kubetest.config-file=$(KUBETEST_WINDOWS_CONF_PATH) -kubetest.repo-list-path=$(KUBETEST_REPO_LIST_PATH) $(E2E_ARGS)" $(KUBE_APISERVER) $(ETCD): ## install test asset kubectl, kube-apiserver, etcd source ./scripts/fetch_ext_bins.sh && fetch_tools diff --git a/test/e2e/conformance_test.go b/test/e2e/conformance_test.go index 6ecf121e546..3e849a4ffb7 100644 --- a/test/e2e/conformance_test.go +++ b/test/e2e/conformance_test.go @@ -50,6 +50,7 @@ var _ = Describe("Conformance Tests", func() { clusterName string namespace *corev1.Namespace specName = "conformance-tests" + repoList = "" ) BeforeEach(func() { @@ -154,8 +155,8 @@ var _ = Describe("Conformance Tests", func() { b.RecordValue("cluster creation", runtime.Seconds()) workloadProxy := bootstrapClusterProxy.GetWorkloadCluster(ctx, namespace.Name, clusterName) - // Windows requires a taint on control nodes nodes since not all conformance tests have ability to run if isWindows(kubetestConfigFilePath) { + // Windows requires a taint on control nodes nodes since not all conformance tests have ability to run options := v1.ListOptions{ LabelSelector: "kubernetes.io/os=linux", } @@ -168,6 +169,11 @@ var _ = Describe("Conformance Tests", func() { err := node.TaintNode(workloadProxy.GetClientSet(), options, noScheduleTaint) Expect(err).NotTo(HaveOccurred()) + + // Windows requires a repo-list because some images are not in k8s gcr + repoList, err = resolveKubetestRepoListPath(kubernetesVersion, kubetestRepoListPath) + Expect(err).NotTo(HaveOccurred()) + fmt.Fprintf(GinkgoWriter, "INFO: Using repo-list %s for version %s\n", repoList, kubernetesVersion) } ginkgoNodes, err := strconv.Atoi(e2eConfig.GetVariable("CONFORMANCE_NODES")) @@ -179,7 +185,7 @@ var _ = Describe("Conformance Tests", func() { ClusterProxy: workloadProxy, NumberOfNodes: int(workerMachineCount), ConfigFilePath: kubetestConfigFilePath, - KubeTestRepoListPath: kubetestRepoListPath, + KubeTestRepoListPath: repoList, ConformanceImage: e2eConfig.GetVariable("CONFORMANCE_IMAGE"), GinkgoNodes: ginkgoNodes, }, diff --git a/test/e2e/data/kubetest/repo-list-k8sprow.yaml b/test/e2e/data/kubetest/repo-list-k8sprow.yaml new file mode 100644 index 00000000000..418c112a82c --- /dev/null +++ b/test/e2e/data/kubetest/repo-list-k8sprow.yaml @@ -0,0 +1,8 @@ +# used for k8s versions <= 1.20 https://github.com/kubernetes-sigs/windows-testing/tree/master/images#image-repository-list +dockerLibraryRegistry: k8sprow.azurecr.io/kubernetes-e2e-test-images +e2eRegistry: k8sprow.azurecr.io/kubernetes-e2e-test-images +promoterE2eRegistry: k8sprow.azurecr.io/kubernetes-e2e-test-images +etcdRegistry: k8sprow.azurecr.io/kubernetes-e2e-test-images +gcRegistry: k8sprow.azurecr.io/kubernetes-e2e-test-images +PrivateRegistry: e2eteam +sampleRegistry: k8sprow.azurecr.io/kubernetes-e2e-test-images diff --git a/test/e2e/data/kubetest/repo-list.yaml b/test/e2e/data/kubetest/repo-list.yaml index 8f84224db48..5d48ea223d5 100644 --- a/test/e2e/data/kubetest/repo-list.yaml +++ b/test/e2e/data/kubetest/repo-list.yaml @@ -1,3 +1,7 @@ -gcAuthenticatedRegistry: e2eprivate +# used for 1.21+ https://github.com/kubernetes-sigs/windows-testing/tree/master/images#image-repository-list gcEtcdRegistry: k8sprow.azurecr.io/kubernetes-e2e-test-images -privateRegistry: e2eteam \ No newline at end of file +# There are two other repos that could be set: +# privateRegistry: e2eteam +# - only used in gce tests +# gcAuthenticatedRegistry: e2eprivate +# - needed for OS versions 20H2+ requires update to capi framework to accepted dockerregistry config file \ No newline at end of file diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 5a38f97a813..ca68bf1102f 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -40,6 +40,7 @@ import ( aadpodv1 "github.com/Azure/aad-pod-identity/pkg/apis/aadpodidentity/v1" "github.com/Azure/azure-sdk-for-go/services/preview/monitor/mgmt/2019-06-01/insights" "github.com/Azure/go-autorest/autorest/azure/auth" + "github.com/Azure/go-autorest/autorest/to" . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/config" "github.com/onsi/ginkgo/reporters" @@ -49,6 +50,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha4" + expv1 "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha4" capi_e2e "sigs.k8s.io/cluster-api/test/e2e" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/bootstrap" @@ -248,7 +250,7 @@ func init() { flag.BoolVar(&skipCleanup, "e2e.skip-resource-cleanup", false, "if true, the resource cleanup after tests will be skipped") flag.BoolVar(&useExistingCluster, "e2e.use-existing-cluster", false, "if true, the test uses the current cluster instead of creating a new one (default discovery rules apply)") flag.StringVar(&kubetestConfigFilePath, "kubetest.config-file", "", "path to the kubetest configuration file") - flag.StringVar(&kubetestRepoListPath, "kubetest.repo-list-file", "", "path to the kubetest repo-list file") + flag.StringVar(&kubetestRepoListPath, "kubetest.repo-list-path", "", "path to the kubetest repo-list path") } func TestE2E(t *testing.T) { diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index f0ffa5372c9..9b92a1bfca8 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -33,6 +33,7 @@ import ( "text/tabwriter" "time" + "github.com/blang/semver" . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/config" . "github.com/onsi/gomega" @@ -576,3 +577,36 @@ func latestCIVersion(label string) (string, error) { return strings.TrimSpace(string(b)), nil } + +// resolveKubetestRepoListPath will set the correct repo list for Windows: +// - if WIN_REPO_URL is set use the custom file downloaded via makefile +// - if CI version is "latest" will use repo-list.yaml +// - if CI version is "latest-1.xx" will compare values and use correct repoList +// - if standard version will compare values and use correct repoList +// - if unable to determine version falls back to using latest +func resolveKubetestRepoListPath(version string, path string) (string, error) { + if _, ok := os.LookupEnv("WIN_REPO_URL"); ok { + return filepath.Join(path, "custom-repo-list.yaml"), nil + } + + if version == "latest" { + return filepath.Join(path, "repo-list.yaml"), nil + } + + version = strings.TrimPrefix(version, "latest-") + currentVersion, err := semver.ParseTolerant(version) + if err != nil { + return "", err + } + + versionCutoff, err := semver.Make("1.21.0") + if err != nil { + return "", err + } + + if currentVersion.LT(versionCutoff) { + return filepath.Join(path, "repo-list-k8sprow.yaml"), nil + } + + return filepath.Join(path, "repo-list.yaml"), nil +}