Resolve Repo list for Windows

Makefile

@@ -49,7 +49,7 @@ GO_INSTALL = ./scripts/go_install.sh
 E2E_DATA_DIR ?= $(ROOT_DIR)/test/e2e/data
 KUBETEST_CONF_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/conformance.yaml)
 KUBETEST_WINDOWS_CONF_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/upstream-windows.yaml)
-KUBETEST_REPO_LIST_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/repo-list.yaml)
+KUBETEST_REPO_LIST_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/)
 AZURE_TEMPLATES := $(E2E_DATA_DIR)/infrastructure-azure
 
 # set --output-base used for conversion-gen which needs to be different for in GOPATH and outside GOPATH dev
@@ -130,7 +130,7 @@ E2E_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/azure-dev.yaml
 E2E_CONF_FILE_ENVSUBST := $(ROOT_DIR)/test/e2e/config/azure-dev-envsubst.yaml
 SKIP_CLEANUP ?= false
 SKIP_CREATE_MGMT_CLUSTER ?= false
-WIN_REPO_LIST ?= https://raw.githubusercontent.com/kubernetes-sigs/windows-testing/master/images/image-repo-list
+WIN_REPO_URL ?=
 
 # Build time versioning details.
 LDFLAGS := $(shell hack/version.sh)
@@ -202,8 +202,10 @@ test-conformance-fast: ## Run conformance test on workload cluster using a subse
 
 .PHONY: test-windows-upstream
 test-windows-upstream: ## Run windows upstream tests on workload cluster.
-	curl --retry $(CURL_RETRIES) $(WIN_REPO_LIST) -o $(KUBETEST_REPO_LIST_PATH)
-	$(MAKE) test-conformance CONFORMANCE_E2E_ARGS="-kubetest.config-file=$(KUBETEST_WINDOWS_CONF_PATH) -kubetest.repo-list-file=$(KUBETEST_REPO_LIST_PATH) $(E2E_ARGS)"
+ifneq ($(WIN_REPO_URL), )
+	curl --retry $(CURL_RETRIES) $(WIN_REPO_URL) -o $(KUBETEST_REPO_LIST_PATH)/custom-repo-list.yaml
+endif
+	$(MAKE) test-conformance CONFORMANCE_E2E_ARGS="-kubetest.config-file=$(KUBETEST_WINDOWS_CONF_PATH) -kubetest.repo-list-path=$(KUBETEST_REPO_LIST_PATH) $(E2E_ARGS)"
 
 $(KUBE_APISERVER) $(ETCD): ## install test asset kubectl, kube-apiserver, etcd
 	source ./scripts/fetch_ext_bins.sh && fetch_tools

test/e2e/conformance_test.go

@@ -50,6 +50,7 @@ var _ = Describe("Conformance Tests", func() {
 		clusterName   string
 		namespace     *corev1.Namespace
 		specName      = "conformance-tests"
+		repoList      = ""
 	)
 
 	BeforeEach(func() {
@@ -154,8 +155,8 @@ var _ = Describe("Conformance Tests", func() {
 		b.RecordValue("cluster creation", runtime.Seconds())
 		workloadProxy := bootstrapClusterProxy.GetWorkloadCluster(ctx, namespace.Name, clusterName)
 
-		// Windows requires a taint on control nodes nodes since not all conformance tests have ability to run
 		if isWindows(kubetestConfigFilePath) {
+			// Windows requires a taint on control nodes nodes since not all conformance tests have ability to run
 			options := v1.ListOptions{
 				LabelSelector: "kubernetes.io/os=linux",
 			}
@@ -168,6 +169,11 @@ var _ = Describe("Conformance Tests", func() {
 
 			err := node.TaintNode(workloadProxy.GetClientSet(), options, noScheduleTaint)
 			Expect(err).NotTo(HaveOccurred())
+
+			// Windows requires a repo-list because some images are not in k8s gcr
+			repoList, err = resolveKubetestRepoListPath(kubernetesVersion, kubetestRepoListPath)
+			Expect(err).NotTo(HaveOccurred())
+			fmt.Fprintf(GinkgoWriter, "INFO: Using repo-list %s for version %s\n", repoList, kubernetesVersion)
 		}
 
 		ginkgoNodes, err := strconv.Atoi(e2eConfig.GetVariable("CONFORMANCE_NODES"))
@@ -179,7 +185,7 @@ var _ = Describe("Conformance Tests", func() {
 					ClusterProxy:         workloadProxy,
 					NumberOfNodes:        int(workerMachineCount),
 					ConfigFilePath:       kubetestConfigFilePath,
-					KubeTestRepoListPath: kubetestRepoListPath,
+					KubeTestRepoListPath: repoList,
 					ConformanceImage:     e2eConfig.GetVariable("CONFORMANCE_IMAGE"),
 					GinkgoNodes:          ginkgoNodes,
 				},

test/e2e/data/kubetest/repo-list-k8sprow.yaml

@@ -0,0 +1,8 @@
+# used for k8s versions <= 1.20 https://github.com/kubernetes-sigs/windows-testing/tree/master/images#image-repository-list
+dockerLibraryRegistry: k8sprow.azurecr.io/kubernetes-e2e-test-images
+e2eRegistry: k8sprow.azurecr.io/kubernetes-e2e-test-images
+promoterE2eRegistry: k8sprow.azurecr.io/kubernetes-e2e-test-images
+etcdRegistry: k8sprow.azurecr.io/kubernetes-e2e-test-images
+gcRegistry: k8sprow.azurecr.io/kubernetes-e2e-test-images
+PrivateRegistry: e2eteam
+sampleRegistry: k8sprow.azurecr.io/kubernetes-e2e-test-images

test/e2e/data/kubetest/repo-list.yaml

@@ -1,3 +1,7 @@
-gcAuthenticatedRegistry: e2eprivate
+# used for 1.21+ https://github.com/kubernetes-sigs/windows-testing/tree/master/images#image-repository-list
 gcEtcdRegistry: k8sprow.azurecr.io/kubernetes-e2e-test-images
-privateRegistry: e2eteam
+# There are two other repos that could be set:
+# privateRegistry: e2eteam
+#  - only used in gce tests
+# gcAuthenticatedRegistry: e2eprivate
+# - needed for OS versions 20H2+ requires update to capi framework to accepted dockerregistry config file

test/e2e/e2e_suite_test.go

@@ -40,6 +40,7 @@ import (
 	aadpodv1 "github.com/Azure/aad-pod-identity/pkg/apis/aadpodidentity/v1"
 	"github.com/Azure/azure-sdk-for-go/services/preview/monitor/mgmt/2019-06-01/insights"
 	"github.com/Azure/go-autorest/autorest/azure/auth"
+	"github.com/Azure/go-autorest/autorest/to"
 	. "github.com/onsi/ginkgo"
 	"github.com/onsi/ginkgo/config"
 	"github.com/onsi/ginkgo/reporters"
@@ -49,6 +50,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha4"
+	expv1 "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha4"
 	capi_e2e "sigs.k8s.io/cluster-api/test/e2e"
 	"sigs.k8s.io/cluster-api/test/framework"
 	"sigs.k8s.io/cluster-api/test/framework/bootstrap"
@@ -248,7 +250,7 @@ func init() {
 	flag.BoolVar(&skipCleanup, "e2e.skip-resource-cleanup", false, "if true, the resource cleanup after tests will be skipped")
 	flag.BoolVar(&useExistingCluster, "e2e.use-existing-cluster", false, "if true, the test uses the current cluster instead of creating a new one (default discovery rules apply)")
 	flag.StringVar(&kubetestConfigFilePath, "kubetest.config-file", "", "path to the kubetest configuration file")
-	flag.StringVar(&kubetestRepoListPath, "kubetest.repo-list-file", "", "path to the kubetest repo-list file")
+	flag.StringVar(&kubetestRepoListPath, "kubetest.repo-list-path", "", "path to the kubetest repo-list path")
 }
 
 func TestE2E(t *testing.T) {

test/e2e/helpers.go

@@ -33,6 +33,7 @@ import (
 	"text/tabwriter"
 	"time"
 
+	"github.com/blang/semver"
 	. "github.com/onsi/ginkgo"
 	"github.com/onsi/ginkgo/config"
 	. "github.com/onsi/gomega"
@@ -576,3 +577,36 @@ func latestCIVersion(label string) (string, error) {
 
 	return strings.TrimSpace(string(b)), nil
 }
+
+// resolveKubetestRepoListPath will set the correct repo list for Windows:
+// - if WIN_REPO_URL is set use the custom file downloaded via makefile
+// - if CI version is "latest" will use repo-list.yaml
+// - if CI version is  "latest-1.xx" will compare values and use correct repoList
+// - if standard version will compare values and use correct repoList
+// - if unable to determine version falls back to using latest
+func resolveKubetestRepoListPath(version string, path string) (string, error) {
+	if _, ok := os.LookupEnv("WIN_REPO_URL"); ok {
+		return filepath.Join(path, "custom-repo-list.yaml"), nil
+	}
+
+	if version == "latest" {
+		return filepath.Join(path, "repo-list.yaml"), nil
+	}
+
+	version = strings.TrimPrefix(version, "latest-")
+	currentVersion, err := semver.ParseTolerant(version)
+	if err != nil {
+		return "", err
+	}
+
+	versionCutoff, err := semver.Make("1.21.0")
+	if err != nil {
+		return "", err
+	}
+
+	if currentVersion.LT(versionCutoff) {
+		return filepath.Join(path, "repo-list-k8sprow.yaml"), nil
+	}
+
+	return filepath.Join(path, "repo-list.yaml"), nil
+}