Implement CAAPH in e2e tests for cloud-provider-azure

Makefile

@@ -164,6 +164,9 @@ GINKGO_ARGS ?=
 ARTIFACTS ?= $(ROOT_DIR)/_artifacts
 E2E_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/azure-dev.yaml
 E2E_CONF_FILE_ENVSUBST := $(ROOT_DIR)/test/e2e/config/azure-dev-envsubst.yaml
+E2E_CLOUD_PROVIDER_AZURE ?= $(ROOT_DIR)/templates/caaph/cloud-provider-azure.yaml
+E2E_CLOUD_PROVIDER_AZURE_CI ?= $(ROOT_DIR)/templates/caaph/cloud-provider-azure-ci.yaml
+
 SKIP_CLEANUP ?= false
 SKIP_LOG_COLLECTION ?= false
 SKIP_CREATE_MGMT_CLUSTER ?= false
@@ -669,6 +672,8 @@ test-e2e-run: generate-e2e-templates install-tools ## Run e2e tests.
     $(GINKGO) -v --trace --timeout=4h --tags=e2e --focus="$(GINKGO_FOCUS)" --skip="$(GINKGO_SKIP)" --nodes=$(GINKGO_NODES) --no-color=$(GINKGO_NOCOLOR) --output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) ./test/e2e -- \
     	-e2e.artifacts-folder="$(ARTIFACTS)" \
     	-e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \
+		-e2e.cloud-provider-azure="$(E2E_CLOUD_PROVIDER_AZURE)" \
+		-e2e.cloud-provider-azure-ci="$(E2E_CLOUD_PROVIDER_AZURE_CI)" \
     	-e2e.skip-log-collection="$(SKIP_LOG_COLLECTION)" \
     	-e2e.skip-resource-cleanup=$(SKIP_CLEANUP) -e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER) $(E2E_ARGS)
 

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/Azure/go-autorest/tracing v0.6.0
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d
 	github.com/blang/semver v3.5.1+incompatible
+	github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46
 	github.com/go-logr/logr v1.2.4
 	github.com/golang/mock v1.6.0
 	github.com/google/go-cmp v0.5.9
@@ -47,9 +48,10 @@ require (
 	k8s.io/utils v0.0.0-20221128185143-99ec85e7a448
 	sigs.k8s.io/cloud-provider-azure v1.25.5
 	sigs.k8s.io/cluster-api v1.4.1
+	sigs.k8s.io/cluster-api-addon-provider-helm v0.1.0-alpha.4
 	sigs.k8s.io/cluster-api/test v1.4.1
-	sigs.k8s.io/controller-runtime v0.14.5
-	sigs.k8s.io/kind v0.17.0
+	sigs.k8s.io/controller-runtime v0.14.6
+	sigs.k8s.io/kind v0.18.0
 )
 
 require (
@@ -91,7 +93,6 @@ require (
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
-	github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 // indirect
 	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
@@ -221,4 +222,6 @@ require (
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
-replace sigs.k8s.io/cluster-api => sigs.k8s.io/cluster-api v1.4.1
+replace sigs.k8s.io/cluster-api => github.com/Jont828/cluster-api v0.4.3-0.20230412015901-405900d8bb69
+
+replace sigs.k8s.io/cluster-api/test => github.com/Jont828/cluster-api/test v0.4.3-0.20230412015901-405900d8bb69

go.sum

@@ -85,6 +85,10 @@ github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak
 github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
+github.com/Jont828/cluster-api v0.4.3-0.20230412015901-405900d8bb69 h1:OF4FuySy7PRvHDBezR9KTzgM/lQ2I4L961oh3ByHZt0=
+github.com/Jont828/cluster-api v0.4.3-0.20230412015901-405900d8bb69/go.mod h1:3NCXYKcbsAB5gl1CmLaiXiz1yaYIVhQiB48IxwqGErI=
+github.com/Jont828/cluster-api/test v0.4.3-0.20230412015901-405900d8bb69 h1:G8yGtS6b8BjdFf+NX8G9SOtFIZuzbbLv4lhpujNyEMI=
+github.com/Jont828/cluster-api/test v0.4.3-0.20230412015901-405900d8bb69/go.mod h1:ff2gsEFgyCoglpTVxf1DCR+4itUBJ7MpiaQhZyri2Eg=
 github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ=
 github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE=
 github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
@@ -1323,16 +1327,14 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/cloud-provider-azure v1.25.5 h1:YRq/wStyUtxgtMrAuRg+4FS0Gylt6MRatA8z5uag0/s=
 sigs.k8s.io/cloud-provider-azure v1.25.5/go.mod h1:estRnm6TL3dWLQTVoej4+0ygXyhIZmpszBXipHrfuZ0=
-sigs.k8s.io/cluster-api v1.4.1 h1:GtA7OJGhLvgJMgEIxKIoGLxXezM3THI/Yi10QpQ0EN4=
-sigs.k8s.io/cluster-api v1.4.1/go.mod h1:IIebZTsqyXU8CHbINV2zuMh0/wykqdr+vEXxQNeteEU=
-sigs.k8s.io/cluster-api/test v1.4.1 h1:4ezS3IVGqL+GmsPx4gzo2e4hs2GOrMBsgch+RQPBEWs=
-sigs.k8s.io/cluster-api/test v1.4.1/go.mod h1:RHazilXiXNuRYYh/aiX6ZvpUaXOzwNjmaEou/TltSOs=
-sigs.k8s.io/controller-runtime v0.14.5 h1:6xaWFqzT5KuAQ9ufgUaj1G/+C4Y1GRkhrxl+BJ9i+5s=
-sigs.k8s.io/controller-runtime v0.14.5/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0=
+sigs.k8s.io/cluster-api-addon-provider-helm v0.1.0-alpha.4 h1:wR3lSwhAL6AC7LvkSf+0ATiE8u5WHF/pLxmCibSTcSE=
+sigs.k8s.io/cluster-api-addon-provider-helm v0.1.0-alpha.4/go.mod h1:s3klPverZSGHRlvJxA35TwtM2+6NeRrcljd3cDsD+Jk=
+sigs.k8s.io/controller-runtime v0.14.6 h1:oxstGVvXGNnMvY7TAESYk+lzr6S3V5VFxQ6d92KcwQA=
+sigs.k8s.io/controller-runtime v0.14.6/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0=
 sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k=
 sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
-sigs.k8s.io/kind v0.17.0 h1:CScmGz/wX66puA06Gj8OZb76Wmk7JIjgWf5JDvY7msM=
-sigs.k8s.io/kind v0.17.0/go.mod h1:Qqp8AiwOlMZmJWs37Hgs31xcbiYXjtXlRBSftcnZXQk=
+sigs.k8s.io/kind v0.18.0 h1:ahgZdVV1pdhXlYe1f+ztISakT23KdrBl/NFY9JMygzs=
+sigs.k8s.io/kind v0.18.0/go.mod h1:Qqp8AiwOlMZmJWs37Hgs31xcbiYXjtXlRBSftcnZXQk=
 sigs.k8s.io/kustomize/api v0.12.1 h1:7YM7gW3kYBwtKvoY216ZzY+8hM+lV53LUayghNRJ0vM=
 sigs.k8s.io/kustomize/api v0.12.1/go.mod h1:y3JUhimkZkR6sbLNwfJHxvo1TCLwuwm14sCYnkH6S1s=
 sigs.k8s.io/kustomize/kyaml v0.13.9 h1:Qz53EAaFFANyNgyOEJbT/yoIHygK40/ZcvU3rgry2Tk=

internal/test/env/env.go

@@ -35,6 +35,7 @@ import (
 	"k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
 	"k8s.io/utils/pointer"
+	addonsv1alpha1 "sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1"
 	infrav1alpha3 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3"
 	infrav1alpha4 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha4"
 	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
@@ -63,6 +64,7 @@ func init() {
 	utilruntime.Must(infrav1exp.AddToScheme(scheme))
 	utilruntime.Must(infrav1alpha3.AddToScheme(scheme))
 	utilruntime.Must(infrav1alpha4.AddToScheme(scheme))
+	utilruntime.Must(addonsv1alpha1.AddToScheme(scheme))
 
 	// Get the root of the current file to use in CRD paths.
 	_, filename, _, _ := goruntime.Caller(0) //nolint:dogsled // Ignore "declaration has 3 blank identifiers" check.

main.go

@@ -37,6 +37,7 @@ import (
 	cgrecord "k8s.io/client-go/tools/record"
 	"k8s.io/klog/v2"
 	"k8s.io/klog/v2/klogr"
+	addonsv1alpha1 "sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1"
 	infrav1alpha3 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3"
 	infrav1alpha4 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha4"
 	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
@@ -81,6 +82,7 @@ func init() {
 	_ = expv1alpha4.AddToScheme(scheme)
 	_ = clusterv1.AddToScheme(scheme)
 	_ = expv1.AddToScheme(scheme)
+	_ = addonsv1alpha1.AddToScheme(scheme)
 	_ = kubeadmv1.AddToScheme(scheme)
 	// +kubebuilder:scaffold:scheme
 

templates/caaph/cloud-provider-azure-ci.yaml

@@ -0,0 +1,25 @@
+apiVersion: addons.cluster.x-k8s.io/v1alpha1
+kind: HelmChartProxy
+metadata:
+  name: cloud-provider-azure-chart-ci
+spec:
+  clusterSelector:
+    matchLabels:
+      enableCloudProviderAzureChart: true
+  repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo
+  chartName: cloud-provider-azure
+  releaseName: cloud-provider-azure
+  valuesTemplate: |
+    infra:
+      clusterName: {{ .Cluster.metadata.name }}
+    cloudControllerManager:
+      clusterCIDR:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }}
+      - {{ $cidr }}{{end}}
+      imageName: ${CCM_IMAGE_NAME}
+      imageRepository: ${CCM_IMAGE_REGISTRY}
+      imageTag: ${IMAGE_TAG_CCM}
+    cloudNodeManager:
+      imageName: ${CNM_IMAGE_NAME}
+      imageRepository: ${CNM_IMAGE_REGISTRY}
+      imageTag: ${IMAGE_TAG_CCM}
+

templates/caaph/cloud-provider-azure.yaml

@@ -0,0 +1,17 @@
+apiVersion: addons.cluster.x-k8s.io/v1alpha1
+kind: HelmChartProxy
+metadata:
+  name: cloud-provider-azure-chart
+spec:
+  clusterSelector:
+    matchLabels:
+      enableCloudProviderAzureChart: true
+  repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo
+  chartName: cloud-provider-azure
+  releaseName: cloud-provider-azure
+  valuesTemplate: |
+    infra:
+      clusterName: {{ .Cluster.metadata.name }}
+    cloudControllerManager:
+      clusterCIDR:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }}
+      - {{ $cidr }}{{end}}

templates/cluster-template.yaml

@@ -1,6 +1,8 @@
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: Cluster
 metadata:
+  labels:
+    enabledCloudProviderAzureChart: true
   name: ${CLUSTER_NAME}
   namespace: default
 spec:

templates/flavors/base/cluster-template.yaml

@@ -3,6 +3,8 @@ apiVersion: cluster.x-k8s.io/v1beta1
 kind: Cluster
 metadata:
   name: ${CLUSTER_NAME}
+  labels:
+    enabledCloudProviderAzureChart: true
 spec:
   clusterNetwork:
     pods: