Install azure disk CSI driver Helm chart with CAAPH

Makefile

@@ -164,6 +164,14 @@ GINKGO_ARGS ?=
 ARTIFACTS ?= $(ROOT_DIR)/_artifacts
 E2E_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/azure-dev.yaml
 E2E_CONF_FILE_ENVSUBST := $(ROOT_DIR)/test/e2e/config/azure-dev-envsubst.yaml
+E2E_CLOUD_PROVIDER_AZURE_PATH ?= $(ROOT_DIR)/templates/caaph/cloud-provider-azure.yaml
+E2E_CLOUD_PROVIDER_AZURE_CI_PATH ?= $(ROOT_DIR)/templates/caaph/cloud-provider-azure-ci.yaml
+E2E_CALICO_PATH ?= $(ROOT_DIR)/templates/caaph/calico.yaml
+E2E_CALICO_IPV6_PATH ?= $(ROOT_DIR)/templates/caaph/calico-ipv6.yaml
+E2E_CALICO_DUAL_STACK_PATH ?= $(ROOT_DIR)/templates/caaph/calico-dual-stack.yaml
+E2E_CALICO_DUAL_STACK_PATH ?= $(ROOT_DIR)/templates/caaph/calico-dual-stack.yaml
+E2E_AZUREDISK_CSI_DRIVER_PATH ?= $(ROOT_DIR)/templates/caaph/azuredisk-csi-driver.yaml
+
 SKIP_CLEANUP ?= false
 SKIP_LOG_COLLECTION ?= false
 SKIP_CREATE_MGMT_CLUSTER ?= false
@@ -649,6 +657,12 @@ test-e2e-run: generate-e2e-templates install-tools ## Run e2e tests.
     $(GINKGO) -v --trace --timeout=4h --tags=e2e --focus="$(GINKGO_FOCUS)" --skip="$(GINKGO_SKIP)" --nodes=$(GINKGO_NODES) --no-color=$(GINKGO_NOCOLOR) --output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) ./test/e2e -- \
     	-e2e.artifacts-folder="$(ARTIFACTS)" \
     	-e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \
+		-e2e.cloud-provider-azure="$(E2E_CLOUD_PROVIDER_AZURE_PATH)" \
+		-e2e.cloud-provider-azure-ci="$(E2E_CLOUD_PROVIDER_AZURE_CI_PATH)" \
+		-e2e.calico="$(E2E_CALICO_PATH)" \
+		-e2e.calico-ipv6="$(E2E_CALICO_IPV6_PATH)" \
+		-e2e.calico-dual-stack="$(E2E_CALICO_DUAL_STACK_PATH)" \
+		-e2e.azuredisk-csi-driver="$(E2E_AZUREDISK_CSI_DRIVER_PATH)" \
     	-e2e.skip-log-collection="$(SKIP_LOG_COLLECTION)" \
     	-e2e.skip-resource-cleanup=$(SKIP_CLEANUP) -e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER) $(E2E_ARGS)
 

config/capz/manager_image_patch.yaml

@@ -8,5 +8,5 @@ spec:
     spec:
       containers:
       # Change the value of image field below to your controller image URL
-      - image: gcr.io/k8s-staging-cluster-api-azure/cluster-api-azure-controller:latest
+      - image: docker.io/jont828/cluster-api-azure-controller-amd64:20230518222332
         name: manager

config/capz/manager_pull_policy.yaml

@@ -8,4 +8,4 @@ spec:
     spec:
       containers:
       - name: manager
-        imagePullPolicy: Always
+        imagePullPolicy: IfNotPresent

go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/Azure/go-autorest/tracing v0.6.0
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d
 	github.com/blang/semver v3.5.1+incompatible
+	github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46
 	github.com/go-logr/logr v1.2.4
 	github.com/golang/mock v1.6.0
 	github.com/google/go-cmp v0.5.9
@@ -47,8 +48,9 @@ require (
 	k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5
 	sigs.k8s.io/cloud-provider-azure v1.26.7
 	sigs.k8s.io/cluster-api v1.4.2
+	sigs.k8s.io/cluster-api-addon-provider-helm v0.1.0-alpha.5
 	sigs.k8s.io/cluster-api/test v1.4.2
-	sigs.k8s.io/controller-runtime v0.14.5
+	sigs.k8s.io/controller-runtime v0.14.6
 	sigs.k8s.io/kind v0.18.0
 )
 
@@ -92,7 +94,6 @@ require (
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
-	github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 // indirect
 	github.com/emicklei/go-restful/v3 v3.10.1 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
@@ -225,4 +226,6 @@ require (
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
-replace sigs.k8s.io/cluster-api => sigs.k8s.io/cluster-api v1.4.2
+replace sigs.k8s.io/cluster-api => github.com/Jont828/cluster-api v0.4.3-0.20230412015901-405900d8bb69
+
+replace sigs.k8s.io/cluster-api/test => github.com/Jont828/cluster-api/test v0.4.3-0.20230412015901-405900d8bb69

go.sum

@@ -92,6 +92,10 @@ github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak
 github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
+github.com/Jont828/cluster-api v0.4.3-0.20230412015901-405900d8bb69 h1:OF4FuySy7PRvHDBezR9KTzgM/lQ2I4L961oh3ByHZt0=
+github.com/Jont828/cluster-api v0.4.3-0.20230412015901-405900d8bb69/go.mod h1:3NCXYKcbsAB5gl1CmLaiXiz1yaYIVhQiB48IxwqGErI=
+github.com/Jont828/cluster-api/test v0.4.3-0.20230412015901-405900d8bb69 h1:G8yGtS6b8BjdFf+NX8G9SOtFIZuzbbLv4lhpujNyEMI=
+github.com/Jont828/cluster-api/test v0.4.3-0.20230412015901-405900d8bb69/go.mod h1:ff2gsEFgyCoglpTVxf1DCR+4itUBJ7MpiaQhZyri2Eg=
 github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ=
 github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE=
 github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
@@ -1300,12 +1304,10 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/cloud-provider-azure v1.26.7 h1:LSVRPyxeTQZPOF7z42zNZGzL1S1N5tg28RBZVy7gea4=
 sigs.k8s.io/cloud-provider-azure v1.26.7/go.mod h1:UIwr0Bk4wQb77wNL9cdT4zZw6DP2AtOQ9EKRt9c5g7Q=
-sigs.k8s.io/cluster-api v1.4.2 h1:hdIz0Ms2j7YaU1qBK5yF2R8ii0GcGb3jQ7EO6i3tAN8=
-sigs.k8s.io/cluster-api v1.4.2/go.mod h1:IIebZTsqyXU8CHbINV2zuMh0/wykqdr+vEXxQNeteEU=
-sigs.k8s.io/cluster-api/test v1.4.2 h1:uHFtn0SFOFOxIbdahLoYo4kz84yLqCmhbVLV4vsk1gQ=
-sigs.k8s.io/cluster-api/test v1.4.2/go.mod h1:/64ycj3YFMW1BGVtCtfwmlVAXGN0DFTZEkIClh68Svo=
-sigs.k8s.io/controller-runtime v0.14.5 h1:6xaWFqzT5KuAQ9ufgUaj1G/+C4Y1GRkhrxl+BJ9i+5s=
-sigs.k8s.io/controller-runtime v0.14.5/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0=
+sigs.k8s.io/cluster-api-addon-provider-helm v0.1.0-alpha.5 h1:F4phJjl9TrdbuRKIQorQo2jQg3AQTgyXkN+PY5E1So4=
+sigs.k8s.io/cluster-api-addon-provider-helm v0.1.0-alpha.5/go.mod h1:s3klPverZSGHRlvJxA35TwtM2+6NeRrcljd3cDsD+Jk=
+sigs.k8s.io/controller-runtime v0.14.6 h1:oxstGVvXGNnMvY7TAESYk+lzr6S3V5VFxQ6d92KcwQA=
+sigs.k8s.io/controller-runtime v0.14.6/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0=
 sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k=
 sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
 sigs.k8s.io/kind v0.18.0 h1:ahgZdVV1pdhXlYe1f+ztISakT23KdrBl/NFY9JMygzs=

internal/test/env/env.go

@@ -35,6 +35,7 @@ import (
 	"k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
 	"k8s.io/utils/pointer"
+	addonsv1alpha1 "sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1"
 	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
 	infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1"
 	"sigs.k8s.io/cluster-api-provider-azure/internal/test/record"
@@ -59,6 +60,7 @@ func init() {
 	utilruntime.Must(expv1.AddToScheme(scheme))
 	utilruntime.Must(infrav1.AddToScheme(scheme))
 	utilruntime.Must(infrav1exp.AddToScheme(scheme))
+	utilruntime.Must(addonsv1alpha1.AddToScheme(scheme))
 
 	// Get the root of the current file to use in CRD paths.
 	_, filename, _, _ := goruntime.Caller(0) //nolint:dogsled // Ignore "declaration has 3 blank identifiers" check.

main.go

@@ -36,6 +36,7 @@ import (
 	cgrecord "k8s.io/client-go/tools/record"
 	"k8s.io/klog/v2"
 	"k8s.io/klog/v2/klogr"
+	addonsv1alpha1 "sigs.k8s.io/cluster-api-addon-provider-helm/api/v1alpha1"
 	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
 	"sigs.k8s.io/cluster-api-provider-azure/controllers"
 	infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1"
@@ -68,6 +69,7 @@ func init() {
 	_ = infrav1exp.AddToScheme(scheme)
 	_ = clusterv1.AddToScheme(scheme)
 	_ = expv1.AddToScheme(scheme)
+	_ = addonsv1alpha1.AddToScheme(scheme)
 	_ = kubeadmv1.AddToScheme(scheme)
 	// +kubebuilder:scaffold:scheme
 

templates/caaph/azuredisk-csi-driver.yaml

@@ -0,0 +1,18 @@
+apiVersion: addons.cluster.x-k8s.io/v1alpha1
+kind: HelmChartProxy
+metadata:
+  name: azuredisk-csi-driver-chart
+spec:
+  clusterSelector:
+    matchLabels:
+      installAzureDiskCSIDriver: "true"
+  repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts
+  chartName: azuredisk-csi-driver
+  releaseName: azuredisk-csi-driver-oot
+  namespace: kube-system
+  valuesTemplate: |
+    controller:
+      replicas: 1
+      runOnControlPlane: true
+    windows:
+      useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }}

templates/caaph/calico-dual-stack.yaml

@@ -0,0 +1,32 @@
+apiVersion: addons.cluster.x-k8s.io/v1alpha1
+kind: HelmChartProxy
+metadata:
+  name: calico-dual-stack-chart
+spec:
+  clusterSelector:
+    matchLabels:
+      cni: calico-dual-stack
+  repoURL: https://docs.tigera.io/calico/charts
+  chartName: tigera-operator
+  releaseName: projectcalico
+  namespace: tigera-operator
+  valuesTemplate: |
+    installation:
+      cni:
+        type: Calico
+        ipam:
+          type: HostLocal
+      calicoNetwork:
+        bgp: Disabled
+        mtu: 1350
+        ipPools:
+        - blockSize: 26
+          cidr: {{ index .Cluster.spec.clusterNetwork.pods.cidrBlocks 0 }}
+          encapsulation: None
+          natOutgoing: Enabled
+          nodeSelector: all()
+        - blockSize: 122
+          cidr: {{ index .Cluster.spec.clusterNetwork.pods.cidrBlocks 1 }}
+          encapsulation: None
+          natOutgoing: Enabled
+          nodeSelector: all()

templates/caaph/calico-ipv6.yaml

@@ -0,0 +1,27 @@
+apiVersion: addons.cluster.x-k8s.io/v1alpha1
+kind: HelmChartProxy
+metadata:
+  name: calico-ipv6-chart
+spec:
+  clusterSelector:
+    matchLabels:
+      cni: calico-ipv6
+  repoURL: https://docs.tigera.io/calico/charts
+  chartName: tigera-operator
+  releaseName: projectcalico
+  namespace: tigera-operator
+  valuesTemplate: |
+    installation:
+      cni:
+        type: Calico
+        ipam:
+          type: HostLocal
+      calicoNetwork:
+        bgp: Disabled
+        mtu: 1350
+        ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }}
+        - blockSize: 122
+          cidr: {{ $cidr }}
+          encapsulation: None
+          natOutgoing: Enabled
+          nodeSelector: all(){{end}}

templates/caaph/calico.yaml

@@ -0,0 +1,23 @@
+apiVersion: addons.cluster.x-k8s.io/v1alpha1
+kind: HelmChartProxy
+metadata:
+  name: calico-chart
+spec:
+  clusterSelector:
+    matchLabels:
+      cni: calico
+  repoURL: https://docs.tigera.io/calico/charts
+  chartName: tigera-operator
+  releaseName: projectcalico
+  namespace: tigera-operator
+  valuesTemplate: |
+    installation:
+      cni:
+        type: Calico
+      calicoNetwork:
+        bgp: Disabled
+        mtu: 1350
+        ipPools:
+        ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }}
+        - cidr: {{ $cidr }}
+          encapsulation: VXLAN{{end}}

templates/caaph/cloud-provider-azure-ci.yaml

@@ -0,0 +1,25 @@
+apiVersion: addons.cluster.x-k8s.io/v1alpha1
+kind: HelmChartProxy
+metadata:
+  name: cloud-provider-azure-chart-ci
+spec:
+  clusterSelector:
+    matchLabels:
+      installCloudProviderAzure: "true"
+  repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo
+  chartName: cloud-provider-azure
+  releaseName: cloud-provider-azure
+  valuesTemplate: |
+    infra:
+      clusterName: {{ .Cluster.metadata.name }}
+    cloudControllerManager:
+      clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }}
+      imageName: ${CCM_IMAGE_NAME}
+      imageRepository: ${CCM_IMAGE_REGISTRY}
+      imageTag: ${IMAGE_TAG_CCM}
+      logVerbosity: 4
+    cloudNodeManager:
+      imageName: ${CNM_IMAGE_NAME}
+      imageRepository: ${CNM_IMAGE_REGISTRY}
+      imageTag: ${IMAGE_TAG_CCM}
+

templates/caaph/cloud-provider-azure.yaml

@@ -0,0 +1,17 @@
+apiVersion: addons.cluster.x-k8s.io/v1alpha1
+kind: HelmChartProxy
+metadata:
+  name: cloud-provider-azure-chart
+spec:
+  clusterSelector:
+    matchLabels:
+      installCloudProviderAzure: "true"
+  repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo
+  chartName: cloud-provider-azure
+  releaseName: cloud-provider-azure
+  valuesTemplate: |
+    infra:
+      clusterName: {{ .Cluster.metadata.name }}
+    cloudControllerManager:
+      clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }}
+      logVerbosity: 4