fix: ensure addon hostNetwork ports don't conflict (Azure#3894)

penggu · Oct 28, 2020 · a519330 · a519330
1 parent d26aa60
commit a519330
Show file tree

Hide file tree

Showing 6 changed files with 32 additions and 9 deletions.
diff --git a/parts/k8s/addons/aad-pod-identity.yaml b/parts/k8s/addons/aad-pod-identity.yaml
@@ -155,6 +155,7 @@ spec:
         args:
           - "--host-ip=$(HOST_IP)"
           - "--node=$(NODE_NAME)"
+          - "--http-probe-port={{ContainerConfig "probePort"}}"
         env:
           - name: HOST_IP
             valueFrom:
@@ -182,7 +183,7 @@ spec:
         livenessProbe:
           httpGet:
             path: /healthz
-            port: 8080
+            port: {{ContainerConfig "probePort"}}
           initialDelaySeconds: 10
           periodSeconds: 5
       nodeSelector:

diff --git a/parts/k8s/addons/secrets-store-csi-driver.yaml b/parts/k8s/addons/secrets-store-csi-driver.yaml
@@ -343,7 +343,7 @@ spec:
             - "--nodeid=$(KUBE_NODE_NAME)"
             - "--provider-volume=/etc/kubernetes/secrets-store-csi-providers"
             - "--grpc-supported-providers=azure"
-            - "--metrics-addr=:8080"
+            - "--metrics-addr=:{{ContainerConfig "metricsPort"}}"
           env:
             - name: CSI_ENDPOINT
               value: unix:///csi/csi.sock

diff --git a/pkg/api/addons.go b/pkg/api/addons.go
@@ -456,6 +456,9 @@ func (cs *ContainerService) setAddonsConfig(isUpgrade bool) {
 	defaultsAADPodIdentityAddonsConfig := KubernetesAddon{
 		Name:    common.AADPodIdentityAddonName,
 		Enabled: to.BoolPtr(DefaultAADPodIdentityAddonEnabled && !cs.Properties.IsAzureStackCloud()),
+		Config: map[string]string{
+			"probePort": "8085",
+		},
 		Containers: []KubernetesContainerSpec{
 			{
 				Name:           common.NMIContainerName,
@@ -839,6 +842,9 @@ func (cs *ContainerService) setAddonsConfig(isUpgrade bool) {
 
 	defaultSecretsStoreCSIDriverAddonsConfig := KubernetesAddon{
 		Name: common.SecretsStoreCSIDriverAddonName,
+		Config: map[string]string{
+			"metricsPort": "8095",
+		},
 		Enabled: to.BoolPtr(!o.KubernetesConfig.IsAddonEnabled(common.KeyVaultFlexVolumeAddonName) && DefaultSecretStoreCSIDriverAddonEnabled &&
 			common.IsKubernetesVersionGe(o.OrchestratorVersion, "1.16.0")),
 		Containers: []KubernetesContainerSpec{

diff --git a/pkg/api/addons_test.go b/pkg/api/addons_test.go
@@ -1916,6 +1916,9 @@ func TestSetAddonsConfig(t *testing.T) {
 				{
 					Name:    common.AADPodIdentityAddonName,
 					Enabled: to.BoolPtr(true),
+					Config: map[string]string{
+						"probePort": "8085",
+					},
 					Containers: []KubernetesContainerSpec{
 						{
 							Name:           common.NMIContainerName,
@@ -4425,6 +4428,9 @@ func TestSetAddonsConfig(t *testing.T) {
 				{
 					Name:    common.SecretsStoreCSIDriverAddonName,
 					Enabled: to.BoolPtr(true),
+					Config: map[string]string{
+						"metricsPort": "8095",
+					},
 				},
 			}, "1.15.4"),
 		},
@@ -5200,6 +5206,9 @@ func getDefaultAddons(version, kubernetesImageBase, kubernetesImageBaseType stri
 		addons = append(addons, KubernetesAddon{
 			Name:    common.SecretsStoreCSIDriverAddonName,
 			Enabled: to.BoolPtr(true),
+			Config: map[string]string{
+				"metricsPort": "8095",
+			},
 			Containers: []KubernetesContainerSpec{
 				{
 					Name:           common.CSILivenessProbeContainerName,

diff --git a/pkg/engine/templates_generated.go b/pkg/engine/templates_generated.go
diff --git a/test/e2e/kubernetes/pod/pod.go b/test/e2e/kubernetes/pod/pod.go
@@ -800,10 +800,16 @@ func AreAllPodsRunning(podPrefix, namespace string) (bool, error) {
 			return false, regexErr
 		}
 		if matched {
-			if pod.Status.Phase != "Running" {
+			if pod.Status.Phase == "Running" {
+				for _, containerStatus := range pod.Status.ContainerStatuses {
+					if containerStatus.Ready {
+						status = append(status, true)
+					} else {
+						status = append(status, false)
+					}
+				}
+			} else if pod.Status.Phase != "Pending" && pod.Status.Phase != "ImagePullBackOff" && pod.Status.Phase != "ContainerCreating" {
 				status = append(status, false)
-			} else {
-				status = append(status, true)
 			}
 		}
 	}
@@ -814,11 +820,11 @@ func AreAllPodsRunning(podPrefix, namespace string) (bool, error) {
 
 	for _, s := range status {
 		if !s {
-			return false, nil
+			return false, errors.Errorf("At least one pod has a container in a non-Ready state")
 		}
 	}
 
-	return true, err
+	return true, nil
 }
 
 // AreAllPodsSucceededResult is a return struct for AreAllPodsSucceeded