Allow passing --volume-attach-limit to the csi driver

Signed-off-by: ialidzhikov <[email protected]>

charts/internal/shoot-system-components/charts/csi-driver-node/templates/daemonset.yaml

@@ -34,6 +34,9 @@ spec:
         args:
         - node
         - --endpoint=$(CSI_ENDPOINT)
+        {{- if .Values.driver.volumeAttachLimit }}
+        - --volume-attach-limit={{ .Values.driver.volumeAttachLimit }}
+        {{- end }}
         - --logtostderr
         - --v=5
         env:

charts/internal/shoot-system-components/charts/csi-driver-node/values.yaml

@@ -6,6 +6,9 @@ images:
 socketPath: /csi/csi.sock
 vpaEnabled: false
 
+driver: {}
+  # volumeAttachLimit: -1
+
 resources:
   driver:
     requests:

docs/usage-as-end-user.md

@@ -308,3 +308,9 @@ Every AWS shoot cluster that has at least Kubernetes v1.18 will be deployed with
 It is compatible with the legacy in-tree volume provisioner that was deprecated by the Kubernetes community and will be removed in future versions of Kubernetes.
 End-users might want to update their custom `StorageClass`es to the new `ebs.csi.aws.com` provisioner.
 Shoot clusters with Kubernetes v1.17 or less will use the in-tree `kubernetes.io/aws-ebs` volume provisioner in the kube-controller-manager and the kubelet.
+
+#### Node-specific Volume Limits
+
+The kubernetes scheduler allows configurable limit for the number of volumes that can be attached to a Node. See https://k8s.io/docs/concepts/storage/storage-limits/#custom-limits.
+
+CSI drivers usually have a different procedure for configuring this custom limit. By default, the EBS CSI driver parses the machine type name and then decides the volume limit. However, this is only a rough approximation and not good enough in most cases. Specifying the volume attach limit via command line flag (`--volume-attach-limit`) is currently the alternative until a more sophisticated solution presents itself (dynamically discovering the maximum number of attachable volume per EC2 machine type, see also https://github.com/kubernetes-sigs/aws-ebs-csi-driver/issues/347). The AWS extension allows the `--volume-attach-limit` flag of the EBS CSI driver to be configurable via `aws.provider.extensions.gardener.cloud/volume-attach-limit` annotation on the Shoot resource. If the annotation is added to existing Shoot, then reconciliation needs to be triggered manually (as adding annotation to resource is not a change that leads to `.metadata.generation` increase in general).

pkg/aws/types.go

@@ -24,6 +24,10 @@ const (
 	// Name is the name of the AWS provider.
 	Name = "provider-aws"
 
+	// VolumeAttachLimit is the key for an annotation on a Shoot object whose value
+	// represents the maximum number of volumes attachable for all nodes.
+	VolumeAttachLimit = "aws.provider.extensions.gardener.cloud/volume-attach-limit"
+
 	// AWSLBReadvertiserImageName is the name of the AWSLBReadvertiser image.
 	AWSLBReadvertiserImageName = "aws-lb-readvertiser"
 	// CloudControllerManagerImageName is the name of the cloud-controller-manager image.

pkg/controller/controlplane/valuesprovider.go

@@ -526,11 +526,19 @@ func getControlPlaneShootChartValues(
 		return nil, err
 	}
 
+	csiDriverNodeValues := map[string]interface{}{
+		"enabled":    !k8sVersionLessThan118,
+		"vpaEnabled": gardencorev1beta1helper.ShootWantsVerticalPodAutoscaler(cluster.Shoot),
+	}
+
+	if value, ok := cluster.Shoot.Annotations[aws.VolumeAttachLimit]; ok {
+		csiDriverNodeValues["driver"] = map[string]interface{}{
+			"volumeAttachLimit": value,
+		}
+	}
+
 	return map[string]interface{}{
 		aws.CloudControllerManagerName: map[string]interface{}{"enabled": true},
-		aws.CSINodeName: map[string]interface{}{
-			"enabled":    !k8sVersionLessThan118,
-			"vpaEnabled": gardencorev1beta1helper.ShootWantsVerticalPodAutoscaler(cluster.Shoot),
-		},
+		aws.CSINodeName:                csiDriverNodeValues,
 	}, nil
 }

pkg/controller/controlplane/valuesprovider_test.go

@@ -103,6 +103,11 @@ var _ = Describe("ValuesProvider", func() {
 		}
 		clusterK8sAtLeast118 = &extensionscontroller.Cluster{
 			Shoot: &gardencorev1beta1.Shoot{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						aws.VolumeAttachLimit: "42",
+					},
+				},
 				Spec: gardencorev1beta1.ShootSpec{
 					Networking: gardencorev1beta1.Networking{
 						Pods: &cidr,
@@ -236,6 +241,9 @@ var _ = Describe("ValuesProvider", func() {
 				aws.CloudControllerManagerName: enabledTrue,
 				aws.CSINodeName: utils.MergeMaps(enabledTrue, map[string]interface{}{
 					"vpaEnabled": true,
+					"driver": map[string]interface{}{
+						"volumeAttachLimit": "42",
+					},
 				}),
 			}))
 		})