diff --git a/charts/internal/shoot-system-components/charts/csi-driver-node/templates/daemonset.yaml b/charts/internal/shoot-system-components/charts/csi-driver-node/templates/daemonset.yaml index fd0a01e20..da64e8c1a 100644 --- a/charts/internal/shoot-system-components/charts/csi-driver-node/templates/daemonset.yaml +++ b/charts/internal/shoot-system-components/charts/csi-driver-node/templates/daemonset.yaml @@ -34,6 +34,9 @@ spec: args: - node - --endpoint=$(CSI_ENDPOINT) + {{- if .Values.driver.volumeAttachLimit }} + - --volume-attach-limit={{ .Values.driver.volumeAttachLimit }} + {{- end }} - --logtostderr - --v=5 env: diff --git a/charts/internal/shoot-system-components/charts/csi-driver-node/values.yaml b/charts/internal/shoot-system-components/charts/csi-driver-node/values.yaml index d9ce44200..109dbdac8 100644 --- a/charts/internal/shoot-system-components/charts/csi-driver-node/values.yaml +++ b/charts/internal/shoot-system-components/charts/csi-driver-node/values.yaml @@ -6,6 +6,9 @@ images: socketPath: /csi/csi.sock vpaEnabled: false +driver: {} + # volumeAttachLimit: -1 + resources: driver: requests: diff --git a/docs/usage-as-end-user.md b/docs/usage-as-end-user.md index 1e43952ce..4d6f4477b 100644 --- a/docs/usage-as-end-user.md +++ b/docs/usage-as-end-user.md @@ -308,3 +308,9 @@ Every AWS shoot cluster that has at least Kubernetes v1.18 will be deployed with It is compatible with the legacy in-tree volume provisioner that was deprecated by the Kubernetes community and will be removed in future versions of Kubernetes. End-users might want to update their custom `StorageClass`es to the new `ebs.csi.aws.com` provisioner. Shoot clusters with Kubernetes v1.17 or less will use the in-tree `kubernetes.io/aws-ebs` volume provisioner in the kube-controller-manager and the kubelet. + +### Node-specific Volume Limits + +The Kubernetes scheduler allows configurable limit for the number of volumes that can be attached to a node. See https://k8s.io/docs/concepts/storage/storage-limits/#custom-limits. + +CSI drivers usually have a different procedure for configuring this custom limit. By default, the EBS CSI driver parses the machine type name and then decides the volume limit. However, this is only a rough approximation and not good enough in most cases. Specifying the volume attach limit via command line flag (`--volume-attach-limit`) is currently the alternative until a more sophisticated solution presents itself (dynamically discovering the maximum number of attachable volume per EC2 machine type, see also https://github.com/kubernetes-sigs/aws-ebs-csi-driver/issues/347). The AWS extension allows the `--volume-attach-limit` flag of the EBS CSI driver to be configurable via `aws.provider.extensions.gardener.cloud/volume-attach-limit` annotation on the `Shoot` resource. If the annotation is added to an existing `Shoot`, then reconciliation needs to be triggered manually (see [Immediate reconciliation](https://github.com/gardener/gardener/blob/master/docs/usage/shoot_operations.md#immediate-reconciliation)), as in general adding annotation to resource is not a change that leads to `.metadata.generation` increase in general. diff --git a/pkg/aws/types.go b/pkg/aws/types.go index 717a4a041..61a328269 100644 --- a/pkg/aws/types.go +++ b/pkg/aws/types.go @@ -24,6 +24,10 @@ const ( // Name is the name of the AWS provider. Name = "provider-aws" + // VolumeAttachLimit is the key for an annotation on a Shoot object whose value + // represents the maximum number of volumes attachable for all nodes. + VolumeAttachLimit = "aws.provider.extensions.gardener.cloud/volume-attach-limit" + // AWSLBReadvertiserImageName is the name of the AWSLBReadvertiser image. AWSLBReadvertiserImageName = "aws-lb-readvertiser" // CloudControllerManagerImageName is the name of the cloud-controller-manager image. diff --git a/pkg/controller/controlplane/valuesprovider.go b/pkg/controller/controlplane/valuesprovider.go index 8af2cd6b4..6d82cc8af 100644 --- a/pkg/controller/controlplane/valuesprovider.go +++ b/pkg/controller/controlplane/valuesprovider.go @@ -526,11 +526,19 @@ func getControlPlaneShootChartValues( return nil, err } + csiDriverNodeValues := map[string]interface{}{ + "enabled": !k8sVersionLessThan118, + "vpaEnabled": gardencorev1beta1helper.ShootWantsVerticalPodAutoscaler(cluster.Shoot), + } + + if value, ok := cluster.Shoot.Annotations[aws.VolumeAttachLimit]; ok { + csiDriverNodeValues["driver"] = map[string]interface{}{ + "volumeAttachLimit": value, + } + } + return map[string]interface{}{ aws.CloudControllerManagerName: map[string]interface{}{"enabled": true}, - aws.CSINodeName: map[string]interface{}{ - "enabled": !k8sVersionLessThan118, - "vpaEnabled": gardencorev1beta1helper.ShootWantsVerticalPodAutoscaler(cluster.Shoot), - }, + aws.CSINodeName: csiDriverNodeValues, }, nil } diff --git a/pkg/controller/controlplane/valuesprovider_test.go b/pkg/controller/controlplane/valuesprovider_test.go index 3b8f816a9..5f55fea79 100644 --- a/pkg/controller/controlplane/valuesprovider_test.go +++ b/pkg/controller/controlplane/valuesprovider_test.go @@ -103,6 +103,11 @@ var _ = Describe("ValuesProvider", func() { } clusterK8sAtLeast118 = &extensionscontroller.Cluster{ Shoot: &gardencorev1beta1.Shoot{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + aws.VolumeAttachLimit: "42", + }, + }, Spec: gardencorev1beta1.ShootSpec{ Networking: gardencorev1beta1.Networking{ Pods: &cidr, @@ -236,6 +241,9 @@ var _ = Describe("ValuesProvider", func() { aws.CloudControllerManagerName: enabledTrue, aws.CSINodeName: utils.MergeMaps(enabledTrue, map[string]interface{}{ "vpaEnabled": true, + "driver": map[string]interface{}{ + "volumeAttachLimit": "42", + }, }), })) })