diff --git a/azure/converters/managedagentpool.go b/azure/converters/managedagentpool.go index 7fd8f01d499..5709da35b84 100644 --- a/azure/converters/managedagentpool.go +++ b/azure/converters/managedagentpool.go @@ -46,6 +46,7 @@ func AgentPoolToManagedClusterAgentPoolProfile(pool containerservice.AgentPool) NodePublicIPPrefixID: properties.NodePublicIPPrefixID, ScaleSetPriority: properties.ScaleSetPriority, Tags: properties.Tags, + KubeletDiskType: properties.KubeletDiskType, } if properties.KubeletConfig != nil { agentPool.KubeletConfig = properties.KubeletConfig diff --git a/azure/scope/managedmachinepool.go b/azure/scope/managedmachinepool.go index 7bdc5b10e1f..a6a32e9b976 100644 --- a/azure/scope/managedmachinepool.go +++ b/azure/scope/managedmachinepool.go @@ -179,6 +179,7 @@ func buildAgentPoolSpec(managedControlPlane *infrav1exp.AzureManagedControlPlane NodePublicIPPrefixID: managedMachinePool.Spec.NodePublicIPPrefixID, ScaleSetPriority: managedMachinePool.Spec.ScaleSetPriority, AdditionalTags: managedMachinePool.Spec.AdditionalTags, + KubeletDiskType: managedMachinePool.Spec.KubeletDiskType, } if managedMachinePool.Spec.OSDiskSizeGB != nil { diff --git a/azure/scope/managedmachinepool_test.go b/azure/scope/managedmachinepool_test.go index afdead1a82e..6f1c3efe286 100644 --- a/azure/scope/managedmachinepool_test.go +++ b/azure/scope/managedmachinepool_test.go @@ -624,6 +624,101 @@ func TestManagedMachinePoolScope_OSDiskType(t *testing.T) { } } +func TestManagedMachinePoolScope_KubeletDiskType(t *testing.T) { + scheme := runtime.NewScheme() + _ = expv1.AddToScheme(scheme) + _ = infrav1exp.AddToScheme(scheme) + + cases := []struct { + Name string + Input ManagedMachinePoolScopeParams + Expected azure.ResourceSpecGetter + }{ + { + Name: "Without KubeletDiskType", + Input: ManagedMachinePoolScopeParams{ + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1exp.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1exp.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, + }, + ManagedMachinePool: ManagedMachinePool{ + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1exp.NodePoolModeSystem), + }, + }, + Expected: &agentpools.AgentPoolSpec{ + Name: "pool0", + SKU: "Standard_D2s_v3", + Replicas: 1, + Mode: "System", + Cluster: "cluster1", + VnetSubnetID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups//providers/Microsoft.Network/virtualNetworks//subnets/", + Headers: map[string]string{}, + }, + }, + { + Name: "With KubeletDiskType", + Input: ManagedMachinePoolScopeParams{ + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1exp.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1exp.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, + }, + ManagedMachinePool: ManagedMachinePool{ + MachinePool: getMachinePool("pool1"), + InfraMachinePool: getAzureMachinePoolWithKubeletDiskType("pool1", (*infrav1exp.KubeletDiskType)(to.StringPtr("Temporary"))), + }, + }, + Expected: &agentpools.AgentPoolSpec{ + Name: "pool1", + SKU: "Standard_D2s_v3", + Mode: "User", + Cluster: "cluster1", + Replicas: 1, + KubeletDiskType: (*infrav1exp.KubeletDiskType)(to.StringPtr("Temporary")), + VnetSubnetID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups//providers/Microsoft.Network/virtualNetworks//subnets/", + Headers: map[string]string{}, + }, + }, + } + + for _, c := range cases { + c := c + t.Run(c.Name, func(t *testing.T) { + g := NewWithT(t) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(c.Input.MachinePool, c.Input.InfraMachinePool, c.Input.ControlPlane).Build() + c.Input.Client = fakeClient + s, err := NewManagedMachinePoolScope(context.TODO(), c.Input) + g.Expect(err).To(Succeed()) + agentPool := s.AgentPoolSpec() + if !reflect.DeepEqual(c.Expected, agentPool) { + t.Errorf("Got difference between expected result and result:\n%s", cmp.Diff(c.Expected, agentPool)) + } + }) + } +} + func getAzureMachinePool(name string, mode infrav1exp.NodePoolMode) *infrav1exp.AzureManagedMachinePool { return &infrav1exp.AzureManagedMachinePool{ ObjectMeta: metav1.ObjectMeta{ @@ -675,6 +770,12 @@ func getAzureMachinePoolWithOsDiskType(name string, osDiskType string) *infrav1e return managedPool } +func getAzureMachinePoolWithKubeletDiskType(name string, kubeletDiskType *infrav1exp.KubeletDiskType) *infrav1exp.AzureManagedMachinePool { + managedPool := getAzureMachinePool(name, infrav1exp.NodePoolModeUser) + managedPool.Spec.KubeletDiskType = kubeletDiskType + return managedPool +} + func getAzureMachinePoolWithLabels(name string, nodeLabels map[string]string) *infrav1exp.AzureManagedMachinePool { managedPool := getAzureMachinePool(name, infrav1exp.NodePoolModeSystem) managedPool.Spec.NodeLabels = nodeLabels diff --git a/azure/services/agentpools/agentpools_test.go b/azure/services/agentpools/agentpools_test.go index c2addf3333e..80362e16962 100644 --- a/azure/services/agentpools/agentpools_test.go +++ b/azure/services/agentpools/agentpools_test.go @@ -29,6 +29,7 @@ import ( "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/services/agentpools/mock_agentpools" "sigs.k8s.io/cluster-api-provider-azure/azure/services/async/mock_async" + infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" gomockinternal "sigs.k8s.io/cluster-api-provider-azure/internal/test/matchers/gomock" ) @@ -54,6 +55,7 @@ var ( EnableUltraSSD: to.BoolPtr(true), OSType: to.StringPtr("fake-os-type"), Headers: map[string]string{"fake-header": "fake-value"}, + KubeletDiskType: (*infrav1exp.KubeletDiskType)(to.StringPtr("fake-kubelet-disk-type")), } internalError = autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: http.StatusInternalServerError}, "Internal Server Error") diff --git a/azure/services/agentpools/spec.go b/azure/services/agentpools/spec.go index b757d0c16bd..c31ae1c7513 100644 --- a/azure/services/agentpools/spec.go +++ b/azure/services/agentpools/spec.go @@ -28,6 +28,7 @@ import ( infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/converters" + infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" azureutil "sigs.k8s.io/cluster-api-provider-azure/util/azure" ) @@ -131,6 +132,9 @@ type AgentPoolSpec struct { // KubeletConfig specifies the kubelet configurations for nodes. KubeletConfig *KubeletConfig `json:"kubeletConfig,omitempty"` + // KubeletDiskType specifies the kubelet disk type for each node in the pool. Allowed values are 'OS' and 'Temporary' + KubeletDiskType *infrav1exp.KubeletDiskType `json:"kubeletDiskType,omitempty"` + // AdditionalTags is an optional set of tags to add to Azure resources managed by the Azure provider, in addition to the ones added by default. AdditionalTags infrav1.Tags } @@ -279,6 +283,7 @@ func (s *AgentPoolSpec) Parameters(ctx context.Context, existing interface{}) (p EnableAutoScaling: s.EnableAutoScaling, EnableUltraSSD: s.EnableUltraSSD, KubeletConfig: kubeletConfig, + KubeletDiskType: containerservice.KubeletDiskType(to.String((*string)(s.KubeletDiskType))), MaxCount: s.MaxCount, MaxPods: s.MaxPods, MinCount: s.MinCount, diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml index d4c2661bb34..a1ed626bd12 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml @@ -289,6 +289,14 @@ spec: - single-numa-node type: string type: object + kubeletDiskType: + description: 'KubeletDiskType specifies the kubelet disk type. Default + to OS. Possible values include: ''OS'', ''Temporary''. Requires + kubeletDisk preview feature to be set.' + enum: + - OS + - Temporary + type: string maxPods: description: MaxPods specifies the kubelet --max-pods configuration for the node pool. diff --git a/docs/book/src/topics/managedcluster.md b/docs/book/src/topics/managedcluster.md index b9670e27204..2bfbd7e4718 100644 --- a/docs/book/src/topics/managedcluster.md +++ b/docs/book/src/topics/managedcluster.md @@ -389,6 +389,30 @@ spec: osDiskType: "Ephemeral" ``` +## AKS Node Pool KubeletDiskType configuration + +You can configure the `KubeletDiskType` value for each AKS node pool (`AzureManagedMachinePool`) that you define in your spec (see [here](https://learn.microsoft.com/en-us/rest/api/aks/agent-pools/create-or-update?tabs=HTTP#kubeletdisktype) for the official AKS documentation). There are two options to choose from: `"OS"` or `"Temporary"`. + +Before this feature can be used, you must register the `KubeletDisk` feature on your Azure subscription with the following az cli command. + +```bash +az feature register --namespace Microsoft.ContainerService --name KubeletDisk +``` + +Below an example `kubeletDiskType` configuration is assigned to `agentpool0`, specifying that the emptyDir volumes, container runtime data root, and Kubelet ephemeral storage will be stored on the temporary disk: + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePool +metadata: + name: agentpool0 +spec: + mode: System + osDiskSizeGB: 30 + sku: Standard_D2s_v3 + kubeletDiskType: "Temporary" +``` + ### AKS Node Pool Taints You can configure the `Taints` value for each AKS node pool (`AzureManagedMachinePool`) that you define in your spec. diff --git a/exp/api/v1alpha3/azuremanagedmachinepool_conversion.go b/exp/api/v1alpha3/azuremanagedmachinepool_conversion.go index 8314f3eb8ff..29126e7e938 100644 --- a/exp/api/v1alpha3/azuremanagedmachinepool_conversion.go +++ b/exp/api/v1alpha3/azuremanagedmachinepool_conversion.go @@ -49,6 +49,7 @@ func (src *AzureManagedMachinePool) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.NodePublicIPPrefixID = restored.Spec.NodePublicIPPrefixID dst.Spec.ScaleSetPriority = restored.Spec.ScaleSetPriority dst.Spec.AdditionalTags = restored.Spec.AdditionalTags + dst.Spec.KubeletDiskType = restored.Spec.KubeletDiskType if restored.Spec.KubeletConfig != nil { dst.Spec.KubeletConfig = restored.Spec.KubeletConfig } diff --git a/exp/api/v1alpha3/zz_generated.conversion.go b/exp/api/v1alpha3/zz_generated.conversion.go index 1df50cfeb61..7c4acc85a27 100644 --- a/exp/api/v1alpha3/zz_generated.conversion.go +++ b/exp/api/v1alpha3/zz_generated.conversion.go @@ -923,6 +923,7 @@ func autoConvert_v1beta1_AzureManagedMachinePoolSpec_To_v1alpha3_AzureManagedMac // WARNING: in.NodePublicIPPrefixID requires manual conversion: does not exist in peer-type // WARNING: in.ScaleSetPriority requires manual conversion: does not exist in peer-type // WARNING: in.KubeletConfig requires manual conversion: does not exist in peer-type + // WARNING: in.KubeletDiskType requires manual conversion: does not exist in peer-type return nil } diff --git a/exp/api/v1alpha4/azuremanagedmachinepool_conversion.go b/exp/api/v1alpha4/azuremanagedmachinepool_conversion.go index 7dca4b8b3d5..90ba3ce0f31 100644 --- a/exp/api/v1alpha4/azuremanagedmachinepool_conversion.go +++ b/exp/api/v1alpha4/azuremanagedmachinepool_conversion.go @@ -49,6 +49,7 @@ func (src *AzureManagedMachinePool) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.NodePublicIPPrefixID = restored.Spec.NodePublicIPPrefixID dst.Spec.ScaleSetPriority = restored.Spec.ScaleSetPriority dst.Spec.AdditionalTags = restored.Spec.AdditionalTags + dst.Spec.KubeletDiskType = restored.Spec.KubeletDiskType if restored.Spec.KubeletConfig != nil { dst.Spec.KubeletConfig = restored.Spec.KubeletConfig } diff --git a/exp/api/v1alpha4/zz_generated.conversion.go b/exp/api/v1alpha4/zz_generated.conversion.go index 0d1d238f5d1..d43f26193d5 100644 --- a/exp/api/v1alpha4/zz_generated.conversion.go +++ b/exp/api/v1alpha4/zz_generated.conversion.go @@ -1223,6 +1223,7 @@ func autoConvert_v1beta1_AzureManagedMachinePoolSpec_To_v1alpha4_AzureManagedMac // WARNING: in.NodePublicIPPrefixID requires manual conversion: does not exist in peer-type // WARNING: in.ScaleSetPriority requires manual conversion: does not exist in peer-type // WARNING: in.KubeletConfig requires manual conversion: does not exist in peer-type + // WARNING: in.KubeletDiskType requires manual conversion: does not exist in peer-type return nil } diff --git a/exp/api/v1beta1/azuremanagedmachinepool_types.go b/exp/api/v1beta1/azuremanagedmachinepool_types.go index 6a977b569c2..5c09c0d0895 100644 --- a/exp/api/v1beta1/azuremanagedmachinepool_types.go +++ b/exp/api/v1beta1/azuremanagedmachinepool_types.go @@ -54,6 +54,16 @@ const ( // TopologyManagerPolicy enumerates the values for KubeletConfig.TopologyManagerPolicy. type TopologyManagerPolicy string +// KubeletDiskType enumerates the values for the agent pool's KubeletDiskType. +type KubeletDiskType string + +const ( + // KubeletDiskTypeOS ... + KubeletDiskTypeOS KubeletDiskType = "OS" + // KubeletDiskTypeTemporary ... + KubeletDiskTypeTemporary KubeletDiskType = "Temporary" +) + const ( // TopologyManagerPolicyNone ... TopologyManagerPolicyNone TopologyManagerPolicy = "none" @@ -189,6 +199,12 @@ type AzureManagedMachinePoolSpec struct { // KubeletConfig specifies the kubelet configurations for nodes. // +optional KubeletConfig *KubeletConfig `json:"kubeletConfig,omitempty"` + + // KubeletDiskType specifies the kubelet disk type. Default to OS. Possible values include: 'OS', 'Temporary'. + // Requires kubeletDisk preview feature to be set. + // +kubebuilder:validation:Enum=OS;Temporary + // +optional + KubeletDiskType *KubeletDiskType `json:"kubeletDiskType,omitempty"` } // ManagedMachinePoolScaling specifies scaling options. diff --git a/exp/api/v1beta1/azuremanagedmachinepool_webhook.go b/exp/api/v1beta1/azuremanagedmachinepool_webhook.go index e244714d954..10f6a4b2dc8 100644 --- a/exp/api/v1beta1/azuremanagedmachinepool_webhook.go +++ b/exp/api/v1beta1/azuremanagedmachinepool_webhook.go @@ -206,6 +206,13 @@ func (m *AzureManagedMachinePool) ValidateUpdate(oldRaw runtime.Object, client c allErrs = append(allErrs, err) } + if err := webhookutils.ValidateImmutable( + field.NewPath("Spec", "KubeletDiskType"), + old.Spec.KubeletDiskType, + m.Spec.KubeletDiskType); err != nil { + allErrs = append(allErrs, err) + } + if len(allErrs) != 0 { return apierrors.NewInvalid(GroupVersion.WithKind("AzureManagedMachinePool").GroupKind(), m.Name, allErrs) } diff --git a/exp/api/v1beta1/zz_generated.deepcopy.go b/exp/api/v1beta1/zz_generated.deepcopy.go index 79ac9e8031a..cbd74fef170 100644 --- a/exp/api/v1beta1/zz_generated.deepcopy.go +++ b/exp/api/v1beta1/zz_generated.deepcopy.go @@ -883,6 +883,11 @@ func (in *AzureManagedMachinePoolSpec) DeepCopyInto(out *AzureManagedMachinePool *out = new(KubeletConfig) (*in).DeepCopyInto(*out) } + if in.KubeletDiskType != nil { + in, out := &in.KubeletDiskType, &out.KubeletDiskType + *out = new(KubeletDiskType) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedMachinePoolSpec.