diff --git a/api/v1beta1/azuremachine_types.go b/api/v1beta1/azuremachine_types.go index b4ae88ac83d..7d68a172eb0 100644 --- a/api/v1beta1/azuremachine_types.go +++ b/api/v1beta1/azuremachine_types.go @@ -132,6 +132,12 @@ type AzureMachineSpec struct { // +optional DNSServers []string `json:"dnsServers,omitempty"` + // DisableExtensionOperations specifies whether extension operations should be disabled on the virtual machine. + // Use this setting only if VMExtensions are not supported by your image, as it disables CAPZ bootstrapping extension used for detecting Kubernetes bootstrap failure. + // This may only be set to True when no extensions are configured on the virtual machine. + // +optional + DisableExtensionOperations *bool `json:"disableExtensionOperations,omitempty"` + // VMExtensions specifies a list of extensions to be added to the virtual machine. // +optional VMExtensions []VMExtension `json:"vmExtensions,omitempty"` diff --git a/api/v1beta1/azuremachine_validation.go b/api/v1beta1/azuremachine_validation.go index c30bba98858..972e9816b3c 100644 --- a/api/v1beta1/azuremachine_validation.go +++ b/api/v1beta1/azuremachine_validation.go @@ -24,6 +24,7 @@ import ( "github.com/google/uuid" "golang.org/x/crypto/ssh" "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/utils/ptr" azureutil "sigs.k8s.io/cluster-api-provider-azure/util/azure" ) @@ -71,6 +72,10 @@ func ValidateAzureMachineSpec(spec AzureMachineSpec) field.ErrorList { allErrs = append(allErrs, errs...) } + if errs := ValidateVMExtensions(spec.DisableExtensionOperations, spec.VMExtensions, field.NewPath("vmExtensions")); len(errs) > 0 { + allErrs = append(allErrs, errs...) + } + return allErrs } @@ -471,3 +476,14 @@ func ValidateCapacityReservationGroupID(capacityReservationGroupID *string, fldP return allErrs } + +// ValidateVMExtensions validates the VMExtensions spec. +func ValidateVMExtensions(disableExtensionOperations *bool, vmExtensions []VMExtension, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + + if ptr.Deref(disableExtensionOperations, false) && len(vmExtensions) > 0 { + allErrs = append(allErrs, field.Forbidden(field.NewPath("AzureMachineTemplate", "spec", "template", "spec", "VMExtensions"), "VMExtensions must be empty when DisableExtensionOperations is true")) + } + + return allErrs +} diff --git a/api/v1beta1/azuremachine_webhook.go b/api/v1beta1/azuremachine_webhook.go index 3d1055f7342..c5edbab1cdc 100644 --- a/api/v1beta1/azuremachine_webhook.go +++ b/api/v1beta1/azuremachine_webhook.go @@ -213,6 +213,13 @@ func (mw *azureMachineWebhook) ValidateUpdate(ctx context.Context, oldObj, newOb allErrs = append(allErrs, err) } + if err := webhookutils.ValidateImmutable( + field.NewPath("spec", "disableExtensionOperations"), + old.Spec.DisableExtensionOperations, + m.Spec.DisableExtensionOperations); err != nil { + allErrs = append(allErrs, err) + } + if len(allErrs) == 0 { return nil, nil } diff --git a/api/v1beta1/azuremachine_webhook_test.go b/api/v1beta1/azuremachine_webhook_test.go index a0e3ea25151..6ee2f029d57 100644 --- a/api/v1beta1/azuremachine_webhook_test.go +++ b/api/v1beta1/azuremachine_webhook_test.go @@ -230,6 +230,16 @@ func TestAzureMachine_ValidateCreate(t *testing.T) { machine: createMachineWithCapacityReservaionGroupID("invalid-capacity-group-id"), wantErr: true, }, + { + name: "azuremachine with DisableExtensionOperations true and without VMExtensions", + machine: createMachineWithDisableExtenionOperations(), + wantErr: false, + }, + { + name: "azuremachine with DisableExtensionOperations true and with VMExtension", + machine: createMachineWithDisableExtenionOperationsAndHasExtension(), + wantErr: true, + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -778,6 +788,34 @@ func TestAzureMachine_ValidateUpdate(t *testing.T) { }, wantErr: true, }, + { + name: "invalidTest: azuremachine.spec.disableExtensionOperations is immutable", + oldMachine: &AzureMachine{ + Spec: AzureMachineSpec{ + DisableExtensionOperations: ptr.To(true), + }, + }, + newMachine: &AzureMachine{ + Spec: AzureMachineSpec{ + DisableExtensionOperations: ptr.To(false), + }, + }, + wantErr: true, + }, + { + name: "validTest: azuremachine.spec.disableExtensionOperations is immutable", + oldMachine: &AzureMachine{ + Spec: AzureMachineSpec{ + DisableExtensionOperations: ptr.To(true), + }, + }, + newMachine: &AzureMachine{ + Spec: AzureMachineSpec{ + DisableExtensionOperations: ptr.To(true), + }, + }, + wantErr: false, + }, { name: "validTest: azuremachine.spec.networkInterfaces is immutable", oldMachine: &AzureMachine{ @@ -1154,3 +1192,28 @@ func createMachineWithCapacityReservaionGroupID(capacityReservationGroupID strin }, } } + +func createMachineWithDisableExtenionOperationsAndHasExtension() *AzureMachine { + return &AzureMachine{ + Spec: AzureMachineSpec{ + SSHPublicKey: validSSHPublicKey, + OSDisk: validOSDisk, + DisableExtensionOperations: ptr.To(true), + VMExtensions: []VMExtension{{ + Name: "test-extension", + Publisher: "test-publiher", + Version: "v0.0.1-test", + }}, + }, + } +} + +func createMachineWithDisableExtenionOperations() *AzureMachine { + return &AzureMachine{ + Spec: AzureMachineSpec{ + SSHPublicKey: validSSHPublicKey, + OSDisk: validOSDisk, + DisableExtensionOperations: ptr.To(true), + }, + } +} diff --git a/api/v1beta1/azuremachinetemplate_webhook.go b/api/v1beta1/azuremachinetemplate_webhook.go index cb4b43a7b0c..01cde89f300 100644 --- a/api/v1beta1/azuremachinetemplate_webhook.go +++ b/api/v1beta1/azuremachinetemplate_webhook.go @@ -24,6 +24,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/utils/ptr" "sigs.k8s.io/cluster-api/util/topology" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -85,6 +86,10 @@ func (r *AzureMachineTemplate) ValidateCreate(ctx context.Context, obj runtime.O } } + if ptr.Deref(r.Spec.Template.Spec.DisableExtensionOperations, false) && len(r.Spec.Template.Spec.VMExtensions) > 0 { + allErrs = append(allErrs, field.Forbidden(field.NewPath("AzureMachineTemplate", "spec", "template", "spec", "VMExtensions"), "VMExtensions must be empty when DisableExtensionOperations is true")) + } + if len(allErrs) == 0 { return nil, nil } diff --git a/api/v1beta1/azuremachinetemplate_webhook_test.go b/api/v1beta1/azuremachinetemplate_webhook_test.go index 8e98ab32d58..210112533c8 100644 --- a/api/v1beta1/azuremachinetemplate_webhook_test.go +++ b/api/v1beta1/azuremachinetemplate_webhook_test.go @@ -143,6 +143,16 @@ func TestAzureMachineTemplate_ValidateCreate(t *testing.T) { machineTemplate: createAzureMachineTemplateFromMachine(createMachineWithRoleAssignmentName()), wantErr: true, }, + { + name: "azuremachinetemplate with DisableExtensionOperations true and without VMExtensions", + machineTemplate: createAzureMachineTemplateFromMachine(createMachineWithDisableExtenionOperations()), + wantErr: false, + }, + { + name: "azuremachinetempalte with DisableExtensionOperations true and with VMExtension", + machineTemplate: createAzureMachineTemplateFromMachine(createMachineWithDisableExtenionOperationsAndHasExtension()), + wantErr: true, + }, { name: "azuremachinetemplate without RoleAssignmentName", machineTemplate: createAzureMachineTemplateFromMachine(createMachineWithoutRoleAssignmentName()), diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index e81679b1262..2b1fc5cff89 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -934,6 +934,11 @@ func (in *AzureMachineSpec) DeepCopyInto(out *AzureMachineSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.DisableExtensionOperations != nil { + in, out := &in.DisableExtensionOperations, &out.DisableExtensionOperations + *out = new(bool) + **out = **in + } if in.VMExtensions != nil { in, out := &in.VMExtensions, &out.VMExtensions *out = make([]VMExtension, len(*in)) diff --git a/azure/scope/machine.go b/azure/scope/machine.go index 67f090d8e35..e5e2aac09fb 100644 --- a/azure/scope/machine.go +++ b/azure/scope/machine.go @@ -178,6 +178,7 @@ func (m *MachineScope) VMSpec() azure.ResourceSpecGetter { SpotVMOptions: m.AzureMachine.Spec.SpotVMOptions, SecurityProfile: m.AzureMachine.Spec.SecurityProfile, DiagnosticsProfile: m.AzureMachine.Spec.Diagnostics, + DisableExtensionOperations: ptr.Deref(m.AzureMachine.Spec.DisableExtensionOperations, false), AdditionalTags: m.AdditionalTags(), AdditionalCapabilities: m.AzureMachine.Spec.AdditionalCapabilities, CapacityReservationGroupID: m.GetCapacityReservationGroupID(), @@ -374,6 +375,10 @@ func (m *MachineScope) HasSystemAssignedIdentity() bool { // VMExtensionSpecs returns the VM extension specs. func (m *MachineScope) VMExtensionSpecs() []azure.ResourceSpecGetter { + if ptr.Deref(m.AzureMachine.Spec.DisableExtensionOperations, false) { + return []azure.ResourceSpecGetter{} + } + var extensionSpecs = []azure.ResourceSpecGetter{} for _, extension := range m.AzureMachine.Spec.VMExtensions { extensionSpecs = append(extensionSpecs, &vmextensions.VMExtensionSpec{ diff --git a/azure/scope/machine_test.go b/azure/scope/machine_test.go index a482e9198ca..b06b3001ffd 100644 --- a/azure/scope/machine_test.go +++ b/azure/scope/machine_test.go @@ -598,6 +598,44 @@ func TestMachineScope_VMExtensionSpecs(t *testing.T) { }, }, }, + { + name: "If OS type is Linux and cloud is AzurePublicCloud and DisableExtensionOperations is true, it returns empty", + machineScope: MachineScope{ + Machine: &clusterv1.Machine{}, + AzureMachine: &infrav1.AzureMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "machine-name", + }, + Spec: infrav1.AzureMachineSpec{ + DisableExtensionOperations: ptr.To(true), + OSDisk: infrav1.OSDisk{ + OSType: "Linux", + }, + }, + }, + ClusterScoper: &ClusterScope{ + AzureClients: AzureClients{ + EnvironmentSettings: auth.EnvironmentSettings{ + Environment: azureautorest.Environment{ + Name: azureautorest.PublicCloud.Name, + }, + }, + }, + AzureCluster: &infrav1.AzureCluster{ + Spec: infrav1.AzureClusterSpec{ + ResourceGroup: "my-rg", + AzureClusterClassSpec: infrav1.AzureClusterClassSpec{ + Location: "westus", + }, + }, + }, + }, + cache: &MachineCache{ + VMSKU: resourceskus.SKU{}, + }, + }, + want: []azure.ResourceSpecGetter{}, + }, { name: "If OS type is Linux and cloud is not AzurePublicCloud, it returns empty", machineScope: MachineScope{ diff --git a/azure/services/virtualmachines/spec.go b/azure/services/virtualmachines/spec.go index fddda163fb1..391d4537b45 100644 --- a/azure/services/virtualmachines/spec.go +++ b/azure/services/virtualmachines/spec.go @@ -53,6 +53,7 @@ type VMSpec struct { AdditionalTags infrav1.Tags AdditionalCapabilities *infrav1.AdditionalCapabilities DiagnosticsProfile *infrav1.Diagnostics + DisableExtensionOperations bool CapacityReservationGroupID string SKU resourceskus.SKU Image *infrav1.Image @@ -263,9 +264,10 @@ func (s *VMSpec) generateOSProfile() (*armcompute.OSProfile, error) { } osProfile := &armcompute.OSProfile{ - ComputerName: ptr.To(s.Name), - AdminUsername: ptr.To(azure.DefaultUserName), - CustomData: ptr.To(s.BootstrapData), + ComputerName: ptr.To(s.Name), + AdminUsername: ptr.To(azure.DefaultUserName), + CustomData: ptr.To(s.BootstrapData), + AllowExtensionOperations: ptr.To(!s.DisableExtensionOperations), } switch s.OSDisk.OSType { diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachines.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachines.yaml index ce6abc26bbb..7bbcf9b277a 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachines.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachines.yaml @@ -238,6 +238,12 @@ spec: - storageAccountType type: object type: object + disableExtensionOperations: + description: |- + DisableExtensionOperations specifies whether extension operations should be disabled on the virtual machine. + Use this setting only if VMExtensions are not supported by your image, as it disables CAPZ bootstrapping extension used for detecting Kubernetes bootstrap failure. + This may only be set to True when no extensions are configured on the virtual machine. + type: boolean dnsServers: description: DNSServers adds a list of DNS Server IP addresses to the VM NICs. diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachinetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachinetemplates.yaml index b78e2c6d0ea..c847db50640 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachinetemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachinetemplates.yaml @@ -253,6 +253,12 @@ spec: - storageAccountType type: object type: object + disableExtensionOperations: + description: |- + DisableExtensionOperations specifies whether extension operations should be disabled on the virtual machine. + Use this setting only if VMExtensions are not supported by your image, as it disables CAPZ bootstrapping extension used for detecting Kubernetes bootstrap failure. + This may only be set to True when no extensions are configured on the virtual machine. + type: boolean dnsServers: description: DNSServers adds a list of DNS Server IP addresses to the VM NICs.