diff --git a/api/v1alpha3/azuremachine_conversion.go b/api/v1alpha3/azuremachine_conversion.go index dcf10192e42..e5994f1836c 100644 --- a/api/v1alpha3/azuremachine_conversion.go +++ b/api/v1alpha3/azuremachine_conversion.go @@ -62,6 +62,10 @@ func (src *AzureMachine) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.DNSServers = restored.Spec.DNSServers } + if len(restored.Spec.VMExtensions) > 0 { + dst.Spec.VMExtensions = restored.Spec.VMExtensions + } + dst.Spec.SubnetName = restored.Spec.SubnetName dst.Status.LongRunningOperationStates = restored.Status.LongRunningOperationStates diff --git a/api/v1alpha3/azuremachinetemplate_conversion.go b/api/v1alpha3/azuremachinetemplate_conversion.go index c3e80912b90..870d42c470f 100644 --- a/api/v1alpha3/azuremachinetemplate_conversion.go +++ b/api/v1alpha3/azuremachinetemplate_conversion.go @@ -65,6 +65,10 @@ func (src *AzureMachineTemplate) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.Template.Spec.DNSServers = restored.Spec.Template.Spec.DNSServers } + if len(restored.Spec.Template.Spec.VMExtensions) > 0 { + dst.Spec.Template.Spec.VMExtensions = restored.Spec.Template.Spec.VMExtensions + } + return nil } diff --git a/api/v1alpha3/zz_generated.conversion.go b/api/v1alpha3/zz_generated.conversion.go index 8cf4bfadfea..f16f2ec7661 100644 --- a/api/v1alpha3/zz_generated.conversion.go +++ b/api/v1alpha3/zz_generated.conversion.go @@ -907,6 +907,7 @@ func autoConvert_v1beta1_AzureMachineSpec_To_v1alpha3_AzureMachineSpec(in *v1bet out.SecurityProfile = (*SecurityProfile)(unsafe.Pointer(in.SecurityProfile)) // WARNING: in.SubnetName requires manual conversion: does not exist in peer-type // WARNING: in.DNSServers requires manual conversion: does not exist in peer-type + // WARNING: in.VMExtensions requires manual conversion: does not exist in peer-type return nil } diff --git a/api/v1alpha4/azuremachine_conversion.go b/api/v1alpha4/azuremachine_conversion.go index cf582ada834..85bb35638a3 100644 --- a/api/v1alpha4/azuremachine_conversion.go +++ b/api/v1alpha4/azuremachine_conversion.go @@ -48,6 +48,10 @@ func (src *AzureMachine) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.DNSServers = restored.Spec.DNSServers } + if len(restored.Spec.VMExtensions) > 0 { + dst.Spec.VMExtensions = restored.Spec.VMExtensions + } + return nil } diff --git a/api/v1alpha4/azuremachinetemplate_conversion.go b/api/v1alpha4/azuremachinetemplate_conversion.go index 4abf57698bd..4b67e7695aa 100644 --- a/api/v1alpha4/azuremachinetemplate_conversion.go +++ b/api/v1alpha4/azuremachinetemplate_conversion.go @@ -50,6 +50,10 @@ func (src *AzureMachineTemplate) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.Template.Spec.DNSServers = restored.Spec.Template.Spec.DNSServers } + if len(restored.Spec.Template.Spec.VMExtensions) > 0 { + dst.Spec.Template.Spec.VMExtensions = restored.Spec.Template.Spec.VMExtensions + } + return nil } diff --git a/api/v1alpha4/zz_generated.conversion.go b/api/v1alpha4/zz_generated.conversion.go index d6670115915..11f9161c194 100644 --- a/api/v1alpha4/zz_generated.conversion.go +++ b/api/v1alpha4/zz_generated.conversion.go @@ -1054,6 +1054,7 @@ func autoConvert_v1beta1_AzureMachineSpec_To_v1alpha4_AzureMachineSpec(in *v1bet out.SecurityProfile = (*SecurityProfile)(unsafe.Pointer(in.SecurityProfile)) out.SubnetName = in.SubnetName // WARNING: in.DNSServers requires manual conversion: does not exist in peer-type + // WARNING: in.VMExtensions requires manual conversion: does not exist in peer-type return nil } diff --git a/api/v1beta1/azuremachine_types.go b/api/v1beta1/azuremachine_types.go index c804823969d..3ba84e784b3 100644 --- a/api/v1beta1/azuremachine_types.go +++ b/api/v1beta1/azuremachine_types.go @@ -122,6 +122,10 @@ type AzureMachineSpec struct { // DNSServers adds a list of DNS Server IP addresses to the VM NICs. // +optional DNSServers []string `json:"dnsServers,omitempty"` + + // VMExtensions specifies a list of extensions to be added to the virtual machine. + // +optional + VMExtensions []VMExtension `json:"vmExtensions,omitempty"` } // SpotVMOptions defines the options relevant to running the Machine on Spot VMs. diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index d60fa3f47a5..70d4f00f234 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -530,6 +530,22 @@ type DataDisk struct { CachingType string `json:"cachingType,omitempty"` } +// VMExtension specifies the parameters for a custom VM extension. +type VMExtension struct { + // Name is the name of the extension. + Name string `json:"name"` + // Publisher is the name of the extension handler publisher. + Publisher string `json:"publisher"` + // Version specifies the version of the script handler. + Version string `json:"version"` + // Settings is a JSON formatted public settings for the extension. + // +optional + Settings Tags `json:"settings,omitempty"` + // ProtectedSettings is a JSON formatted protected settings for the extension. + // +optional + ProtectedSettings Tags `json:"protectedSettings,omitempty"` +} + // ManagedDiskParameters defines the parameters of a managed disk. type ManagedDiskParameters struct { // +optional diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 19b3f359952..7aae51b16dc 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -628,6 +628,13 @@ func (in *AzureMachineSpec) DeepCopyInto(out *AzureMachineSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.VMExtensions != nil { + in, out := &in.VMExtensions, &out.VMExtensions + *out = make([]VMExtension, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureMachineSpec. @@ -1701,6 +1708,35 @@ func (in *UserAssignedIdentity) DeepCopy() *UserAssignedIdentity { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VMExtension) DeepCopyInto(out *VMExtension) { + *out = *in + if in.Settings != nil { + in, out := &in.Settings, &out.Settings + *out = make(Tags, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.ProtectedSettings != nil { + in, out := &in.ProtectedSettings, &out.ProtectedSettings + *out = make(Tags, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VMExtension. +func (in *VMExtension) DeepCopy() *VMExtension { + if in == nil { + return nil + } + out := new(VMExtension) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VnetClassSpec) DeepCopyInto(out *VnetClassSpec) { *out = *in diff --git a/azure/scope/machine.go b/azure/scope/machine.go index 46830c0d3c1..78d6b9a9a3f 100644 --- a/azure/scope/machine.go +++ b/azure/scope/machine.go @@ -333,6 +333,21 @@ func (m *MachineScope) HasSystemAssignedIdentity() bool { // VMExtensionSpecs returns the VM extension specs. func (m *MachineScope) VMExtensionSpecs() []azure.ResourceSpecGetter { var extensionSpecs = []azure.ResourceSpecGetter{} + for _, extension := range m.AzureMachine.Spec.VMExtensions { + extensionSpecs = append(extensionSpecs, &vmextensions.VMExtensionSpec{ + ExtensionSpec: azure.ExtensionSpec{ + Name: extension.Name, + VMName: m.Name(), + Publisher: extension.Publisher, + Version: extension.Version, + Settings: extension.Settings, + ProtectedSettings: extension.ProtectedSettings, + }, + ResourceGroup: m.ResourceGroup(), + Location: m.Location(), + }) + } + bootstrapExtensionSpec := azure.GetBootstrappingVMExtension(m.AzureMachine.Spec.OSDisk.OSType, m.CloudEnvironment(), m.Name()) if bootstrapExtensionSpec != nil { @@ -359,9 +374,9 @@ func (m *MachineScope) Subnet() infrav1.SubnetSpec { // AvailabilityZone returns the AzureMachine Availability Zone. // Priority for selecting the AZ is -// 1) Machine.Spec.FailureDomain -// 2) AzureMachine.Spec.FailureDomain (This is to support deprecated AZ) -// 3) No AZ +// 1. Machine.Spec.FailureDomain +// 2. AzureMachine.Spec.FailureDomain (This is to support deprecated AZ) +// 3. No AZ func (m *MachineScope) AvailabilityZone() string { if m.Machine.Spec.FailureDomain != nil { return *m.Machine.Spec.FailureDomain diff --git a/azure/scope/machine_test.go b/azure/scope/machine_test.go index 3ce296756ed..64194c05527 100644 --- a/azure/scope/machine_test.go +++ b/azure/scope/machine_test.go @@ -728,6 +728,83 @@ func TestMachineScope_VMExtensionSpecs(t *testing.T) { }, want: []azure.ResourceSpecGetter{}, }, + { + name: "If a custom VM extension is specified, it returns the custom VM extension", + machineScope: MachineScope{ + Machine: &clusterv1.Machine{}, + AzureMachine: &infrav1.AzureMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "machine-name", + }, + Spec: infrav1.AzureMachineSpec{ + OSDisk: infrav1.OSDisk{ + OSType: "Linux", + }, + VMExtensions: []infrav1.VMExtension{ + { + Name: "custom-vm-extension", + Publisher: "Microsoft.Azure.Extensions", + Version: "2.0", + Settings: map[string]string{ + "timestamp": "1234567890", + }, + ProtectedSettings: map[string]string{ + "commandToExecute": "echo hello world", + }, + }, + }, + }, + }, + ClusterScoper: &ClusterScope{ + AzureClients: AzureClients{ + EnvironmentSettings: auth.EnvironmentSettings{ + Environment: autorestazure.Environment{ + Name: autorestazure.PublicCloud.Name, + }, + }, + }, + AzureCluster: &infrav1.AzureCluster{ + Spec: infrav1.AzureClusterSpec{ + ResourceGroup: "my-rg", + AzureClusterClassSpec: infrav1.AzureClusterClassSpec{ + Location: "westus", + }, + }, + }, + }, + }, + want: []azure.ResourceSpecGetter{ + &vmextensions.VMExtensionSpec{ + ExtensionSpec: azure.ExtensionSpec{ + Name: "custom-vm-extension", + VMName: "machine-name", + Publisher: "Microsoft.Azure.Extensions", + Version: "2.0", + Settings: map[string]string{ + "timestamp": "1234567890", + }, + ProtectedSettings: map[string]string{ + "commandToExecute": "echo hello world", + }, + }, + ResourceGroup: "my-rg", + Location: "westus", + }, + &vmextensions.VMExtensionSpec{ + ExtensionSpec: azure.ExtensionSpec{ + Name: "CAPZ.Linux.Bootstrapping", + VMName: "machine-name", + Publisher: "Microsoft.Azure.ContainerUpstream", + Version: "1.0", + ProtectedSettings: map[string]string{ + "commandToExecute": azure.LinuxBootstrapExtensionCommand, + }, + }, + ResourceGroup: "my-rg", + Location: "westus", + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/azure/scope/machinepool.go b/azure/scope/machinepool.go index 6286b9c753f..9140862920e 100644 --- a/azure/scope/machinepool.go +++ b/azure/scope/machinepool.go @@ -127,6 +127,7 @@ func (m *MachinePoolScope) ScaleSetSpec() azure.ScaleSetSpec { SpotVMOptions: m.AzureMachinePool.Spec.Template.SpotVMOptions, FailureDomains: m.MachinePool.Spec.FailureDomains, TerminateNotificationTimeout: m.AzureMachinePool.Spec.Template.TerminateNotificationTimeout, + VMExtensions: m.AzureMachinePool.Spec.Template.VMExtensions, } } @@ -613,6 +614,21 @@ func (m *MachinePoolScope) HasSystemAssignedIdentity() bool { // VMSSExtensionSpecs returns the VMSS extension specs. func (m *MachinePoolScope) VMSSExtensionSpecs() []azure.ResourceSpecGetter { var extensionSpecs = []azure.ResourceSpecGetter{} + + for _, extension := range m.AzureMachinePool.Spec.Template.VMExtensions { + extensionSpecs = append(extensionSpecs, &scalesets.VMSSExtensionSpec{ + ExtensionSpec: azure.ExtensionSpec{ + Name: extension.Name, + VMName: m.Name(), + Publisher: extension.Publisher, + Version: extension.Version, + Settings: extension.Settings, + ProtectedSettings: extension.ProtectedSettings, + }, + ResourceGroup: m.ResourceGroup(), + }) + } + bootstrapExtensionSpec := azure.GetBootstrappingVMExtension(m.AzureMachinePool.Spec.Template.OSDisk.OSType, m.CloudEnvironment(), m.Name()) if bootstrapExtensionSpec != nil { diff --git a/azure/scope/machinepool_test.go b/azure/scope/machinepool_test.go index cd82f910349..fbc6ead0291 100644 --- a/azure/scope/machinepool_test.go +++ b/azure/scope/machinepool_test.go @@ -872,6 +872,83 @@ func TestMachinePoolScope_VMSSExtensionSpecs(t *testing.T) { }, want: []azure.ResourceSpecGetter{}, }, + { + name: "If a custom VM extension is specified, it returns the custom VM extension", + machinePoolScope: MachinePoolScope{ + MachinePool: &expv1.MachinePool{}, + AzureMachinePool: &infrav1exp.AzureMachinePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "machinepool-name", + }, + Spec: infrav1exp.AzureMachinePoolSpec{ + Template: infrav1exp.AzureMachinePoolMachineTemplate{ + OSDisk: infrav1.OSDisk{ + OSType: "Linux", + }, + VMExtensions: []infrav1.VMExtension{ + { + Name: "custom-vm-extension", + Publisher: "Microsoft.Azure.Extensions", + Version: "2.0", + Settings: map[string]string{ + "timestamp": "1234567890", + }, + ProtectedSettings: map[string]string{ + "commandToExecute": "echo hello world", + }, + }, + }, + }, + }, + }, + ClusterScoper: &ClusterScope{ + AzureClients: AzureClients{ + EnvironmentSettings: auth.EnvironmentSettings{ + Environment: autorestazure.Environment{ + Name: autorestazure.PublicCloud.Name, + }, + }, + }, + AzureCluster: &infrav1.AzureCluster{ + Spec: infrav1.AzureClusterSpec{ + ResourceGroup: "my-rg", + AzureClusterClassSpec: infrav1.AzureClusterClassSpec{ + Location: "westus", + }, + }, + }, + }, + }, + want: []azure.ResourceSpecGetter{ + &scalesets.VMSSExtensionSpec{ + ExtensionSpec: azure.ExtensionSpec{ + Name: "custom-vm-extension", + VMName: "machinepool-name", + Publisher: "Microsoft.Azure.Extensions", + Version: "2.0", + Settings: map[string]string{ + "timestamp": "1234567890", + }, + ProtectedSettings: map[string]string{ + "commandToExecute": "echo hello world", + }, + }, + ResourceGroup: "my-rg", + }, + &scalesets.VMSSExtensionSpec{ + ExtensionSpec: azure.ExtensionSpec{ + Name: "CAPZ.Linux.Bootstrapping", + VMName: "machinepool-name", + Publisher: "Microsoft.Azure.ContainerUpstream", + Version: "1.0", + ProtectedSettings: map[string]string{ + "commandToExecute": azure.LinuxBootstrapExtensionCommand, + }, + }, + ResourceGroup: "my-rg", + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/azure/services/scalesets/scalesets_test.go b/azure/services/scalesets/scalesets_test.go index e179b8cd776..e8700e4a755 100644 --- a/azure/services/scalesets/scalesets_test.go +++ b/azure/services/scalesets/scalesets_test.go @@ -1151,6 +1151,9 @@ func newDefaultVMSS(vmSize string) compute.VirtualMachineScaleSet { Publisher: to.StringPtr("somePublisher"), Type: to.StringPtr("someExtension"), TypeHandlerVersion: to.StringPtr("someVersion"), + Settings: map[string]string{ + "someSetting": "someValue", + }, ProtectedSettings: map[string]string{ "commandToExecute": "echo hello", }, @@ -1361,6 +1364,9 @@ func setupVMSSExpectationsWithoutVMImage(s *mock_scalesets.MockScaleSetScopeMock VMName: "my-vmss", Publisher: "somePublisher", Version: "someVersion", + Settings: map[string]string{ + "someSetting": "someValue", + }, ProtectedSettings: map[string]string{ "commandToExecute": "echo hello", }, diff --git a/azure/services/scalesets/vmssextension_spec.go b/azure/services/scalesets/vmssextension_spec.go index 27f663c8153..0c77c5c25d8 100644 --- a/azure/services/scalesets/vmssextension_spec.go +++ b/azure/services/scalesets/vmssextension_spec.go @@ -62,7 +62,7 @@ func (s *VMSSExtensionSpec) Parameters(existing interface{}) (interface{}, error Publisher: to.StringPtr(s.Publisher), Type: to.StringPtr(s.Name), TypeHandlerVersion: to.StringPtr(s.Version), - Settings: nil, + Settings: s.Settings, ProtectedSettings: s.ProtectedSettings, }, }, nil diff --git a/azure/services/scalesets/vmssextension_spec_test.go b/azure/services/scalesets/vmssextension_spec_test.go new file mode 100644 index 00000000000..8671e19766b --- /dev/null +++ b/azure/services/scalesets/vmssextension_spec_test.go @@ -0,0 +1,96 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scalesets + +import ( + "testing" + + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute" + "github.com/Azure/go-autorest/autorest/to" + . "github.com/onsi/gomega" + "sigs.k8s.io/cluster-api-provider-azure/azure" +) + +var ( + fakeVMSSExtensionSpec = VMSSExtensionSpec{ + azure.ExtensionSpec{ + Name: "my-vm-extension", + VMName: "my-vm", + Publisher: "my-publisher", + Version: "1.0", + Settings: map[string]string{"my-setting": "my-value"}, + ProtectedSettings: map[string]string{"my-protected-setting": "my-protected-value"}, + }, + "my-rg", + } + + fakeVMSSExtensionParams = compute.VirtualMachineScaleSetExtension{ + Name: to.StringPtr("my-vm-extension"), + VirtualMachineScaleSetExtensionProperties: &compute.VirtualMachineScaleSetExtensionProperties{ + Publisher: to.StringPtr("my-publisher"), + Type: to.StringPtr("my-vm-extension"), + TypeHandlerVersion: to.StringPtr("1.0"), + Settings: map[string]string{"my-setting": "my-value"}, + ProtectedSettings: map[string]string{"my-protected-setting": "my-protected-value"}, + }, + } +) + +func TestParameters(t *testing.T) { + testcases := []struct { + name string + spec *VMSSExtensionSpec + existing interface{} + expect func(g *WithT, result interface{}) + expectedError string + }{ + { + name: "get parameters for vmextension", + spec: &fakeVMSSExtensionSpec, + existing: nil, + expect: func(g *WithT, result interface{}) { + g.Expect(result).To(Equal(fakeVMSSExtensionParams)) + }, + expectedError: "", + }, + { + name: "vmextension that already exists", + spec: &fakeVMSSExtensionSpec, + existing: fakeVMSSExtensionParams, + expect: func(g *WithT, result interface{}) { + g.Expect(result).To(BeNil()) + }, + expectedError: "", + }, + } + for _, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + t.Parallel() + + result, err := tc.spec.Parameters(tc.existing) + if tc.expectedError != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err).To(MatchError(tc.expectedError)) + } else { + g.Expect(err).NotTo(HaveOccurred()) + } + tc.expect(g, result) + }) + } +} diff --git a/azure/services/virtualmachines/spec.go b/azure/services/virtualmachines/spec.go index e77dab9c343..0e7d7bcc1c0 100644 --- a/azure/services/virtualmachines/spec.go +++ b/azure/services/virtualmachines/spec.go @@ -54,6 +54,7 @@ type VMSpec struct { Image *infrav1.Image BootstrapData string ProviderID string + VMExtensions []infrav1.VMExtension } // ResourceName returns the name of the virtual machine. diff --git a/azure/services/vmextensions/spec.go b/azure/services/vmextensions/spec.go index 89d491b37cd..e99cf16ac0e 100644 --- a/azure/services/vmextensions/spec.go +++ b/azure/services/vmextensions/spec.go @@ -62,7 +62,7 @@ func (s *VMExtensionSpec) Parameters(existing interface{}) (interface{}, error) Publisher: to.StringPtr(s.Publisher), Type: to.StringPtr(s.Name), TypeHandlerVersion: to.StringPtr(s.Version), - Settings: nil, + Settings: s.Settings, ProtectedSettings: s.ProtectedSettings, }, Location: to.StringPtr(s.Location), diff --git a/azure/services/vmextensions/spec_test.go b/azure/services/vmextensions/spec_test.go new file mode 100644 index 00000000000..67d0bce0334 --- /dev/null +++ b/azure/services/vmextensions/spec_test.go @@ -0,0 +1,97 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vmextensions + +import ( + "testing" + + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute" + "github.com/Azure/go-autorest/autorest/to" + . "github.com/onsi/gomega" + "sigs.k8s.io/cluster-api-provider-azure/azure" +) + +var ( + fakeVMExtensionSpec = VMExtensionSpec{ + azure.ExtensionSpec{ + Name: "my-vm-extension", + VMName: "my-vm", + Publisher: "my-publisher", + Version: "1.0", + Settings: map[string]string{"my-setting": "my-value"}, + ProtectedSettings: map[string]string{"my-protected-setting": "my-protected-value"}, + }, + "my-rg", + "my-location", + } + + fakeVMExtensionParams = compute.VirtualMachineExtension{ + VirtualMachineExtensionProperties: &compute.VirtualMachineExtensionProperties{ + Publisher: to.StringPtr("my-publisher"), + Type: to.StringPtr("my-vm-extension"), + TypeHandlerVersion: to.StringPtr("1.0"), + Settings: map[string]string{"my-setting": "my-value"}, + ProtectedSettings: map[string]string{"my-protected-setting": "my-protected-value"}, + }, + Location: to.StringPtr("my-location"), + } +) + +func TestParameters(t *testing.T) { + testcases := []struct { + name string + spec *VMExtensionSpec + existing interface{} + expect func(g *WithT, result interface{}) + expectedError string + }{ + { + name: "get parameters for vmextension", + spec: &fakeVMExtensionSpec, + existing: nil, + expect: func(g *WithT, result interface{}) { + g.Expect(result).To(Equal(fakeVMExtensionParams)) + }, + expectedError: "", + }, + { + name: "vmextension that already exists", + spec: &fakeVMExtensionSpec, + existing: fakeVMExtensionParams, + expect: func(g *WithT, result interface{}) { + g.Expect(result).To(BeNil()) + }, + expectedError: "", + }, + } + for _, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + t.Parallel() + + result, err := tc.spec.Parameters(tc.existing) + if tc.expectedError != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err).To(MatchError(tc.expectedError)) + } else { + g.Expect(err).NotTo(HaveOccurred()) + } + tc.expect(g, result) + }) + } +} diff --git a/azure/types.go b/azure/types.go index df20d352657..81756ab4e69 100644 --- a/azure/types.go +++ b/azure/types.go @@ -64,6 +64,7 @@ type ScaleSetSpec struct { SpotVMOptions *infrav1.SpotVMOptions AdditionalCapabilities *infrav1.AdditionalCapabilities FailureDomains []string + VMExtensions []infrav1.VMExtension } // TagsSpec defines the specification for a set of tags. @@ -82,6 +83,7 @@ type ExtensionSpec struct { VMName string Publisher string Version string + Settings map[string]string ProtectedSettings map[string]string } diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachinepools.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachinepools.yaml index a45716ba497..78566278b69 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachinepools.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachinepools.yaml @@ -1792,6 +1792,42 @@ spec: VMSS scheduled events termination notification with specified timeout allowed values are between 5 and 15 (mins) type: integer + vmExtensions: + description: VMExtensions specifies a list of extensions to be + added to the virtual machine. + items: + description: VMExtension specifies the parameters for a custom + VM extension. + properties: + name: + description: Name is the name of the extension. + type: string + protectedSettings: + additionalProperties: + type: string + description: ProtectedSettings is a JSON formatted protected + settings for the extension. + type: object + publisher: + description: Publisher is the name of the extension handler + publisher. + type: string + settings: + additionalProperties: + type: string + description: Settings is a JSON formatted public settings + for the extension. + type: object + version: + description: Version specifies the version of the script + handler. + type: string + required: + - name + - publisher + - version + type: object + type: array vmSize: description: VMSize is the size of the Virtual Machine to build. See https://docs.microsoft.com/en-us/rest/api/compute/virtualmachines/createorupdate#virtualmachinesizetypes diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachines.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachines.yaml index ddf1a5928f7..c3cb0303399 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachines.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachines.yaml @@ -1434,6 +1434,41 @@ spec: - providerID type: object type: array + vmExtensions: + description: VMExtensions specifies a list of extensions to be added + to the virtual machine. + items: + description: VMExtension specifies the parameters for a custom VM + extension. + properties: + name: + description: Name is the name of the extension. + type: string + protectedSettings: + additionalProperties: + type: string + description: ProtectedSettings is a JSON formatted protected + settings for the extension. + type: object + publisher: + description: Publisher is the name of the extension handler + publisher. + type: string + settings: + additionalProperties: + type: string + description: Settings is a JSON formatted public settings for + the extension. + type: object + version: + description: Version specifies the version of the script handler. + type: string + required: + - name + - publisher + - version + type: object + type: array vmSize: type: string required: diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachinetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachinetemplates.yaml index 1a246a2e66b..a223d3fee34 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachinetemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachinetemplates.yaml @@ -1219,6 +1219,42 @@ spec: - providerID type: object type: array + vmExtensions: + description: VMExtensions specifies a list of extensions to + be added to the virtual machine. + items: + description: VMExtension specifies the parameters for a + custom VM extension. + properties: + name: + description: Name is the name of the extension. + type: string + protectedSettings: + additionalProperties: + type: string + description: ProtectedSettings is a JSON formatted protected + settings for the extension. + type: object + publisher: + description: Publisher is the name of the extension + handler publisher. + type: string + settings: + additionalProperties: + type: string + description: Settings is a JSON formatted public settings + for the extension. + type: object + version: + description: Version specifies the version of the script + handler. + type: string + required: + - name + - publisher + - version + type: object + type: array vmSize: type: string required: diff --git a/config/default/manager_pull_policy.yaml b/config/default/manager_pull_policy.yaml index 74a0879c604..cd7ae12c01e 100644 --- a/config/default/manager_pull_policy.yaml +++ b/config/default/manager_pull_policy.yaml @@ -8,4 +8,4 @@ spec: spec: containers: - name: manager - imagePullPolicy: Always + imagePullPolicy: IfNotPresent diff --git a/exp/api/v1alpha3/azuremachinepool_conversion.go b/exp/api/v1alpha3/azuremachinepool_conversion.go index 86574c7663f..1bf5fec349d 100644 --- a/exp/api/v1alpha3/azuremachinepool_conversion.go +++ b/exp/api/v1alpha3/azuremachinepool_conversion.go @@ -85,6 +85,10 @@ func (src *AzureMachinePool) ConvertTo(dstRaw conversion.Hub) error { } } + if len(restored.Spec.Template.VMExtensions) > 0 { + dst.Spec.Template.VMExtensions = restored.Spec.Template.VMExtensions + } + return nil } diff --git a/exp/api/v1alpha3/zz_generated.conversion.go b/exp/api/v1alpha3/zz_generated.conversion.go index 1012c6945dd..e6e001a507b 100644 --- a/exp/api/v1alpha3/zz_generated.conversion.go +++ b/exp/api/v1alpha3/zz_generated.conversion.go @@ -459,6 +459,7 @@ func autoConvert_v1beta1_AzureMachinePoolMachineTemplate_To_v1alpha3_AzureMachin out.SecurityProfile = (*clusterapiproviderazureapiv1alpha3.SecurityProfile)(unsafe.Pointer(in.SecurityProfile)) out.SpotVMOptions = (*clusterapiproviderazureapiv1alpha3.SpotVMOptions)(unsafe.Pointer(in.SpotVMOptions)) // WARNING: in.SubnetName requires manual conversion: does not exist in peer-type + // WARNING: in.VMExtensions requires manual conversion: does not exist in peer-type return nil } diff --git a/exp/api/v1alpha4/azuremachinepool_conversion.go b/exp/api/v1alpha4/azuremachinepool_conversion.go index 90c19148545..18de2d99cf5 100644 --- a/exp/api/v1alpha4/azuremachinepool_conversion.go +++ b/exp/api/v1alpha4/azuremachinepool_conversion.go @@ -17,6 +17,7 @@ limitations under the License. package v1alpha4 import ( + convert "k8s.io/apimachinery/pkg/conversion" infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" utilconversion "sigs.k8s.io/cluster-api/util/conversion" "sigs.k8s.io/controller-runtime/pkg/conversion" @@ -43,6 +44,10 @@ func (src *AzureMachinePool) ConvertTo(dstRaw conversion.Hub) error { dst.Status.Image.ComputeGallery = restored.Status.Image.ComputeGallery } + if len(restored.Spec.Template.VMExtensions) > 0 { + dst.Spec.Template.VMExtensions = restored.Spec.Template.VMExtensions + } + return nil } @@ -68,3 +73,8 @@ func (dst *AzureMachinePoolList) ConvertFrom(srcRaw conversion.Hub) error { src := srcRaw.(*infrav1exp.AzureMachinePoolList) return Convert_v1beta1_AzureMachinePoolList_To_v1alpha4_AzureMachinePoolList(src, dst, nil) } + +// Convert_v1beta1_AzureMachinePoolMachineTemplate_To_v1alpha4_AzureMachinePoolMachineTemplate converts an Azure Machine Pool Machine Template from v1beta1 to v1alpha4. +func Convert_v1beta1_AzureMachinePoolMachineTemplate_To_v1alpha4_AzureMachinePoolMachineTemplate(in *infrav1exp.AzureMachinePoolMachineTemplate, out *AzureMachinePoolMachineTemplate, s convert.Scope) error { + return autoConvert_v1beta1_AzureMachinePoolMachineTemplate_To_v1alpha4_AzureMachinePoolMachineTemplate(in, out, s) +} diff --git a/exp/api/v1alpha4/zz_generated.conversion.go b/exp/api/v1alpha4/zz_generated.conversion.go index c95dade2bd2..c34c29438d4 100644 --- a/exp/api/v1alpha4/zz_generated.conversion.go +++ b/exp/api/v1alpha4/zz_generated.conversion.go @@ -149,11 +149,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1beta1.AzureMachinePoolMachineTemplate)(nil), (*AzureMachinePoolMachineTemplate)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_AzureMachinePoolMachineTemplate_To_v1alpha4_AzureMachinePoolMachineTemplate(a.(*v1beta1.AzureMachinePoolMachineTemplate), b.(*AzureMachinePoolMachineTemplate), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*AzureMachinePoolSpec)(nil), (*v1beta1.AzureMachinePoolSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha4_AzureMachinePoolSpec_To_v1beta1_AzureMachinePoolSpec(a.(*AzureMachinePoolSpec), b.(*v1beta1.AzureMachinePoolSpec), scope) }); err != nil { @@ -344,6 +339,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1beta1.AzureMachinePoolMachineTemplate)(nil), (*AzureMachinePoolMachineTemplate)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_AzureMachinePoolMachineTemplate_To_v1alpha4_AzureMachinePoolMachineTemplate(a.(*v1beta1.AzureMachinePoolMachineTemplate), b.(*AzureMachinePoolMachineTemplate), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1beta1.AzureManagedControlPlaneSpec)(nil), (*AzureManagedControlPlaneSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_AzureManagedControlPlaneSpec_To_v1alpha4_AzureManagedControlPlaneSpec(a.(*v1beta1.AzureManagedControlPlaneSpec), b.(*AzureManagedControlPlaneSpec), scope) }); err != nil { @@ -715,14 +715,10 @@ func autoConvert_v1beta1_AzureMachinePoolMachineTemplate_To_v1alpha4_AzureMachin out.SecurityProfile = (*clusterapiproviderazureapiv1alpha4.SecurityProfile)(unsafe.Pointer(in.SecurityProfile)) out.SpotVMOptions = (*clusterapiproviderazureapiv1alpha4.SpotVMOptions)(unsafe.Pointer(in.SpotVMOptions)) out.SubnetName = in.SubnetName + // WARNING: in.VMExtensions requires manual conversion: does not exist in peer-type return nil } -// Convert_v1beta1_AzureMachinePoolMachineTemplate_To_v1alpha4_AzureMachinePoolMachineTemplate is an autogenerated conversion function. -func Convert_v1beta1_AzureMachinePoolMachineTemplate_To_v1alpha4_AzureMachinePoolMachineTemplate(in *v1beta1.AzureMachinePoolMachineTemplate, out *AzureMachinePoolMachineTemplate, s conversion.Scope) error { - return autoConvert_v1beta1_AzureMachinePoolMachineTemplate_To_v1alpha4_AzureMachinePoolMachineTemplate(in, out, s) -} - func autoConvert_v1alpha4_AzureMachinePoolSpec_To_v1beta1_AzureMachinePoolSpec(in *AzureMachinePoolSpec, out *v1beta1.AzureMachinePoolSpec, s conversion.Scope) error { out.Location = in.Location if err := Convert_v1alpha4_AzureMachinePoolMachineTemplate_To_v1beta1_AzureMachinePoolMachineTemplate(&in.Template, &out.Template, s); err != nil { diff --git a/exp/api/v1beta1/azuremachinepool_types.go b/exp/api/v1beta1/azuremachinepool_types.go index 7f3b6a50546..54cbb140796 100644 --- a/exp/api/v1beta1/azuremachinepool_types.go +++ b/exp/api/v1beta1/azuremachinepool_types.go @@ -87,6 +87,10 @@ type ( // SubnetName selects the Subnet where the VMSS will be placed // +optional SubnetName string `json:"subnetName,omitempty"` + + // VMExtensions specifies a list of extensions to be added to the virtual machine. + // +optional + VMExtensions []infrav1.VMExtension `json:"vmExtensions,omitempty"` } // AzureMachinePoolSpec defines the desired state of AzureMachinePool. diff --git a/exp/api/v1beta1/zz_generated.deepcopy.go b/exp/api/v1beta1/zz_generated.deepcopy.go index 777e563d3df..03250a54dc9 100644 --- a/exp/api/v1beta1/zz_generated.deepcopy.go +++ b/exp/api/v1beta1/zz_generated.deepcopy.go @@ -364,6 +364,13 @@ func (in *AzureMachinePoolMachineTemplate) DeepCopyInto(out *AzureMachinePoolMac *out = new(apiv1beta1.SpotVMOptions) (*in).DeepCopyInto(*out) } + if in.VMExtensions != nil { + in, out := &in.VMExtensions, &out.VMExtensions + *out = make([]apiv1beta1.VMExtension, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureMachinePoolMachineTemplate. diff --git a/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml b/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml index f6af9380e41..f1813547f7b 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml @@ -278,6 +278,15 @@ spec: diskSizeGB: 128 osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmExtensions: + - name: CustomScript + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "Updating packages ..." + touch test_file + publisher: Microsoft.Azure.Extensions + version: "2.1" vmSize: ${AZURE_NODE_MACHINE_TYPE} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 diff --git a/templates/test/ci/cluster-template-prow-ci-version.yaml b/templates/test/ci/cluster-template-prow-ci-version.yaml index 7f2cc727767..db285d1bb7f 100644 --- a/templates/test/ci/cluster-template-prow-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version.yaml @@ -278,6 +278,15 @@ spec: diskSizeGB: 128 osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmExtensions: + - name: CustomScript + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "Updating packages ..." + touch test_file + publisher: Microsoft.Azure.Extensions + version: "2.1" vmSize: ${AZURE_NODE_MACHINE_TYPE} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 diff --git a/templates/test/ci/cluster-template-prow-external-cloud-provider-ci-version.yaml b/templates/test/ci/cluster-template-prow-external-cloud-provider-ci-version.yaml index a3c41af6aa1..4d9755e71eb 100644 --- a/templates/test/ci/cluster-template-prow-external-cloud-provider-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-external-cloud-provider-ci-version.yaml @@ -281,6 +281,15 @@ spec: diskSizeGB: 128 osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmExtensions: + - name: CustomScript + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "Updating packages ..." + touch test_file + publisher: Microsoft.Azure.Extensions + version: "2.1" vmSize: ${AZURE_NODE_MACHINE_TYPE} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 diff --git a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml index fd1a15b37bf..8759a6b85cd 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml @@ -280,6 +280,15 @@ spec: storageAccountType: Premium_LRS osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmExtensions: + - name: CustomScript + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "Updating packages ..." + touch test_file + publisher: Microsoft.Azure.Extensions + version: "2.1" vmSize: ${AZURE_NODE_MACHINE_TYPE} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 diff --git a/templates/test/ci/cluster-template-prow-machine-pool.yaml b/templates/test/ci/cluster-template-prow-machine-pool.yaml index d5bde2ff7dc..38e4b35b13a 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool.yaml @@ -197,6 +197,15 @@ spec: storageAccountType: Premium_LRS osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmExtensions: + - name: CustomScript + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "Updating packages ..." + touch test_file + publisher: Microsoft.Azure.Extensions + version: "2.1" vmSize: ${AZURE_NODE_MACHINE_TYPE} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 diff --git a/templates/test/ci/cluster-template-prow.yaml b/templates/test/ci/cluster-template-prow.yaml index 87a995d91fa..47aeaedc691 100644 --- a/templates/test/ci/cluster-template-prow.yaml +++ b/templates/test/ci/cluster-template-prow.yaml @@ -193,6 +193,15 @@ spec: diskSizeGB: 128 osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmExtensions: + - name: CustomScript + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "Updating packages ..." + touch test_file + publisher: Microsoft.Azure.Extensions + version: "2.1" vmSize: ${AZURE_NODE_MACHINE_TYPE} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 diff --git a/templates/test/ci/patches/azuremachinepool-vmextension.yaml b/templates/test/ci/patches/azuremachinepool-vmextension.yaml new file mode 100644 index 00000000000..ecf228a77e1 --- /dev/null +++ b/templates/test/ci/patches/azuremachinepool-vmextension.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachinePool +metadata: + name: ${CLUSTER_NAME}-mp-0 + namespace: default +spec: + template: + vmExtensions: + - name: CustomScript + publisher: Microsoft.Azure.Extensions + version: '2.1' + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "Updating packages ..." + touch test_file diff --git a/templates/test/ci/patches/azuremachinetemplate-vmextension.yaml b/templates/test/ci/patches/azuremachinetemplate-vmextension.yaml new file mode 100644 index 00000000000..01e430013c2 --- /dev/null +++ b/templates/test/ci/patches/azuremachinetemplate-vmextension.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + vmExtensions: + - name: CustomScript + publisher: Microsoft.Azure.Extensions + version: '2.1' + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "Updating packages ..." + touch test_file diff --git a/templates/test/ci/prow-machine-pool/kustomization.yaml b/templates/test/ci/prow-machine-pool/kustomization.yaml index 0663d7e8a39..f669195d9fe 100644 --- a/templates/test/ci/prow-machine-pool/kustomization.yaml +++ b/templates/test/ci/prow-machine-pool/kustomization.yaml @@ -5,6 +5,7 @@ resources: - ../../../flavors/machinepool-windows - ../prow/cni-resource-set.yaml patchesStrategicMerge: + - ../patches/azuremachinepool-vmextension.yaml - ../patches/tags.yaml - ../patches/cluster-cni.yaml - ../patches/controller-manager.yaml diff --git a/templates/test/ci/prow/kustomization.yaml b/templates/test/ci/prow/kustomization.yaml index 4cd9a663d8e..6af851cb58a 100644 --- a/templates/test/ci/prow/kustomization.yaml +++ b/templates/test/ci/prow/kustomization.yaml @@ -18,6 +18,7 @@ patchesStrategicMerge: - ../patches/machine-deployment-worker-counts.yaml - ../../../azure-cluster-identity/azurecluster-identity-ref.yaml - ../../../flavors/base-windows-containerd/kubeadm-control-plane.yaml + - ../patches/azuremachinetemplate-vmextension.yaml - ../patches/windows-containerd-patch.yaml - ../patches/windows-enable-containerd-logger.yaml - ../patches/windows-csi-proxy-enabled.yaml diff --git a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml index 94c1f43d04c..5d791e4a003 100644 --- a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml +++ b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml @@ -269,6 +269,15 @@ spec: storageAccountType: Premium_LRS osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmExtensions: + - name: CustomScript + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "Updating packages ..." + touch test_file + publisher: Microsoft.Azure.Extensions + version: "2.1" vmSize: ${AZURE_NODE_MACHINE_TYPE} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml index eff06bb24a0..1202286f54b 100644 --- a/templates/test/dev/cluster-template-custom-builds.yaml +++ b/templates/test/dev/cluster-template-custom-builds.yaml @@ -267,6 +267,15 @@ spec: diskSizeGB: 128 osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmExtensions: + - name: CustomScript + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "Updating packages ..." + touch test_file + publisher: Microsoft.Azure.Extensions + version: "2.1" vmSize: ${AZURE_NODE_MACHINE_TYPE} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index 293baa5cf38..62356075e39 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -234,6 +234,18 @@ var _ = Describe("Workload cluster creation", func() { }, }, result) + By("Verifying a VM extension is present on the node", func() { + AzureVMExtensionsSpec(ctx, func() AzureVMExtensionsSpecInput { + return AzureVMExtensionsSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + VMName: clusterName + "-md-0", + ClusterName: clusterName, + SkipCleanup: skipCleanup, + } + }) + }) + By("Validating failure domains", func() { AzureFailureDomainsSpec(ctx, func() AzureFailureDomainsSpecInput { return AzureFailureDomainsSpecInput{ @@ -357,6 +369,18 @@ var _ = Describe("Workload cluster creation", func() { }, }, result) + By("Verifying a VM extension is present on the node", func() { + AzureVMSSExtensionsSpec(ctx, func() AzureVMSSExtensionsSpecInput { + return AzureVMSSExtensionsSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + VMName: clusterName + "-mp-0", + ClusterName: clusterName, + SkipCleanup: skipCleanup, + } + }) + }) + By("Running a security scanner", func() { KubescapeSpec(ctx, func() KubescapeSpecInput { return KubescapeSpecInput{ diff --git a/test/e2e/azure_vmextensions.go b/test/e2e/azure_vmextensions.go new file mode 100644 index 00000000000..e3fe14bc941 --- /dev/null +++ b/test/e2e/azure_vmextensions.go @@ -0,0 +1,120 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "strings" + + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute" + "github.com/Azure/go-autorest/autorest/azure/auth" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// AzureVMExtensionsSpecInput is the input for AzureVMExtensionsSpec. +type AzureVMExtensionsSpecInput struct { + BootstrapClusterProxy framework.ClusterProxy + Namespace *corev1.Namespace + VMName string + ClusterName string + SkipCleanup bool +} + +// AzureVMExtensionsSpec implements a test that verifies VM extensions are created and deleted. +func AzureVMExtensionsSpec(ctx context.Context, inputGetter func() AzureVMExtensionsSpecInput) { + var ( + specName = "azure-vmextensions" + input AzureVMExtensionsSpecInput + ) + + Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName) + + input = inputGetter() + Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName) + Expect(input.Namespace).ToNot(BeNil(), "Invalid argument. input.Namespace can't be nil when calling %s spec", specName) + Expect(input.ClusterName).ToNot(BeEmpty(), "Invalid argument. input.ClusterName can't be empty when calling %s spec", specName) + + By("creating a Kubernetes client to the workload cluster") + workloadClusterProxy := input.BootstrapClusterProxy.GetWorkloadCluster(ctx, input.Namespace.Name, input.ClusterName) + Expect(workloadClusterProxy).NotTo(BeNil()) + mgmtClient := bootstrapClusterProxy.GetClient() + Expect(mgmtClient).NotTo(BeNil()) + + By("Retrieving all machines from the machine template spec") + machineList := &infrav1.AzureMachineList{} + // list all of the requested objects within the cluster namespace with the cluster name label + Logf("Listing machines in namespace %s with label %s=%s", input.Namespace.Name, clusterv1.ClusterLabelName, workloadClusterProxy.GetName()) + err := mgmtClient.List(ctx, machineList, client.InNamespace(input.Namespace.Name), client.MatchingLabels{clusterv1.ClusterLabelName: workloadClusterProxy.GetName()}) + Expect(err).NotTo(HaveOccurred()) + + By("Ensuring the machine spec contains expected VM extensions") + for _, machine := range machineList.Items { + if strings.HasPrefix(machine.Name, input.VMName) { + Expect(len(machine.Spec.VMExtensions)).To(Equal(1)) + Expect(machine.Spec.VMExtensions[0].Name).To(Equal("CustomScript")) + } else { + Expect(len(machine.Spec.VMExtensions)).To(Equal(0)) + } + } + + By("Creating a VM and VM extension client") + // get subscription id + settings, err := auth.GetSettingsFromEnvironment() + Expect(err).NotTo(HaveOccurred()) + subscriptionID := settings.GetSubscriptionID() + auth, err := settings.GetAuthorizer() + Expect(err).NotTo(HaveOccurred()) + + // create a VM client + vmClient := compute.NewVirtualMachinesClient(subscriptionID) + vmClient.Authorizer = auth + + // create a VM extension client + extensionsClient := compute.NewVirtualMachineExtensionsClient(subscriptionID) + extensionsClient.Authorizer = auth + + vmListResults, err := vmClient.List(ctx, input.ClusterName, "") + Expect(err).NotTo(HaveOccurred()) + + By("Verifying VM extensions are created on Azure") + for _, machine := range vmListResults.Values() { + vmExtensionListResult, err := extensionsClient.List(ctx, input.ClusterName, *machine.Name, "") + Expect(err).NotTo(HaveOccurred()) + vmExtensionList := *vmExtensionListResult.Value + var vmExtensionNames []string + for _, vmExtension := range vmExtensionList { + vmExtensionNames = append(vmExtensionNames, *vmExtension.Name) + } + osName := string(machine.VirtualMachineProperties.StorageProfile.OsDisk.OsType) + if strings.HasPrefix(*machine.Name, input.VMName) { + Expect(len(vmExtensionList)).To(Equal(2)) + Expect(vmExtensionNames).To(ConsistOf("CustomScript", "CAPZ."+osName+".Bootstrapping")) + } else { + Expect(len(vmExtensionList)).To(Equal(1)) + Expect(vmExtensionNames).To(ConsistOf("CAPZ." + osName + ".Bootstrapping")) + } + } +} diff --git a/test/e2e/azure_vmssextensions.go b/test/e2e/azure_vmssextensions.go new file mode 100644 index 00000000000..cc21237bcd4 --- /dev/null +++ b/test/e2e/azure_vmssextensions.go @@ -0,0 +1,120 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "strings" + + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute" + "github.com/Azure/go-autorest/autorest/azure/auth" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// AzureVMSSExtensionsSpecInput is the input for AzureVMSSExtensionsSpec. +type AzureVMSSExtensionsSpecInput struct { + BootstrapClusterProxy framework.ClusterProxy + Namespace *corev1.Namespace + VMName string + ClusterName string + SkipCleanup bool +} + +// AzureVMSSExtensionsSpec implements a test that verifies VM extensions are created and deleted. +func AzureVMSSExtensionsSpec(ctx context.Context, inputGetter func() AzureVMSSExtensionsSpecInput) { + var ( + specName = "azure-vmssextensions" + input AzureVMSSExtensionsSpecInput + ) + + Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName) + + input = inputGetter() + Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName) + Expect(input.Namespace).ToNot(BeNil(), "Invalid argument. input.Namespace can't be nil when calling %s spec", specName) + Expect(input.ClusterName).ToNot(BeEmpty(), "Invalid argument. input.ClusterName can't be empty when calling %s spec", specName) + + By("creating a Kubernetes client to the workload cluster") + workloadClusterProxy := input.BootstrapClusterProxy.GetWorkloadCluster(ctx, input.Namespace.Name, input.ClusterName) + Expect(workloadClusterProxy).NotTo(BeNil()) + mgmtClient := bootstrapClusterProxy.GetClient() + Expect(mgmtClient).NotTo(BeNil()) + + By("Retrieving all machines from the machine template spec") + machinePoolList := &infrav1exp.AzureMachinePoolList{} + // list all of the requested objects within the cluster namespace with the cluster name label + Logf("Listing machine pools in namespace %s with label %s=%s", input.Namespace.Name, clusterv1.ClusterLabelName, workloadClusterProxy.GetName()) + err := mgmtClient.List(ctx, machinePoolList, client.InNamespace(input.Namespace.Name), client.MatchingLabels{clusterv1.ClusterLabelName: workloadClusterProxy.GetName()}) + Expect(err).NotTo(HaveOccurred()) + + By("Ensuring the machine spec contains expected VMSS extensions") + for _, machine := range machinePoolList.Items { + if strings.HasPrefix(machine.Name, input.VMName) { + Expect(len(machine.Spec.Template.VMExtensions)).To(Equal(1)) + Expect(machine.Spec.Template.VMExtensions[0].Name).To(Equal("CustomScript")) + } else { + Expect(len(machine.Spec.Template.VMExtensions)).To(Equal(0)) + } + } + + By("Creating a VMSS and VMSS extension client") + // get subscription id + settings, err := auth.GetSettingsFromEnvironment() + Expect(err).NotTo(HaveOccurred()) + subscriptionID := settings.GetSubscriptionID() + auth, err := settings.GetAuthorizer() + Expect(err).NotTo(HaveOccurred()) + + // create a VMSS client + vmssClient := compute.NewVirtualMachineScaleSetsClient(subscriptionID) + vmssClient.Authorizer = auth + + // create a VMSS extension client + extensionsClient := compute.NewVirtualMachineScaleSetExtensionsClient(subscriptionID) + extensionsClient.Authorizer = auth + + vmssListResults, err := vmssClient.List(ctx, input.ClusterName) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying VMSS extensions are created on Azure") + for _, machine := range vmssListResults.Values() { + vmssExtensionListResult, err := extensionsClient.List(ctx, input.ClusterName, *machine.Name) + Expect(err).NotTo(HaveOccurred()) + vmssExtensionList := vmssExtensionListResult.Values() + var vmssExtensionNames []string + for _, vmssExtension := range vmssExtensionList { + vmssExtensionNames = append(vmssExtensionNames, *vmssExtension.Name) + } + osName := string(machine.VirtualMachineScaleSetProperties.VirtualMachineProfile.StorageProfile.OsDisk.OsType) + if strings.HasPrefix(*machine.Name, input.VMName) { + Expect(len(vmssExtensionList)).To(Equal(2)) + Expect(vmssExtensionNames).To(ConsistOf("CustomScript", "CAPZ."+osName+".Bootstrapping")) + } else { + Expect(len(vmssExtensionList)).To(Equal(1)) + Expect(vmssExtensionNames).To(ConsistOf("CAPZ." + osName + ".Bootstrapping")) + } + } +}