From 2634777f4c4605fdc21e31b4f141cb161888c410 Mon Sep 17 00:00:00 2001 From: willie-yao Date: Tue, 6 Sep 2022 20:21:24 +0000 Subject: [PATCH] Add support for custom vm extensions --- api/v1alpha3/azuremachine_conversion.go | 4 + .../azuremachinetemplate_conversion.go | 4 + api/v1alpha3/zz_generated.conversion.go | 1 + api/v1alpha4/azuremachine_conversion.go | 4 + .../azuremachinetemplate_conversion.go | 4 + api/v1alpha4/zz_generated.conversion.go | 1 + api/v1beta1/azuremachine_types.go | 4 + api/v1beta1/types.go | 16 ++ api/v1beta1/zz_generated.deepcopy.go | 36 ++++ azure/scope/machine.go | 15 ++ azure/scope/machine_test.go | 77 +++++++++ azure/scope/machinepool.go | 15 ++ azure/scope/machinepool_test.go | 77 +++++++++ azure/services/scalesets/scalesets_test.go | 6 + .../services/scalesets/vmssextension_spec.go | 2 +- .../scalesets/vmssextension_spec_test.go | 96 +++++++++++ azure/services/vmextensions/spec.go | 2 +- azure/services/vmextensions/spec_test.go | 97 +++++++++++ azure/types.go | 2 + ...re.cluster.x-k8s.io_azuremachinepools.yaml | 36 ++++ ...ucture.cluster.x-k8s.io_azuremachines.yaml | 35 ++++ ...luster.x-k8s.io_azuremachinetemplates.yaml | 36 ++++ docs/book/src/SUMMARY.md | 1 + docs/book/src/topics/custom-vm-extensions.md | 68 ++++++++ .../v1alpha3/azuremachinepool_conversion.go | 4 + exp/api/v1alpha3/zz_generated.conversion.go | 1 + .../v1alpha4/azuremachinepool_conversion.go | 10 ++ exp/api/v1alpha4/zz_generated.conversion.go | 16 +- exp/api/v1beta1/azuremachinepool_types.go | 4 + exp/api/v1beta1/zz_generated.deepcopy.go | 7 + ...ow-ci-version-windows-containerd-2022.yaml | 9 + .../ci/cluster-template-prow-ci-version.yaml | 9 + ...ow-external-cloud-provider-ci-version.yaml | 9 + ...template-prow-machine-pool-ci-version.yaml | 9 + .../cluster-template-prow-machine-pool.yaml | 9 + templates/test/ci/cluster-template-prow.yaml | 9 + .../patches/azuremachinepool-vmextension.yaml | 17 ++ .../azuremachinetemplate-vmextension.yaml | 18 ++ .../ci/prow-machine-pool/kustomization.yaml | 1 + templates/test/ci/prow/kustomization.yaml | 1 + ...r-template-custom-builds-machine-pool.yaml | 9 + .../dev/cluster-template-custom-builds.yaml | 9 + test/e2e/azure_test.go | 70 ++++++++ test/e2e/azure_vmextensions.go | 156 ++++++++++++++++++ 44 files changed, 1004 insertions(+), 12 deletions(-) create mode 100644 azure/services/scalesets/vmssextension_spec_test.go create mode 100644 azure/services/vmextensions/spec_test.go create mode 100644 docs/book/src/topics/custom-vm-extensions.md create mode 100644 templates/test/ci/patches/azuremachinepool-vmextension.yaml create mode 100644 templates/test/ci/patches/azuremachinetemplate-vmextension.yaml create mode 100644 test/e2e/azure_vmextensions.go diff --git a/api/v1alpha3/azuremachine_conversion.go b/api/v1alpha3/azuremachine_conversion.go index dcf10192e42..e5994f1836c 100644 --- a/api/v1alpha3/azuremachine_conversion.go +++ b/api/v1alpha3/azuremachine_conversion.go @@ -62,6 +62,10 @@ func (src *AzureMachine) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.DNSServers = restored.Spec.DNSServers } + if len(restored.Spec.VMExtensions) > 0 { + dst.Spec.VMExtensions = restored.Spec.VMExtensions + } + dst.Spec.SubnetName = restored.Spec.SubnetName dst.Status.LongRunningOperationStates = restored.Status.LongRunningOperationStates diff --git a/api/v1alpha3/azuremachinetemplate_conversion.go b/api/v1alpha3/azuremachinetemplate_conversion.go index c3e80912b90..870d42c470f 100644 --- a/api/v1alpha3/azuremachinetemplate_conversion.go +++ b/api/v1alpha3/azuremachinetemplate_conversion.go @@ -65,6 +65,10 @@ func (src *AzureMachineTemplate) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.Template.Spec.DNSServers = restored.Spec.Template.Spec.DNSServers } + if len(restored.Spec.Template.Spec.VMExtensions) > 0 { + dst.Spec.Template.Spec.VMExtensions = restored.Spec.Template.Spec.VMExtensions + } + return nil } diff --git a/api/v1alpha3/zz_generated.conversion.go b/api/v1alpha3/zz_generated.conversion.go index 8cf4bfadfea..f16f2ec7661 100644 --- a/api/v1alpha3/zz_generated.conversion.go +++ b/api/v1alpha3/zz_generated.conversion.go @@ -907,6 +907,7 @@ func autoConvert_v1beta1_AzureMachineSpec_To_v1alpha3_AzureMachineSpec(in *v1bet out.SecurityProfile = (*SecurityProfile)(unsafe.Pointer(in.SecurityProfile)) // WARNING: in.SubnetName requires manual conversion: does not exist in peer-type // WARNING: in.DNSServers requires manual conversion: does not exist in peer-type + // WARNING: in.VMExtensions requires manual conversion: does not exist in peer-type return nil } diff --git a/api/v1alpha4/azuremachine_conversion.go b/api/v1alpha4/azuremachine_conversion.go index cf582ada834..85bb35638a3 100644 --- a/api/v1alpha4/azuremachine_conversion.go +++ b/api/v1alpha4/azuremachine_conversion.go @@ -48,6 +48,10 @@ func (src *AzureMachine) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.DNSServers = restored.Spec.DNSServers } + if len(restored.Spec.VMExtensions) > 0 { + dst.Spec.VMExtensions = restored.Spec.VMExtensions + } + return nil } diff --git a/api/v1alpha4/azuremachinetemplate_conversion.go b/api/v1alpha4/azuremachinetemplate_conversion.go index 4abf57698bd..4b67e7695aa 100644 --- a/api/v1alpha4/azuremachinetemplate_conversion.go +++ b/api/v1alpha4/azuremachinetemplate_conversion.go @@ -50,6 +50,10 @@ func (src *AzureMachineTemplate) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.Template.Spec.DNSServers = restored.Spec.Template.Spec.DNSServers } + if len(restored.Spec.Template.Spec.VMExtensions) > 0 { + dst.Spec.Template.Spec.VMExtensions = restored.Spec.Template.Spec.VMExtensions + } + return nil } diff --git a/api/v1alpha4/zz_generated.conversion.go b/api/v1alpha4/zz_generated.conversion.go index d6670115915..11f9161c194 100644 --- a/api/v1alpha4/zz_generated.conversion.go +++ b/api/v1alpha4/zz_generated.conversion.go @@ -1054,6 +1054,7 @@ func autoConvert_v1beta1_AzureMachineSpec_To_v1alpha4_AzureMachineSpec(in *v1bet out.SecurityProfile = (*SecurityProfile)(unsafe.Pointer(in.SecurityProfile)) out.SubnetName = in.SubnetName // WARNING: in.DNSServers requires manual conversion: does not exist in peer-type + // WARNING: in.VMExtensions requires manual conversion: does not exist in peer-type return nil } diff --git a/api/v1beta1/azuremachine_types.go b/api/v1beta1/azuremachine_types.go index c804823969d..3ba84e784b3 100644 --- a/api/v1beta1/azuremachine_types.go +++ b/api/v1beta1/azuremachine_types.go @@ -122,6 +122,10 @@ type AzureMachineSpec struct { // DNSServers adds a list of DNS Server IP addresses to the VM NICs. // +optional DNSServers []string `json:"dnsServers,omitempty"` + + // VMExtensions specifies a list of extensions to be added to the virtual machine. + // +optional + VMExtensions []VMExtension `json:"vmExtensions,omitempty"` } // SpotVMOptions defines the options relevant to running the Machine on Spot VMs. diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index d60fa3f47a5..70d4f00f234 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -530,6 +530,22 @@ type DataDisk struct { CachingType string `json:"cachingType,omitempty"` } +// VMExtension specifies the parameters for a custom VM extension. +type VMExtension struct { + // Name is the name of the extension. + Name string `json:"name"` + // Publisher is the name of the extension handler publisher. + Publisher string `json:"publisher"` + // Version specifies the version of the script handler. + Version string `json:"version"` + // Settings is a JSON formatted public settings for the extension. + // +optional + Settings Tags `json:"settings,omitempty"` + // ProtectedSettings is a JSON formatted protected settings for the extension. + // +optional + ProtectedSettings Tags `json:"protectedSettings,omitempty"` +} + // ManagedDiskParameters defines the parameters of a managed disk. type ManagedDiskParameters struct { // +optional diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 19b3f359952..7aae51b16dc 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -628,6 +628,13 @@ func (in *AzureMachineSpec) DeepCopyInto(out *AzureMachineSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.VMExtensions != nil { + in, out := &in.VMExtensions, &out.VMExtensions + *out = make([]VMExtension, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureMachineSpec. @@ -1701,6 +1708,35 @@ func (in *UserAssignedIdentity) DeepCopy() *UserAssignedIdentity { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VMExtension) DeepCopyInto(out *VMExtension) { + *out = *in + if in.Settings != nil { + in, out := &in.Settings, &out.Settings + *out = make(Tags, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.ProtectedSettings != nil { + in, out := &in.ProtectedSettings, &out.ProtectedSettings + *out = make(Tags, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VMExtension. +func (in *VMExtension) DeepCopy() *VMExtension { + if in == nil { + return nil + } + out := new(VMExtension) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VnetClassSpec) DeepCopyInto(out *VnetClassSpec) { *out = *in diff --git a/azure/scope/machine.go b/azure/scope/machine.go index c480d75b1ce..5f7b65bff12 100644 --- a/azure/scope/machine.go +++ b/azure/scope/machine.go @@ -333,6 +333,21 @@ func (m *MachineScope) HasSystemAssignedIdentity() bool { // VMExtensionSpecs returns the VM extension specs. func (m *MachineScope) VMExtensionSpecs() []azure.ResourceSpecGetter { var extensionSpecs = []azure.ResourceSpecGetter{} + for _, extension := range m.AzureMachine.Spec.VMExtensions { + extensionSpecs = append(extensionSpecs, &vmextensions.VMExtensionSpec{ + ExtensionSpec: azure.ExtensionSpec{ + Name: extension.Name, + VMName: m.Name(), + Publisher: extension.Publisher, + Version: extension.Version, + Settings: extension.Settings, + ProtectedSettings: extension.ProtectedSettings, + }, + ResourceGroup: m.ResourceGroup(), + Location: m.Location(), + }) + } + bootstrapExtensionSpec := azure.GetBootstrappingVMExtension(m.AzureMachine.Spec.OSDisk.OSType, m.CloudEnvironment(), m.Name()) if bootstrapExtensionSpec != nil { diff --git a/azure/scope/machine_test.go b/azure/scope/machine_test.go index 3ce296756ed..64194c05527 100644 --- a/azure/scope/machine_test.go +++ b/azure/scope/machine_test.go @@ -728,6 +728,83 @@ func TestMachineScope_VMExtensionSpecs(t *testing.T) { }, want: []azure.ResourceSpecGetter{}, }, + { + name: "If a custom VM extension is specified, it returns the custom VM extension", + machineScope: MachineScope{ + Machine: &clusterv1.Machine{}, + AzureMachine: &infrav1.AzureMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "machine-name", + }, + Spec: infrav1.AzureMachineSpec{ + OSDisk: infrav1.OSDisk{ + OSType: "Linux", + }, + VMExtensions: []infrav1.VMExtension{ + { + Name: "custom-vm-extension", + Publisher: "Microsoft.Azure.Extensions", + Version: "2.0", + Settings: map[string]string{ + "timestamp": "1234567890", + }, + ProtectedSettings: map[string]string{ + "commandToExecute": "echo hello world", + }, + }, + }, + }, + }, + ClusterScoper: &ClusterScope{ + AzureClients: AzureClients{ + EnvironmentSettings: auth.EnvironmentSettings{ + Environment: autorestazure.Environment{ + Name: autorestazure.PublicCloud.Name, + }, + }, + }, + AzureCluster: &infrav1.AzureCluster{ + Spec: infrav1.AzureClusterSpec{ + ResourceGroup: "my-rg", + AzureClusterClassSpec: infrav1.AzureClusterClassSpec{ + Location: "westus", + }, + }, + }, + }, + }, + want: []azure.ResourceSpecGetter{ + &vmextensions.VMExtensionSpec{ + ExtensionSpec: azure.ExtensionSpec{ + Name: "custom-vm-extension", + VMName: "machine-name", + Publisher: "Microsoft.Azure.Extensions", + Version: "2.0", + Settings: map[string]string{ + "timestamp": "1234567890", + }, + ProtectedSettings: map[string]string{ + "commandToExecute": "echo hello world", + }, + }, + ResourceGroup: "my-rg", + Location: "westus", + }, + &vmextensions.VMExtensionSpec{ + ExtensionSpec: azure.ExtensionSpec{ + Name: "CAPZ.Linux.Bootstrapping", + VMName: "machine-name", + Publisher: "Microsoft.Azure.ContainerUpstream", + Version: "1.0", + ProtectedSettings: map[string]string{ + "commandToExecute": azure.LinuxBootstrapExtensionCommand, + }, + }, + ResourceGroup: "my-rg", + Location: "westus", + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/azure/scope/machinepool.go b/azure/scope/machinepool.go index b55096c96ef..1d515a76e73 100644 --- a/azure/scope/machinepool.go +++ b/azure/scope/machinepool.go @@ -615,6 +615,21 @@ func (m *MachinePoolScope) HasSystemAssignedIdentity() bool { // VMSSExtensionSpecs returns the VMSS extension specs. func (m *MachinePoolScope) VMSSExtensionSpecs() []azure.ResourceSpecGetter { var extensionSpecs = []azure.ResourceSpecGetter{} + + for _, extension := range m.AzureMachinePool.Spec.Template.VMExtensions { + extensionSpecs = append(extensionSpecs, &scalesets.VMSSExtensionSpec{ + ExtensionSpec: azure.ExtensionSpec{ + Name: extension.Name, + VMName: m.Name(), + Publisher: extension.Publisher, + Version: extension.Version, + Settings: extension.Settings, + ProtectedSettings: extension.ProtectedSettings, + }, + ResourceGroup: m.ResourceGroup(), + }) + } + bootstrapExtensionSpec := azure.GetBootstrappingVMExtension(m.AzureMachinePool.Spec.Template.OSDisk.OSType, m.CloudEnvironment(), m.Name()) if bootstrapExtensionSpec != nil { diff --git a/azure/scope/machinepool_test.go b/azure/scope/machinepool_test.go index cd82f910349..fbc6ead0291 100644 --- a/azure/scope/machinepool_test.go +++ b/azure/scope/machinepool_test.go @@ -872,6 +872,83 @@ func TestMachinePoolScope_VMSSExtensionSpecs(t *testing.T) { }, want: []azure.ResourceSpecGetter{}, }, + { + name: "If a custom VM extension is specified, it returns the custom VM extension", + machinePoolScope: MachinePoolScope{ + MachinePool: &expv1.MachinePool{}, + AzureMachinePool: &infrav1exp.AzureMachinePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "machinepool-name", + }, + Spec: infrav1exp.AzureMachinePoolSpec{ + Template: infrav1exp.AzureMachinePoolMachineTemplate{ + OSDisk: infrav1.OSDisk{ + OSType: "Linux", + }, + VMExtensions: []infrav1.VMExtension{ + { + Name: "custom-vm-extension", + Publisher: "Microsoft.Azure.Extensions", + Version: "2.0", + Settings: map[string]string{ + "timestamp": "1234567890", + }, + ProtectedSettings: map[string]string{ + "commandToExecute": "echo hello world", + }, + }, + }, + }, + }, + }, + ClusterScoper: &ClusterScope{ + AzureClients: AzureClients{ + EnvironmentSettings: auth.EnvironmentSettings{ + Environment: autorestazure.Environment{ + Name: autorestazure.PublicCloud.Name, + }, + }, + }, + AzureCluster: &infrav1.AzureCluster{ + Spec: infrav1.AzureClusterSpec{ + ResourceGroup: "my-rg", + AzureClusterClassSpec: infrav1.AzureClusterClassSpec{ + Location: "westus", + }, + }, + }, + }, + }, + want: []azure.ResourceSpecGetter{ + &scalesets.VMSSExtensionSpec{ + ExtensionSpec: azure.ExtensionSpec{ + Name: "custom-vm-extension", + VMName: "machinepool-name", + Publisher: "Microsoft.Azure.Extensions", + Version: "2.0", + Settings: map[string]string{ + "timestamp": "1234567890", + }, + ProtectedSettings: map[string]string{ + "commandToExecute": "echo hello world", + }, + }, + ResourceGroup: "my-rg", + }, + &scalesets.VMSSExtensionSpec{ + ExtensionSpec: azure.ExtensionSpec{ + Name: "CAPZ.Linux.Bootstrapping", + VMName: "machinepool-name", + Publisher: "Microsoft.Azure.ContainerUpstream", + Version: "1.0", + ProtectedSettings: map[string]string{ + "commandToExecute": azure.LinuxBootstrapExtensionCommand, + }, + }, + ResourceGroup: "my-rg", + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/azure/services/scalesets/scalesets_test.go b/azure/services/scalesets/scalesets_test.go index 7e66d1f12bc..715c534668f 100644 --- a/azure/services/scalesets/scalesets_test.go +++ b/azure/services/scalesets/scalesets_test.go @@ -1153,6 +1153,9 @@ func newDefaultVMSS(vmSize string) compute.VirtualMachineScaleSet { Publisher: to.StringPtr("somePublisher"), Type: to.StringPtr("someExtension"), TypeHandlerVersion: to.StringPtr("someVersion"), + Settings: map[string]string{ + "someSetting": "someValue", + }, ProtectedSettings: map[string]string{ "commandToExecute": "echo hello", }, @@ -1364,6 +1367,9 @@ func setupVMSSExpectationsWithoutVMImage(s *mock_scalesets.MockScaleSetScopeMock VMName: "my-vmss", Publisher: "somePublisher", Version: "someVersion", + Settings: map[string]string{ + "someSetting": "someValue", + }, ProtectedSettings: map[string]string{ "commandToExecute": "echo hello", }, diff --git a/azure/services/scalesets/vmssextension_spec.go b/azure/services/scalesets/vmssextension_spec.go index 27f663c8153..0c77c5c25d8 100644 --- a/azure/services/scalesets/vmssextension_spec.go +++ b/azure/services/scalesets/vmssextension_spec.go @@ -62,7 +62,7 @@ func (s *VMSSExtensionSpec) Parameters(existing interface{}) (interface{}, error Publisher: to.StringPtr(s.Publisher), Type: to.StringPtr(s.Name), TypeHandlerVersion: to.StringPtr(s.Version), - Settings: nil, + Settings: s.Settings, ProtectedSettings: s.ProtectedSettings, }, }, nil diff --git a/azure/services/scalesets/vmssextension_spec_test.go b/azure/services/scalesets/vmssextension_spec_test.go new file mode 100644 index 00000000000..8671e19766b --- /dev/null +++ b/azure/services/scalesets/vmssextension_spec_test.go @@ -0,0 +1,96 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scalesets + +import ( + "testing" + + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute" + "github.com/Azure/go-autorest/autorest/to" + . "github.com/onsi/gomega" + "sigs.k8s.io/cluster-api-provider-azure/azure" +) + +var ( + fakeVMSSExtensionSpec = VMSSExtensionSpec{ + azure.ExtensionSpec{ + Name: "my-vm-extension", + VMName: "my-vm", + Publisher: "my-publisher", + Version: "1.0", + Settings: map[string]string{"my-setting": "my-value"}, + ProtectedSettings: map[string]string{"my-protected-setting": "my-protected-value"}, + }, + "my-rg", + } + + fakeVMSSExtensionParams = compute.VirtualMachineScaleSetExtension{ + Name: to.StringPtr("my-vm-extension"), + VirtualMachineScaleSetExtensionProperties: &compute.VirtualMachineScaleSetExtensionProperties{ + Publisher: to.StringPtr("my-publisher"), + Type: to.StringPtr("my-vm-extension"), + TypeHandlerVersion: to.StringPtr("1.0"), + Settings: map[string]string{"my-setting": "my-value"}, + ProtectedSettings: map[string]string{"my-protected-setting": "my-protected-value"}, + }, + } +) + +func TestParameters(t *testing.T) { + testcases := []struct { + name string + spec *VMSSExtensionSpec + existing interface{} + expect func(g *WithT, result interface{}) + expectedError string + }{ + { + name: "get parameters for vmextension", + spec: &fakeVMSSExtensionSpec, + existing: nil, + expect: func(g *WithT, result interface{}) { + g.Expect(result).To(Equal(fakeVMSSExtensionParams)) + }, + expectedError: "", + }, + { + name: "vmextension that already exists", + spec: &fakeVMSSExtensionSpec, + existing: fakeVMSSExtensionParams, + expect: func(g *WithT, result interface{}) { + g.Expect(result).To(BeNil()) + }, + expectedError: "", + }, + } + for _, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + t.Parallel() + + result, err := tc.spec.Parameters(tc.existing) + if tc.expectedError != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err).To(MatchError(tc.expectedError)) + } else { + g.Expect(err).NotTo(HaveOccurred()) + } + tc.expect(g, result) + }) + } +} diff --git a/azure/services/vmextensions/spec.go b/azure/services/vmextensions/spec.go index 89d491b37cd..e99cf16ac0e 100644 --- a/azure/services/vmextensions/spec.go +++ b/azure/services/vmextensions/spec.go @@ -62,7 +62,7 @@ func (s *VMExtensionSpec) Parameters(existing interface{}) (interface{}, error) Publisher: to.StringPtr(s.Publisher), Type: to.StringPtr(s.Name), TypeHandlerVersion: to.StringPtr(s.Version), - Settings: nil, + Settings: s.Settings, ProtectedSettings: s.ProtectedSettings, }, Location: to.StringPtr(s.Location), diff --git a/azure/services/vmextensions/spec_test.go b/azure/services/vmextensions/spec_test.go new file mode 100644 index 00000000000..67d0bce0334 --- /dev/null +++ b/azure/services/vmextensions/spec_test.go @@ -0,0 +1,97 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vmextensions + +import ( + "testing" + + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute" + "github.com/Azure/go-autorest/autorest/to" + . "github.com/onsi/gomega" + "sigs.k8s.io/cluster-api-provider-azure/azure" +) + +var ( + fakeVMExtensionSpec = VMExtensionSpec{ + azure.ExtensionSpec{ + Name: "my-vm-extension", + VMName: "my-vm", + Publisher: "my-publisher", + Version: "1.0", + Settings: map[string]string{"my-setting": "my-value"}, + ProtectedSettings: map[string]string{"my-protected-setting": "my-protected-value"}, + }, + "my-rg", + "my-location", + } + + fakeVMExtensionParams = compute.VirtualMachineExtension{ + VirtualMachineExtensionProperties: &compute.VirtualMachineExtensionProperties{ + Publisher: to.StringPtr("my-publisher"), + Type: to.StringPtr("my-vm-extension"), + TypeHandlerVersion: to.StringPtr("1.0"), + Settings: map[string]string{"my-setting": "my-value"}, + ProtectedSettings: map[string]string{"my-protected-setting": "my-protected-value"}, + }, + Location: to.StringPtr("my-location"), + } +) + +func TestParameters(t *testing.T) { + testcases := []struct { + name string + spec *VMExtensionSpec + existing interface{} + expect func(g *WithT, result interface{}) + expectedError string + }{ + { + name: "get parameters for vmextension", + spec: &fakeVMExtensionSpec, + existing: nil, + expect: func(g *WithT, result interface{}) { + g.Expect(result).To(Equal(fakeVMExtensionParams)) + }, + expectedError: "", + }, + { + name: "vmextension that already exists", + spec: &fakeVMExtensionSpec, + existing: fakeVMExtensionParams, + expect: func(g *WithT, result interface{}) { + g.Expect(result).To(BeNil()) + }, + expectedError: "", + }, + } + for _, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + t.Parallel() + + result, err := tc.spec.Parameters(tc.existing) + if tc.expectedError != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err).To(MatchError(tc.expectedError)) + } else { + g.Expect(err).NotTo(HaveOccurred()) + } + tc.expect(g, result) + }) + } +} diff --git a/azure/types.go b/azure/types.go index df20d352657..81756ab4e69 100644 --- a/azure/types.go +++ b/azure/types.go @@ -64,6 +64,7 @@ type ScaleSetSpec struct { SpotVMOptions *infrav1.SpotVMOptions AdditionalCapabilities *infrav1.AdditionalCapabilities FailureDomains []string + VMExtensions []infrav1.VMExtension } // TagsSpec defines the specification for a set of tags. @@ -82,6 +83,7 @@ type ExtensionSpec struct { VMName string Publisher string Version string + Settings map[string]string ProtectedSettings map[string]string } diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachinepools.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachinepools.yaml index a45716ba497..31f2ac78907 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachinepools.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachinepools.yaml @@ -1792,6 +1792,42 @@ spec: VMSS scheduled events termination notification with specified timeout allowed values are between 5 and 15 (mins) type: integer + vmExtensions: + description: VMExtensions specifies a list of extensions to be + added to the scale set. + items: + description: VMExtension specifies the parameters for a custom + VM extension. + properties: + name: + description: Name is the name of the extension. + type: string + protectedSettings: + additionalProperties: + type: string + description: ProtectedSettings is a JSON formatted protected + settings for the extension. + type: object + publisher: + description: Publisher is the name of the extension handler + publisher. + type: string + settings: + additionalProperties: + type: string + description: Settings is a JSON formatted public settings + for the extension. + type: object + version: + description: Version specifies the version of the script + handler. + type: string + required: + - name + - publisher + - version + type: object + type: array vmSize: description: VMSize is the size of the Virtual Machine to build. See https://docs.microsoft.com/en-us/rest/api/compute/virtualmachines/createorupdate#virtualmachinesizetypes diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachines.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachines.yaml index ddf1a5928f7..c3cb0303399 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachines.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachines.yaml @@ -1434,6 +1434,41 @@ spec: - providerID type: object type: array + vmExtensions: + description: VMExtensions specifies a list of extensions to be added + to the virtual machine. + items: + description: VMExtension specifies the parameters for a custom VM + extension. + properties: + name: + description: Name is the name of the extension. + type: string + protectedSettings: + additionalProperties: + type: string + description: ProtectedSettings is a JSON formatted protected + settings for the extension. + type: object + publisher: + description: Publisher is the name of the extension handler + publisher. + type: string + settings: + additionalProperties: + type: string + description: Settings is a JSON formatted public settings for + the extension. + type: object + version: + description: Version specifies the version of the script handler. + type: string + required: + - name + - publisher + - version + type: object + type: array vmSize: type: string required: diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachinetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachinetemplates.yaml index 1a246a2e66b..a223d3fee34 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachinetemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremachinetemplates.yaml @@ -1219,6 +1219,42 @@ spec: - providerID type: object type: array + vmExtensions: + description: VMExtensions specifies a list of extensions to + be added to the virtual machine. + items: + description: VMExtension specifies the parameters for a + custom VM extension. + properties: + name: + description: Name is the name of the extension. + type: string + protectedSettings: + additionalProperties: + type: string + description: ProtectedSettings is a JSON formatted protected + settings for the extension. + type: object + publisher: + description: Publisher is the name of the extension + handler publisher. + type: string + settings: + additionalProperties: + type: string + description: Settings is a JSON formatted public settings + for the extension. + type: object + version: + description: Version specifies the version of the script + handler. + type: string + required: + - name + - publisher + - version + type: object + type: array vmSize: type: string required: diff --git a/docs/book/src/SUMMARY.md b/docs/book/src/SUMMARY.md index e238c746648..cde1474bc20 100644 --- a/docs/book/src/SUMMARY.md +++ b/docs/book/src/SUMMARY.md @@ -11,6 +11,7 @@ - [Control Plane Outbound Load Balancer](./topics/control-plane-outbound-lb.md) - [Custom Private DNS Zone Name](./topics/custom-dns.md) - [Custom Images](./topics/custom-images.md) + - [Custom VM Extensions](./topics/custom-vm-extensions.md) - [Data Disks](./topics/data-disks.md) - [OS Disk](./topics/os-disk.md) - [Dual-Stack](./topics/dual-stack.md) diff --git a/docs/book/src/topics/custom-vm-extensions.md b/docs/book/src/topics/custom-vm-extensions.md new file mode 100644 index 00000000000..b81b1f6ae97 --- /dev/null +++ b/docs/book/src/topics/custom-vm-extensions.md @@ -0,0 +1,68 @@ +# Custom VM Extensions + +## Overview +CAPZ allows you to specify custom extensions for your Azure resources. This is useful for running custom scripts or installing custom software on your machines. You can specify custom extensions for the following resources: + - AzureMachine + - AzureMachinePool + +## Discovering available extensions +The user is responsible for ensuring that the custom extension is compatible with the underlying image. Many VM extensions are available for use with Azure VMs. To see a complete list, use the Azure CLI command `az vm extension image list`. + +```bash +$ az vm extension image list --location westus --output table +``` + +## Warning +VM extensions are specific to the operating system of the VM. For example, a Linux extension will not work on a Windows VM and vice versa. See the Azure documentation for more information. +- [Virtual machine extensions and features for Linux](https://learn.microsoft.com/en-us/azure/virtual-machines/extensions/features-linux?tabs=azure-cli) +- [Virtual machine extensions and features for Windows](https://learn.microsoft.com/en-us/azure/virtual-machines/extensions/features-windows?tabs=azure-cli) + +## Custom extensions for AzureMachine +To specify custom extensions for AzureMachines, you can add them to the `spec.template.spec.vmExtensions` field of your `AzureMachineTemplate`. The following fields are available: +- `name` (required): The name of the extension. +- `publisher` (required): The name of the extension publisher. +- `version` (required): The version of the extension. +- `settings` (optional): A set of key-value pairs containing settings for the extension. +- `protectedSettings` (optional): A set of key-value pairs containing protected settings for the extension. The information in this field is encrypted and decrypted only on the VM itself. + +For example, the following `AzureMachineTemplate` spec specifies a custom extension that installs the `CustomScript` extension on the machine: + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: test-machine-template + namespace: default +spec: + template: + spec: + vmExtensions: + - name: CustomScript + publisher: Microsoft.Azure.Extensions + version: '2.1' + settings: + fileUris: https://raw.githubusercontent.com/me/project/hello.sh + protectedSettings: + commandToExecute: ./hello.sh +``` + +## Custom extensions for AzureMachinePool +Similarly, to specify custom extensions for AzureMachinePools, you can add them to the `spec.template.vmExtensions` field of your `AzureMachinePool`. For example, the following `AzureMachinePool` spec specifies a custom extension that installs the `CustomScript` extension on the machine: + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachinePool +metadata: + name: test-machine-pool + namespace: default +spec: + template: + vmExtensions: + - name: CustomScript + publisher: Microsoft.Azure.Extensions + version: '2.1' + settings: + fileUris: https://raw.githubusercontent.com/me/project/hello.sh + protectedSettings: + commandToExecute: ./hello.sh +``` diff --git a/exp/api/v1alpha3/azuremachinepool_conversion.go b/exp/api/v1alpha3/azuremachinepool_conversion.go index 86574c7663f..1bf5fec349d 100644 --- a/exp/api/v1alpha3/azuremachinepool_conversion.go +++ b/exp/api/v1alpha3/azuremachinepool_conversion.go @@ -85,6 +85,10 @@ func (src *AzureMachinePool) ConvertTo(dstRaw conversion.Hub) error { } } + if len(restored.Spec.Template.VMExtensions) > 0 { + dst.Spec.Template.VMExtensions = restored.Spec.Template.VMExtensions + } + return nil } diff --git a/exp/api/v1alpha3/zz_generated.conversion.go b/exp/api/v1alpha3/zz_generated.conversion.go index 1012c6945dd..e6e001a507b 100644 --- a/exp/api/v1alpha3/zz_generated.conversion.go +++ b/exp/api/v1alpha3/zz_generated.conversion.go @@ -459,6 +459,7 @@ func autoConvert_v1beta1_AzureMachinePoolMachineTemplate_To_v1alpha3_AzureMachin out.SecurityProfile = (*clusterapiproviderazureapiv1alpha3.SecurityProfile)(unsafe.Pointer(in.SecurityProfile)) out.SpotVMOptions = (*clusterapiproviderazureapiv1alpha3.SpotVMOptions)(unsafe.Pointer(in.SpotVMOptions)) // WARNING: in.SubnetName requires manual conversion: does not exist in peer-type + // WARNING: in.VMExtensions requires manual conversion: does not exist in peer-type return nil } diff --git a/exp/api/v1alpha4/azuremachinepool_conversion.go b/exp/api/v1alpha4/azuremachinepool_conversion.go index 90c19148545..18de2d99cf5 100644 --- a/exp/api/v1alpha4/azuremachinepool_conversion.go +++ b/exp/api/v1alpha4/azuremachinepool_conversion.go @@ -17,6 +17,7 @@ limitations under the License. package v1alpha4 import ( + convert "k8s.io/apimachinery/pkg/conversion" infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" utilconversion "sigs.k8s.io/cluster-api/util/conversion" "sigs.k8s.io/controller-runtime/pkg/conversion" @@ -43,6 +44,10 @@ func (src *AzureMachinePool) ConvertTo(dstRaw conversion.Hub) error { dst.Status.Image.ComputeGallery = restored.Status.Image.ComputeGallery } + if len(restored.Spec.Template.VMExtensions) > 0 { + dst.Spec.Template.VMExtensions = restored.Spec.Template.VMExtensions + } + return nil } @@ -68,3 +73,8 @@ func (dst *AzureMachinePoolList) ConvertFrom(srcRaw conversion.Hub) error { src := srcRaw.(*infrav1exp.AzureMachinePoolList) return Convert_v1beta1_AzureMachinePoolList_To_v1alpha4_AzureMachinePoolList(src, dst, nil) } + +// Convert_v1beta1_AzureMachinePoolMachineTemplate_To_v1alpha4_AzureMachinePoolMachineTemplate converts an Azure Machine Pool Machine Template from v1beta1 to v1alpha4. +func Convert_v1beta1_AzureMachinePoolMachineTemplate_To_v1alpha4_AzureMachinePoolMachineTemplate(in *infrav1exp.AzureMachinePoolMachineTemplate, out *AzureMachinePoolMachineTemplate, s convert.Scope) error { + return autoConvert_v1beta1_AzureMachinePoolMachineTemplate_To_v1alpha4_AzureMachinePoolMachineTemplate(in, out, s) +} diff --git a/exp/api/v1alpha4/zz_generated.conversion.go b/exp/api/v1alpha4/zz_generated.conversion.go index c95dade2bd2..c34c29438d4 100644 --- a/exp/api/v1alpha4/zz_generated.conversion.go +++ b/exp/api/v1alpha4/zz_generated.conversion.go @@ -149,11 +149,6 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1beta1.AzureMachinePoolMachineTemplate)(nil), (*AzureMachinePoolMachineTemplate)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta1_AzureMachinePoolMachineTemplate_To_v1alpha4_AzureMachinePoolMachineTemplate(a.(*v1beta1.AzureMachinePoolMachineTemplate), b.(*AzureMachinePoolMachineTemplate), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*AzureMachinePoolSpec)(nil), (*v1beta1.AzureMachinePoolSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha4_AzureMachinePoolSpec_To_v1beta1_AzureMachinePoolSpec(a.(*AzureMachinePoolSpec), b.(*v1beta1.AzureMachinePoolSpec), scope) }); err != nil { @@ -344,6 +339,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1beta1.AzureMachinePoolMachineTemplate)(nil), (*AzureMachinePoolMachineTemplate)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta1_AzureMachinePoolMachineTemplate_To_v1alpha4_AzureMachinePoolMachineTemplate(a.(*v1beta1.AzureMachinePoolMachineTemplate), b.(*AzureMachinePoolMachineTemplate), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1beta1.AzureManagedControlPlaneSpec)(nil), (*AzureManagedControlPlaneSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta1_AzureManagedControlPlaneSpec_To_v1alpha4_AzureManagedControlPlaneSpec(a.(*v1beta1.AzureManagedControlPlaneSpec), b.(*AzureManagedControlPlaneSpec), scope) }); err != nil { @@ -715,14 +715,10 @@ func autoConvert_v1beta1_AzureMachinePoolMachineTemplate_To_v1alpha4_AzureMachin out.SecurityProfile = (*clusterapiproviderazureapiv1alpha4.SecurityProfile)(unsafe.Pointer(in.SecurityProfile)) out.SpotVMOptions = (*clusterapiproviderazureapiv1alpha4.SpotVMOptions)(unsafe.Pointer(in.SpotVMOptions)) out.SubnetName = in.SubnetName + // WARNING: in.VMExtensions requires manual conversion: does not exist in peer-type return nil } -// Convert_v1beta1_AzureMachinePoolMachineTemplate_To_v1alpha4_AzureMachinePoolMachineTemplate is an autogenerated conversion function. -func Convert_v1beta1_AzureMachinePoolMachineTemplate_To_v1alpha4_AzureMachinePoolMachineTemplate(in *v1beta1.AzureMachinePoolMachineTemplate, out *AzureMachinePoolMachineTemplate, s conversion.Scope) error { - return autoConvert_v1beta1_AzureMachinePoolMachineTemplate_To_v1alpha4_AzureMachinePoolMachineTemplate(in, out, s) -} - func autoConvert_v1alpha4_AzureMachinePoolSpec_To_v1beta1_AzureMachinePoolSpec(in *AzureMachinePoolSpec, out *v1beta1.AzureMachinePoolSpec, s conversion.Scope) error { out.Location = in.Location if err := Convert_v1alpha4_AzureMachinePoolMachineTemplate_To_v1beta1_AzureMachinePoolMachineTemplate(&in.Template, &out.Template, s); err != nil { diff --git a/exp/api/v1beta1/azuremachinepool_types.go b/exp/api/v1beta1/azuremachinepool_types.go index 7f3b6a50546..da873195063 100644 --- a/exp/api/v1beta1/azuremachinepool_types.go +++ b/exp/api/v1beta1/azuremachinepool_types.go @@ -87,6 +87,10 @@ type ( // SubnetName selects the Subnet where the VMSS will be placed // +optional SubnetName string `json:"subnetName,omitempty"` + + // VMExtensions specifies a list of extensions to be added to the scale set. + // +optional + VMExtensions []infrav1.VMExtension `json:"vmExtensions,omitempty"` } // AzureMachinePoolSpec defines the desired state of AzureMachinePool. diff --git a/exp/api/v1beta1/zz_generated.deepcopy.go b/exp/api/v1beta1/zz_generated.deepcopy.go index 777e563d3df..03250a54dc9 100644 --- a/exp/api/v1beta1/zz_generated.deepcopy.go +++ b/exp/api/v1beta1/zz_generated.deepcopy.go @@ -364,6 +364,13 @@ func (in *AzureMachinePoolMachineTemplate) DeepCopyInto(out *AzureMachinePoolMac *out = new(apiv1beta1.SpotVMOptions) (*in).DeepCopyInto(*out) } + if in.VMExtensions != nil { + in, out := &in.VMExtensions, &out.VMExtensions + *out = make([]apiv1beta1.VMExtension, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureMachinePoolMachineTemplate. diff --git a/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml b/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml index f6af9380e41..c8ae4577faa 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-windows-containerd-2022.yaml @@ -278,6 +278,15 @@ spec: diskSizeGB: 128 osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmExtensions: + - name: CustomScript + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "This script is a no-op used for extension testing purposes ..." + touch test_file + publisher: Microsoft.Azure.Extensions + version: "2.1" vmSize: ${AZURE_NODE_MACHINE_TYPE} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 diff --git a/templates/test/ci/cluster-template-prow-ci-version.yaml b/templates/test/ci/cluster-template-prow-ci-version.yaml index 7f2cc727767..574356614da 100644 --- a/templates/test/ci/cluster-template-prow-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version.yaml @@ -278,6 +278,15 @@ spec: diskSizeGB: 128 osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmExtensions: + - name: CustomScript + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "This script is a no-op used for extension testing purposes ..." + touch test_file + publisher: Microsoft.Azure.Extensions + version: "2.1" vmSize: ${AZURE_NODE_MACHINE_TYPE} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 diff --git a/templates/test/ci/cluster-template-prow-external-cloud-provider-ci-version.yaml b/templates/test/ci/cluster-template-prow-external-cloud-provider-ci-version.yaml index a3c41af6aa1..261d017688a 100644 --- a/templates/test/ci/cluster-template-prow-external-cloud-provider-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-external-cloud-provider-ci-version.yaml @@ -281,6 +281,15 @@ spec: diskSizeGB: 128 osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmExtensions: + - name: CustomScript + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "This script is a no-op used for extension testing purposes ..." + touch test_file + publisher: Microsoft.Azure.Extensions + version: "2.1" vmSize: ${AZURE_NODE_MACHINE_TYPE} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 diff --git a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml index fd1a15b37bf..1a859269414 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml @@ -280,6 +280,15 @@ spec: storageAccountType: Premium_LRS osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmExtensions: + - name: CustomScript + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "This script is a no-op used for extension testing purposes ..." + touch test_file + publisher: Microsoft.Azure.Extensions + version: "2.1" vmSize: ${AZURE_NODE_MACHINE_TYPE} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 diff --git a/templates/test/ci/cluster-template-prow-machine-pool.yaml b/templates/test/ci/cluster-template-prow-machine-pool.yaml index d5bde2ff7dc..d5118c86ef9 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool.yaml @@ -197,6 +197,15 @@ spec: storageAccountType: Premium_LRS osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmExtensions: + - name: CustomScript + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "This script is a no-op used for extension testing purposes ..." + touch test_file + publisher: Microsoft.Azure.Extensions + version: "2.1" vmSize: ${AZURE_NODE_MACHINE_TYPE} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 diff --git a/templates/test/ci/cluster-template-prow.yaml b/templates/test/ci/cluster-template-prow.yaml index 87a995d91fa..a8a4fbce37d 100644 --- a/templates/test/ci/cluster-template-prow.yaml +++ b/templates/test/ci/cluster-template-prow.yaml @@ -193,6 +193,15 @@ spec: diskSizeGB: 128 osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmExtensions: + - name: CustomScript + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "This script is a no-op used for extension testing purposes ..." + touch test_file + publisher: Microsoft.Azure.Extensions + version: "2.1" vmSize: ${AZURE_NODE_MACHINE_TYPE} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 diff --git a/templates/test/ci/patches/azuremachinepool-vmextension.yaml b/templates/test/ci/patches/azuremachinepool-vmextension.yaml new file mode 100644 index 00000000000..6ad6f7190eb --- /dev/null +++ b/templates/test/ci/patches/azuremachinepool-vmextension.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachinePool +metadata: + name: ${CLUSTER_NAME}-mp-0 + namespace: default +spec: + template: + vmExtensions: + - name: CustomScript + publisher: Microsoft.Azure.Extensions + version: '2.1' + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "This script is a no-op used for extension testing purposes ..." + touch test_file diff --git a/templates/test/ci/patches/azuremachinetemplate-vmextension.yaml b/templates/test/ci/patches/azuremachinetemplate-vmextension.yaml new file mode 100644 index 00000000000..fd8fd45e5c9 --- /dev/null +++ b/templates/test/ci/patches/azuremachinetemplate-vmextension.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + vmExtensions: + - name: CustomScript + publisher: Microsoft.Azure.Extensions + version: '2.1' + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "This script is a no-op used for extension testing purposes ..." + touch test_file diff --git a/templates/test/ci/prow-machine-pool/kustomization.yaml b/templates/test/ci/prow-machine-pool/kustomization.yaml index 0663d7e8a39..f669195d9fe 100644 --- a/templates/test/ci/prow-machine-pool/kustomization.yaml +++ b/templates/test/ci/prow-machine-pool/kustomization.yaml @@ -5,6 +5,7 @@ resources: - ../../../flavors/machinepool-windows - ../prow/cni-resource-set.yaml patchesStrategicMerge: + - ../patches/azuremachinepool-vmextension.yaml - ../patches/tags.yaml - ../patches/cluster-cni.yaml - ../patches/controller-manager.yaml diff --git a/templates/test/ci/prow/kustomization.yaml b/templates/test/ci/prow/kustomization.yaml index 4cd9a663d8e..6af851cb58a 100644 --- a/templates/test/ci/prow/kustomization.yaml +++ b/templates/test/ci/prow/kustomization.yaml @@ -18,6 +18,7 @@ patchesStrategicMerge: - ../patches/machine-deployment-worker-counts.yaml - ../../../azure-cluster-identity/azurecluster-identity-ref.yaml - ../../../flavors/base-windows-containerd/kubeadm-control-plane.yaml + - ../patches/azuremachinetemplate-vmextension.yaml - ../patches/windows-containerd-patch.yaml - ../patches/windows-enable-containerd-logger.yaml - ../patches/windows-csi-proxy-enabled.yaml diff --git a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml index 94c1f43d04c..95722511e25 100644 --- a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml +++ b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml @@ -269,6 +269,15 @@ spec: storageAccountType: Premium_LRS osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmExtensions: + - name: CustomScript + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "This script is a no-op used for extension testing purposes ..." + touch test_file + publisher: Microsoft.Azure.Extensions + version: "2.1" vmSize: ${AZURE_NODE_MACHINE_TYPE} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml index eff06bb24a0..14fc8ee451d 100644 --- a/templates/test/dev/cluster-template-custom-builds.yaml +++ b/templates/test/dev/cluster-template-custom-builds.yaml @@ -267,6 +267,15 @@ spec: diskSizeGB: 128 osType: Linux sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmExtensions: + - name: CustomScript + protectedSettings: + commandToExecute: | + #!/bin/sh + echo "This script is a no-op used for extension testing purposes ..." + touch test_file + publisher: Microsoft.Azure.Extensions + version: "2.1" vmSize: ${AZURE_NODE_MACHINE_TYPE} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index 293baa5cf38..ddec8e0a156 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -234,6 +234,16 @@ var _ = Describe("Workload cluster creation", func() { }, }, result) + By("Verifying expected VM extensions are present on the node", func() { + AzureVMExtensionsSpec(ctx, func() AzureVMExtensionsSpecInput { + return AzureVMExtensionsSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + } + }) + }) + By("Validating failure domains", func() { AzureFailureDomainsSpec(ctx, func() AzureFailureDomainsSpecInput { return AzureFailureDomainsSpecInput{ @@ -308,6 +318,16 @@ var _ = Describe("Workload cluster creation", func() { }, }, result) + By("Verifying expected VM extensions are present on the node", func() { + AzureVMExtensionsSpec(ctx, func() AzureVMExtensionsSpecInput { + return AzureVMExtensionsSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + } + }) + }) + By("Creating an accessible ipv6 load balancer", func() { AzureLBSpec(ctx, func() AzureLBSpecInput { return AzureLBSpecInput{ @@ -357,6 +377,16 @@ var _ = Describe("Workload cluster creation", func() { }, }, result) + By("Verifying expected VM extensions are present on the node", func() { + AzureVMExtensionsSpec(ctx, func() AzureVMExtensionsSpecInput { + return AzureVMExtensionsSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + } + }) + }) + By("Running a security scanner", func() { KubescapeSpec(ctx, func() KubescapeSpecInput { return KubescapeSpecInput{ @@ -445,6 +475,16 @@ var _ = Describe("Workload cluster creation", func() { Args: []string{"--server-side"}, }, result) + By("Verifying expected VM extensions are present on the node", func() { + AzureVMExtensionsSpec(ctx, func() AzureVMExtensionsSpecInput { + return AzureVMExtensionsSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + } + }) + }) + By("Running a GPU-based calculation", func() { AzureGPUSpec(ctx, func() AzureGPUSpecInput { return AzureGPUSpecInput{ @@ -489,6 +529,16 @@ var _ = Describe("Workload cluster creation", func() { }, }, result) + By("Verifying expected VM extensions are present on the node", func() { + AzureVMExtensionsSpec(ctx, func() AzureVMExtensionsSpecInput { + return AzureVMExtensionsSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + } + }) + }) + By("Creating an accessible load balancer", func() { AzureLBSpec(ctx, func() AzureLBSpecInput { return AzureLBSpecInput{ @@ -575,6 +625,16 @@ var _ = Describe("Workload cluster creation", func() { }, }, result) + By("Verifying expected VM extensions are present on the node", func() { + AzureVMExtensionsSpec(ctx, func() AzureVMExtensionsSpecInput { + return AzureVMExtensionsSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + } + }) + }) + // dual-stack external IP for dual-stack clusters is not yet supported // first ip family in ipFamilies is used for the primary clusterIP and cloud-provider // determines the elb/ilb ip family based on the primary clusterIP @@ -638,6 +698,16 @@ var _ = Describe("Workload cluster creation", func() { }, }, result) + By("Verifying expected VM extensions are present on the node", func() { + AzureVMExtensionsSpec(ctx, func() AzureVMExtensionsSpecInput { + return AzureVMExtensionsSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + } + }) + }) + By("PASSED!") }) }) diff --git a/test/e2e/azure_vmextensions.go b/test/e2e/azure_vmextensions.go new file mode 100644 index 00000000000..df45dd0dce5 --- /dev/null +++ b/test/e2e/azure_vmextensions.go @@ -0,0 +1,156 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute" + "github.com/Azure/go-autorest/autorest/azure/auth" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// AzureVMExtensionsSpecInput is the input for AzureVMExtensionsSpec. +type AzureVMExtensionsSpecInput struct { + BootstrapClusterProxy framework.ClusterProxy + Namespace *corev1.Namespace + ClusterName string +} + +// AzureVMExtensionsSpec implements a test that verifies VM extensions are created and deleted. +func AzureVMExtensionsSpec(ctx context.Context, inputGetter func() AzureVMExtensionsSpecInput) { + var ( + specName = "azure-vmextensions" + input AzureVMExtensionsSpecInput + ) + + Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName) + + input = inputGetter() + Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName) + Expect(input.Namespace).ToNot(BeNil(), "Invalid argument. input.Namespace can't be nil when calling %s spec", specName) + Expect(input.ClusterName).ToNot(BeEmpty(), "Invalid argument. input.ClusterName can't be empty when calling %s spec", specName) + + By("creating a Kubernetes client to the workload cluster") + workloadClusterProxy := input.BootstrapClusterProxy.GetWorkloadCluster(ctx, input.Namespace.Name, input.ClusterName) + Expect(workloadClusterProxy).NotTo(BeNil()) + mgmtClient := bootstrapClusterProxy.GetClient() + Expect(mgmtClient).NotTo(BeNil()) + + By("Retrieving all machines from the machine template spec") + machineList := &infrav1.AzureMachineList{} + // list all of the requested objects within the cluster namespace with the cluster name label + Logf("Listing machines in namespace %s with label %s=%s", input.Namespace.Name, clusterv1.ClusterLabelName, workloadClusterProxy.GetName()) + err := mgmtClient.List(ctx, machineList, client.InNamespace(input.Namespace.Name), client.MatchingLabels{clusterv1.ClusterLabelName: workloadClusterProxy.GetName()}) + Expect(err).NotTo(HaveOccurred()) + + // get subscription id + settings, err := auth.GetSettingsFromEnvironment() + Expect(err).NotTo(HaveOccurred()) + subscriptionID := settings.GetSubscriptionID() + auth, err := settings.GetAuthorizer() + Expect(err).NotTo(HaveOccurred()) + + if len(machineList.Items) > 0 { + By("Creating a mapping of machine IDs to array of expected VM extensions") + expectedVMExtensionMap := make(map[string][]string) + for _, machine := range machineList.Items { + for _, extension := range machine.Spec.VMExtensions { + expectedVMExtensionMap[*machine.Spec.ProviderID] = append(expectedVMExtensionMap[*machine.Spec.ProviderID], extension.Name) + } + } + + By("Creating a VM and VM extension client") + // create a VM client + vmClient := compute.NewVirtualMachinesClient(subscriptionID) + vmClient.Authorizer = auth + + // create a VM extension client + vmExtensionsClient := compute.NewVirtualMachineExtensionsClient(subscriptionID) + vmExtensionsClient.Authorizer = auth + + vmListResults, err := vmClient.List(ctx, input.ClusterName, "") + Expect(err).NotTo(HaveOccurred()) + + By("Verifying specified VM extensions are created on Azure") + for _, machine := range vmListResults.Values() { + vmExtensionListResult, err := vmExtensionsClient.List(ctx, input.ClusterName, *machine.Name, "") + Expect(err).NotTo(HaveOccurred()) + vmExtensionList := *vmExtensionListResult.Value + var vmExtensionNames []string + for _, vmExtension := range vmExtensionList { + vmExtensionNames = append(vmExtensionNames, *vmExtension.Name) + } + osName := string(machine.VirtualMachineProperties.StorageProfile.OsDisk.OsType) + Expect(vmExtensionNames).To(ContainElements("CAPZ." + osName + ".Bootstrapping")) + Expect(vmExtensionNames).To(ContainElements(expectedVMExtensionMap[*machine.ID])) + } + } + + By("Retrieving all machine pools from the machine template spec") + machinePoolList := &infrav1exp.AzureMachinePoolList{} + // list all of the requested objects within the cluster namespace with the cluster name label + Logf("Listing machine pools in namespace %s with label %s=%s", input.Namespace.Name, clusterv1.ClusterLabelName, workloadClusterProxy.GetName()) + err = mgmtClient.List(ctx, machinePoolList, client.InNamespace(input.Namespace.Name), client.MatchingLabels{clusterv1.ClusterLabelName: workloadClusterProxy.GetName()}) + Expect(err).NotTo(HaveOccurred()) + + if len(machinePoolList.Items) > 0 { + By("Creating a mapping of machine pool IDs to array of expected VMSS extensions") + expectedVMSSExtensionMap := make(map[string][]string) + for _, machinePool := range machinePoolList.Items { + for _, extension := range machinePool.Spec.Template.VMExtensions { + expectedVMSSExtensionMap[machinePool.Spec.ProviderID] = append(expectedVMSSExtensionMap[machinePool.Spec.ProviderID], extension.Name) + } + } + + By("Creating a VMSS and VMSS extension client") + // create a VMSS client + vmssClient := compute.NewVirtualMachineScaleSetsClient(subscriptionID) + vmssClient.Authorizer = auth + + // create a VMSS extension client + vmssExtensionsClient := compute.NewVirtualMachineScaleSetExtensionsClient(subscriptionID) + vmssExtensionsClient.Authorizer = auth + + vmssListResults, err := vmssClient.List(ctx, input.ClusterName) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying VMSS extensions are created on Azure") + for _, machinePool := range vmssListResults.Values() { + vmssExtensionListResult, err := vmssExtensionsClient.List(ctx, input.ClusterName, *machinePool.Name) + Expect(err).NotTo(HaveOccurred()) + vmssExtensionList := vmssExtensionListResult.Values() + var vmssExtensionNames []string + for _, vmssExtension := range vmssExtensionList { + vmssExtensionNames = append(vmssExtensionNames, *vmssExtension.Name) + } + osName := string(machinePool.VirtualMachineScaleSetProperties.VirtualMachineProfile.StorageProfile.OsDisk.OsType) + Expect(vmssExtensionNames).To(ContainElements("CAPZ." + osName + ".Bootstrapping")) + Expect(vmssExtensionNames).To(ContainElements(expectedVMSSExtensionMap[*machinePool.ID])) + } + } +}