From 236428feb0701c6241a5b38d8541eba02eceb6b5 Mon Sep 17 00:00:00 2001 From: Jont828 Date: Wed, 14 Jun 2023 20:06:44 -0400 Subject: [PATCH] Make scaleset service async --- azure/converters/vmss.go | 4 +- azure/converters/vmss_test.go | 6 +- azure/scope/machinepool.go | 68 +- .../roleassignments/roleassignments.go | 18 +- .../roleassignments/roleassignments_test.go | 13 +- azure/services/scalesets/client.go | 238 ++-- .../scalesets/mock_scalesets/client_mock.go | 120 +- .../mock_scalesets/scalesets_mock.go | 96 +- azure/services/scalesets/scalesets.go | 764 ++--------- azure/services/scalesets/scalesets_test.go | 1130 ++++------------- azure/services/scalesets/spec.go | 523 ++++++++ azure/services/scalesets/spec_test.go | 707 +++++++++++ .../scalesets/vmssextension_spec_test.go | 2 +- .../azuremachinepool_controller.go | 6 +- 14 files changed, 1750 insertions(+), 1945 deletions(-) create mode 100644 azure/services/scalesets/spec.go create mode 100644 azure/services/scalesets/spec_test.go diff --git a/azure/converters/vmss.go b/azure/converters/vmss.go index 2d25eb5b298..3e8dd69a07d 100644 --- a/azure/converters/vmss.go +++ b/azure/converters/vmss.go @@ -34,8 +34,8 @@ const ( ) // SDKToVMSS converts an Azure SDK VirtualMachineScaleSet to the AzureMachinePool type. -func SDKToVMSS(sdkvmss compute.VirtualMachineScaleSet, sdkinstances []compute.VirtualMachineScaleSetVM) *azure.VMSS { - vmss := &azure.VMSS{ +func SDKToVMSS(sdkvmss compute.VirtualMachineScaleSet, sdkinstances []compute.VirtualMachineScaleSetVM) azure.VMSS { + vmss := azure.VMSS{ ID: ptr.Deref(sdkvmss.ID, ""), Name: ptr.Deref(sdkvmss.Name, ""), State: infrav1.ProvisioningState(ptr.Deref(sdkvmss.ProvisioningState, "")), diff --git a/azure/converters/vmss_test.go b/azure/converters/vmss_test.go index 25992ba7031..69a0a9c7312 100644 --- a/azure/converters/vmss_test.go +++ b/azure/converters/vmss_test.go @@ -32,7 +32,7 @@ func Test_SDKToVMSS(t *testing.T) { cases := []struct { Name string SubjectFactory func(*gomega.GomegaWithT) (compute.VirtualMachineScaleSet, []compute.VirtualMachineScaleSetVM) - Expect func(*gomega.GomegaWithT, *azure.VMSS) + Expect func(*gomega.GomegaWithT, azure.VMSS) }{ { Name: "ShouldPopulateWithData", @@ -84,7 +84,7 @@ func Test_SDKToVMSS(t *testing.T) { }, } }, - Expect: func(g *gomega.GomegaWithT, actual *azure.VMSS) { + Expect: func(g *gomega.GomegaWithT, actual azure.VMSS) { expected := azure.VMSS{ ID: "vmssID", Name: "vmssName", @@ -107,7 +107,7 @@ func Test_SDKToVMSS(t *testing.T) { State: "Succeeded", } } - g.Expect(actual).To(gomega.Equal(&expected)) + g.Expect(actual).To(gomega.Equal(expected)) }, }, } diff --git a/azure/scope/machinepool.go b/azure/scope/machinepool.go index f91a3dd384a..3fb599d0f39 100644 --- a/azure/scope/machinepool.go +++ b/azure/scope/machinepool.go @@ -62,7 +62,7 @@ type ( MachinePool *expv1.MachinePool AzureMachinePool *infrav1exp.AzureMachinePool ClusterScope azure.ClusterScoper - Caches *MachinePoolCache + Cache *MachinePoolCache } // MachinePoolScope defines a scope defined around a machine pool and its cluster. @@ -77,16 +77,20 @@ type ( cache *MachinePoolCache } - // MachinePoolCache stores common machine pool information so we don't have to hit the API multiple times within the same reconcile loop. - MachinePoolCache struct { - VMSKU resourceskus.SKU - } - // NodeStatus represents the status of a Kubernetes node. NodeStatus struct { Ready bool Version string } + + // MachinePoolCache stores common machine pool information so we don't have to hit the API multiple times within the same reconcile loop. + MachinePoolCache struct { + BootstrapData string + HasBootstrapDataChanges bool + VMImage *infrav1.Image + VMSKU resourceskus.SKU + MaxSurge int + } ) // NewMachinePoolScope creates a new MachinePoolScope from the supplied parameters. @@ -133,6 +137,27 @@ func (m *MachinePoolScope) InitMachinePoolCache(ctx context.Context) error { var err error m.cache = &MachinePoolCache{} + m.cache.BootstrapData, err = m.GetBootstrapData(ctx) + if err != nil { + return err + } + + m.cache.HasBootstrapDataChanges, err = m.HasBootstrapDataChanges(ctx) + if err != nil { + return err + } + + m.cache.VMImage, err = m.GetVMImage(ctx) + if err != nil { + return err + } + m.SaveVMImageToStatus(m.cache.VMImage) + + m.cache.MaxSurge, err = m.MaxSurge() + if err != nil { + return err + } + skuCache, err := resourceskus.GetCache(m, m.Location()) if err != nil { return err @@ -148,9 +173,19 @@ func (m *MachinePoolScope) InitMachinePoolCache(ctx context.Context) error { } // ScaleSetSpec returns the scale set spec. -func (m *MachinePoolScope) ScaleSetSpec() azure.ScaleSetSpec { - return azure.ScaleSetSpec{ +func (m *MachinePoolScope) ScaleSetSpec(ctx context.Context) azure.ResourceSpecGetter { + ctx, log, done := tele.StartSpanWithLogger(ctx, "scope.MachinePoolScope.ScaleSetSpec") + defer done() + + shouldPatchCustomData := false + if m.HasReplicasExternallyManaged(ctx) { + shouldPatchCustomData = m.cache.HasBootstrapDataChanges + log.V(4).Info("has bootstrap data changed?", "shouldPatchCustomData", "shouldPatchCustomData") + } + + return &scalesets.ScaleSetSpec{ Name: m.Name(), + ResourceGroup: m.ResourceGroup(), Size: m.AzureMachinePool.Spec.Template.VMSize, Capacity: int64(ptr.Deref[int32](m.MachinePool.Spec.Replicas, 0)), SSHKeyData: m.AzureMachinePool.Spec.Template.SSHPublicKey, @@ -172,6 +207,16 @@ func (m *MachinePoolScope) ScaleSetSpec() azure.ScaleSetSpec { NetworkInterfaces: m.AzureMachinePool.Spec.Template.NetworkInterfaces, IPv6Enabled: m.IsIPv6Enabled(), OrchestrationMode: m.AzureMachinePool.Spec.OrchestrationMode, + Location: m.AzureMachinePool.Spec.Location, + SubscriptionID: m.SubscriptionID(), + VMSSExtensionSpecs: m.VMSSExtensionSpecs(), + HasReplicasExternallyManaged: m.HasReplicasExternallyManaged(ctx), + ClusterName: m.ClusterName(), + AdditionalTags: m.AzureMachinePool.Spec.AdditionalTags, + SKU: m.cache.VMSKU, + VMImage: m.cache.VMImage, + BootstrapData: m.cache.BootstrapData, + ShouldPatchCustomData: shouldPatchCustomData, } } @@ -614,11 +659,8 @@ func (m *MachinePoolScope) GetBootstrapData(ctx context.Context) (string, error) } // calculateBootstrapDataHash calculates the sha256 hash of the bootstrap data. -func (m *MachinePoolScope) calculateBootstrapDataHash(ctx context.Context) (string, error) { - bootstrapData, err := m.GetBootstrapData(ctx) - if err != nil { - return "", err - } +func (m *MachinePoolScope) calculateBootstrapDataHash(_ context.Context) (string, error) { + bootstrapData := m.cache.BootstrapData h := sha256.New() n, err := io.WriteString(h, bootstrapData) if err != nil || n == 0 { diff --git a/azure/services/roleassignments/roleassignments.go b/azure/services/roleassignments/roleassignments.go index 99dd72f1d3e..ba8d85ee301 100644 --- a/azure/services/roleassignments/roleassignments.go +++ b/azure/services/roleassignments/roleassignments.go @@ -47,7 +47,7 @@ type Service struct { Scope RoleAssignmentScope virtualMachinesGetter async.Getter async.Reconciler - virtualMachineScaleSetClient scalesets.Client + virtualMachineScaleSetGetter async.Getter } // New creates a new service. @@ -56,7 +56,7 @@ func New(scope RoleAssignmentScope) *Service { return &Service{ Scope: scope, virtualMachinesGetter: virtualmachines.NewClient(scope), - virtualMachineScaleSetClient: scalesets.NewClient(scope), + virtualMachineScaleSetGetter: scalesets.NewClient(scope), Reconciler: async.New(scope, client, client), } } @@ -141,10 +141,20 @@ func (s *Service) getVMSSPrincipalID(ctx context.Context) (*string, error) { ctx, log, done := tele.StartSpanWithLogger(ctx, "roleassignments.Service.getVMPrincipalID") defer done() log.V(2).Info("fetching principal ID for VMSS") - resultVMSS, err := s.virtualMachineScaleSetClient.Get(ctx, s.Scope.ResourceGroup(), s.Scope.Name()) + spec := &scalesets.ScaleSetSpec{ + Name: s.Scope.Name(), + ResourceGroup: s.Scope.ResourceGroup(), + } + + resultVMSSIface, err := s.virtualMachineScaleSetGetter.Get(ctx, spec) if err != nil { - return nil, errors.Wrap(err, "failed to get principal ID for VMSS") + return nil, errors.Wrap(err, "failed to get VMSS") } + resultVMSS, ok := resultVMSSIface.(compute.VirtualMachineScaleSet) + if !ok { + return nil, errors.Errorf("%T is not a compute.VirtualMachineScaleSet", resultVMSSIface) + } + return resultVMSS.Identity.PrincipalID, nil } diff --git a/azure/services/roleassignments/roleassignments_test.go b/azure/services/roleassignments/roleassignments_test.go index d1119ef6a30..f7b55d58862 100644 --- a/azure/services/roleassignments/roleassignments_test.go +++ b/azure/services/roleassignments/roleassignments_test.go @@ -30,6 +30,7 @@ import ( "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/services/async/mock_async" "sigs.k8s.io/cluster-api-provider-azure/azure/services/roleassignments/mock_roleassignments" + "sigs.k8s.io/cluster-api-provider-azure/azure/services/scalesets" "sigs.k8s.io/cluster-api-provider-azure/azure/services/scalesets/mock_scalesets" "sigs.k8s.io/cluster-api-provider-azure/azure/services/virtualmachines" gomockinternal "sigs.k8s.io/cluster-api-provider-azure/internal/test/matchers/gomock" @@ -55,6 +56,10 @@ var ( emptyRoleAssignmentSpec = RoleAssignmentSpec{} fakeRoleAssignmentSpecs = []azure.ResourceSpecGetter{&fakeRoleAssignment1, &fakeRoleAssignment2, &emptyRoleAssignmentSpec} + fakeVMSSSpec = scalesets.ScaleSetSpec{ + Name: "test-vmss", + ResourceGroup: "my-rg", + } ) func TestReconcileRoleAssignmentsVM(t *testing.T) { @@ -169,7 +174,7 @@ func TestReconcileRoleAssignmentsVMSS(t *testing.T) { s.RoleAssignmentResourceType().Return(azure.VirtualMachineScaleSet) s.ResourceGroup().Return("my-rg") s.Name().Return("test-vmss") - mvmss.Get(gomockinternal.AContext(), "my-rg", "test-vmss").Return(compute.VirtualMachineScaleSet{ + mvmss.Get(gomockinternal.AContext(), &fakeVMSSSpec).Return(compute.VirtualMachineScaleSet{ Identity: &compute.VirtualMachineScaleSetIdentity{ PrincipalID: &fakePrincipalID, }, @@ -187,7 +192,7 @@ func TestReconcileRoleAssignmentsVMSS(t *testing.T) { s.ResourceGroup().Return("my-rg") s.Name().Return("test-vmss") s.HasSystemAssignedIdentity().Return(true) - mvmss.Get(gomockinternal.AContext(), "my-rg", "test-vmss").Return(compute.VirtualMachineScaleSet{}, + mvmss.Get(gomockinternal.AContext(), &fakeVMSSSpec).Return(compute.VirtualMachineScaleSet{}, autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: http.StatusInternalServerError}, "Internal Server Error")) }, }, @@ -202,7 +207,7 @@ func TestReconcileRoleAssignmentsVMSS(t *testing.T) { s.RoleAssignmentResourceType().Return(azure.VirtualMachineScaleSet) s.ResourceGroup().Return("my-rg") s.Name().Return("test-vmss") - mvmss.Get(gomockinternal.AContext(), "my-rg", "test-vmss").Return(compute.VirtualMachineScaleSet{ + mvmss.Get(gomockinternal.AContext(), &fakeVMSSSpec).Return(compute.VirtualMachineScaleSet{ Identity: &compute.VirtualMachineScaleSetIdentity{ PrincipalID: &fakePrincipalID, }, @@ -229,7 +234,7 @@ func TestReconcileRoleAssignmentsVMSS(t *testing.T) { s := &Service{ Scope: scopeMock, Reconciler: asyncMock, - virtualMachineScaleSetClient: vmMock, + virtualMachineScaleSetGetter: vmMock, } err := s.Reconcile(context.TODO()) diff --git a/azure/services/scalesets/client.go b/azure/services/scalesets/client.go index f662ab0f380..4d95a2a04ae 100644 --- a/azure/services/scalesets/client.go +++ b/azure/services/scalesets/client.go @@ -18,10 +18,8 @@ package scalesets import ( "context" - "encoding/base64" "encoding/json" "fmt" - "time" "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute" "github.com/Azure/go-autorest/autorest" @@ -30,44 +28,28 @@ import ( "k8s.io/utils/ptr" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" - "sigs.k8s.io/cluster-api-provider-azure/azure/converters" "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" "sigs.k8s.io/cluster-api-provider-azure/util/tele" ) // Client wraps go-sdk. type Client interface { + Get(context.Context, azure.ResourceSpecGetter) (interface{}, error) List(context.Context, string) ([]compute.VirtualMachineScaleSet, error) ListInstances(context.Context, string, string) ([]compute.VirtualMachineScaleSetVM, error) - Get(context.Context, string, string) (compute.VirtualMachineScaleSet, error) - CreateOrUpdateAsync(context.Context, string, string, compute.VirtualMachineScaleSet) (*infrav1.Future, error) - UpdateAsync(context.Context, string, string, compute.VirtualMachineScaleSetUpdate) (*infrav1.Future, error) - GetResultIfDone(ctx context.Context, future *infrav1.Future) (compute.VirtualMachineScaleSet, error) UpdateInstances(context.Context, string, string, []string) error - DeleteAsync(context.Context, string, string) (*infrav1.Future, error) -} - -type ( - // AzureClient contains the Azure go-sdk Client. - AzureClient struct { - scalesetvms compute.VirtualMachineScaleSetVMsClient - scalesets compute.VirtualMachineScaleSetsClient - } - genericScaleSetFuture interface { - DoneWithContext(ctx context.Context, sender autorest.Sender) (done bool, err error) - Result(client compute.VirtualMachineScaleSetsClient) (vmss compute.VirtualMachineScaleSet, err error) - } - - genericScaleSetFutureImpl struct { - azureautorest.FutureAPI - result func(client compute.VirtualMachineScaleSetsClient) (vmss compute.VirtualMachineScaleSet, err error) - } + CreateOrUpdateAsync(ctx context.Context, spec azure.ResourceSpecGetter, parameters interface{}) (result interface{}, future azureautorest.FutureAPI, err error) + DeleteAsync(ctx context.Context, spec azure.ResourceSpecGetter) (future azureautorest.FutureAPI, err error) + IsDone(ctx context.Context, future azureautorest.FutureAPI) (isDone bool, err error) + Result(ctx context.Context, future azureautorest.FutureAPI, futureType string) (result interface{}, err error) +} - deleteResultAdapter struct { - compute.VirtualMachineScaleSetsDeleteFuture - } -) +// AzureClient contains the Azure go-sdk Client. +type AzureClient struct { + scalesetvms compute.VirtualMachineScaleSetVMsClient + scalesets compute.VirtualMachineScaleSetsClient +} var _ Client = &AzureClient{} @@ -94,11 +76,11 @@ func newVirtualMachineScaleSetsClient(subscriptionID string, baseURI string, aut } // ListInstances retrieves information about the model views of a virtual machine scale set. -func (ac *AzureClient) ListInstances(ctx context.Context, resourceGroupName, vmssName string) ([]compute.VirtualMachineScaleSetVM, error) { +func (ac *AzureClient) ListInstances(ctx context.Context, resourceGroupName string, resourceName string) ([]compute.VirtualMachineScaleSetVM, error) { ctx, _, done := tele.StartSpanWithLogger(ctx, "scalesets.AzureClient.ListInstances") defer done() - itr, err := ac.scalesetvms.ListComplete(ctx, resourceGroupName, vmssName, "", "", "") + itr, err := ac.scalesetvms.ListComplete(ctx, resourceGroupName, resourceName, "", "", "") if err != nil { return nil, err } @@ -136,131 +118,43 @@ func (ac *AzureClient) List(ctx context.Context, resourceGroupName string) ([]co } // Get retrieves information about the model view of a virtual machine scale set. -func (ac *AzureClient) Get(ctx context.Context, resourceGroupName, vmssName string) (compute.VirtualMachineScaleSet, error) { +func (ac *AzureClient) Get(ctx context.Context, spec azure.ResourceSpecGetter) (interface{}, error) { ctx, _, done := tele.StartSpanWithLogger(ctx, "scalesets.AzureClient.Get") defer done() - return ac.scalesets.Get(ctx, resourceGroupName, vmssName, "") + return ac.scalesets.Get(ctx, spec.ResourceGroupName(), spec.ResourceName(), "") } -// CreateOrUpdateAsync the operation to create or update a virtual machine scale set without waiting for the operation -// to complete. -func (ac *AzureClient) CreateOrUpdateAsync(ctx context.Context, resourceGroupName, vmssName string, vmss compute.VirtualMachineScaleSet) (*infrav1.Future, error) { +// CreateOrUpdateAsync creates or updates a virtual machine scale set asynchronously. +// It sends a PUT request to Azure and if accepted without error, the func will return a Future which can be used to track the ongoing +// progress of the operation. +func (ac *AzureClient) CreateOrUpdateAsync(ctx context.Context, spec azure.ResourceSpecGetter, parameters interface{}) (result interface{}, future azureautorest.FutureAPI, err error) { ctx, _, done := tele.StartSpanWithLogger(ctx, "scalesets.AzureClient.CreateOrUpdateAsync") defer done() - future, err := ac.scalesets.CreateOrUpdate(ctx, resourceGroupName, vmssName, vmss) - if err != nil { - return nil, err + scaleset, ok := parameters.(compute.VirtualMachineScaleSet) + if !ok { + return nil, nil, errors.Errorf("%T is not a compute.VirtualMachineScaleSet", parameters) } - ctx, cancel := context.WithTimeout(ctx, reconciler.DefaultAzureCallTimeout) - defer cancel() - - err = future.WaitForCompletionRef(ctx, ac.scalesets.Client) + createFuture, err := ac.scalesets.CreateOrUpdate(ctx, spec.ResourceGroupName(), spec.ResourceName(), scaleset) if err != nil { - // if an error occurs, return the future. - // this means the long-running operation didn't finish in the specified timeout. - return converters.SDKToFuture(&future, infrav1.PutFuture, serviceName, vmssName, resourceGroupName) - } - - // todo: this returns the result VMSS, we should use it - _, err = future.Result(ac.scalesets) - - // if the operation completed, return a nil future. - return nil, err -} - -// UpdateAsync update a VM scale set without waiting for the result of the operation. UpdateAsync sends a PATCH -// request to Azure and if accepted without error, the func will return a Future which can be used to track the ongoing -// progress of the operation. -// -// Parameters: -// -// resourceGroupName - the name of the resource group. -// vmssName - the name of the VM scale set to create or update. parameters - the scale set object. -func (ac *AzureClient) UpdateAsync(ctx context.Context, resourceGroupName, vmssName string, parameters compute.VirtualMachineScaleSetUpdate) (*infrav1.Future, error) { - ctx, _, done := tele.StartSpanWithLogger(ctx, "scalesets.AzureClient.UpdateAsync") - defer done() - - future, err := ac.scalesets.Update(ctx, resourceGroupName, vmssName, parameters) - if err != nil { - return nil, errors.Wrapf(err, "failed updating vmss named %q", vmssName) + return nil, nil, err } ctx, cancel := context.WithTimeout(ctx, reconciler.DefaultAzureCallTimeout) defer cancel() - err = future.WaitForCompletionRef(ctx, ac.scalesets.Client) + err = createFuture.WaitForCompletionRef(ctx, ac.scalesets.Client) if err != nil { // if an error occurs, return the future. // this means the long-running operation didn't finish in the specified timeout. - return converters.SDKToFuture(&future, infrav1.PatchFuture, serviceName, vmssName, resourceGroupName) - } - // todo: this returns the result VMSS, we should use it - _, err = future.Result(ac.scalesets) - - // if the operation completed, return a nil future. - return nil, err -} - -// GetResultIfDone fetches the result of a long-running operation future if it is done. -func (ac *AzureClient) GetResultIfDone(ctx context.Context, future *infrav1.Future) (compute.VirtualMachineScaleSet, error) { - var genericFuture genericScaleSetFuture - futureData, err := base64.URLEncoding.DecodeString(future.Data) - if err != nil { - return compute.VirtualMachineScaleSet{}, errors.Wrap(err, "failed to base64 decode future data") - } - - switch future.Type { - case infrav1.PatchFuture: - var future compute.VirtualMachineScaleSetsUpdateFuture - if err := json.Unmarshal(futureData, &future); err != nil { - return compute.VirtualMachineScaleSet{}, errors.Wrap(err, "failed to unmarshal future data") - } - - genericFuture = &genericScaleSetFutureImpl{ - FutureAPI: &future, - result: future.Result, - } - case infrav1.PutFuture: - var future compute.VirtualMachineScaleSetsCreateOrUpdateFuture - if err := json.Unmarshal(futureData, &future); err != nil { - return compute.VirtualMachineScaleSet{}, errors.Wrap(err, "failed to unmarshal future data") - } - - genericFuture = &genericScaleSetFutureImpl{ - FutureAPI: &future, - result: future.Result, - } - case infrav1.DeleteFuture: - var future compute.VirtualMachineScaleSetsDeleteFuture - if err := json.Unmarshal(futureData, &future); err != nil { - return compute.VirtualMachineScaleSet{}, errors.Wrap(err, "failed to unmarshal future data") - } - - genericFuture = &deleteResultAdapter{ - VirtualMachineScaleSetsDeleteFuture: future, - } - default: - return compute.VirtualMachineScaleSet{}, errors.Errorf("unknown future type %q", future.Type) - } - - done, err := genericFuture.DoneWithContext(ctx, ac.scalesets) - if err != nil { - return compute.VirtualMachineScaleSet{}, errors.Wrap(err, "failed checking if the operation was complete") + return nil, &createFuture, err } - if !done { - return compute.VirtualMachineScaleSet{}, azure.WithTransientError(azure.NewOperationNotDoneError(future), 15*time.Second) - } - - vmss, err := genericFuture.Result(ac.scalesets) - if err != nil { - return vmss, errors.Wrap(err, "failed fetching the result of operation for vmss") - } - - return vmss, nil + result, err = createFuture.Result(ac.scalesets) + // if the operation completed, return a nil future + return result, nil, err } // UpdateInstances update instances of a VM scale set. @@ -289,40 +183,80 @@ func (ac *AzureClient) UpdateInstances(ctx context.Context, resourceGroupName, v // // Parameters: // -// resourceGroupName - the name of the resource group. -// vmssName - the name of the VM scale set to create or update. parameters - the scale set object. -func (ac *AzureClient) DeleteAsync(ctx context.Context, resourceGroupName, vmssName string) (*infrav1.Future, error) { +// spec - The ResourceSpecGetter containing used for name and resource group of the virtual machine scale set. +func (ac *AzureClient) DeleteAsync(ctx context.Context, spec azure.ResourceSpecGetter) (future azureautorest.FutureAPI, err error) { ctx, _, done := tele.StartSpanWithLogger(ctx, "scalesets.AzureClient.DeleteAsync") defer done() - future, err := ac.scalesets.Delete(ctx, resourceGroupName, vmssName, ptr.To(false)) + deleteFuture, err := ac.scalesets.Delete(ctx, spec.ResourceGroupName(), spec.ResourceName(), ptr.To(false)) if err != nil { - return nil, errors.Wrapf(err, "failed deleting vmss named %q", vmssName) + return nil, err } ctx, cancel := context.WithTimeout(ctx, reconciler.DefaultAzureCallTimeout) defer cancel() - err = future.WaitForCompletionRef(ctx, ac.scalesets.Client) + err = deleteFuture.WaitForCompletionRef(ctx, ac.scalesets.Client) if err != nil { // if an error occurs, return the future. // this means the long-running operation didn't finish in the specified timeout. - return converters.SDKToFuture(&future, infrav1.DeleteFuture, serviceName, vmssName, resourceGroupName) + return &deleteFuture, err } - _, err = future.Result(ac.scalesets) - + _, err = deleteFuture.Result(ac.scalesets) // if the operation completed, return a nil future. return nil, err } -// Result wraps the delete result so that we can treat it generically. The only thing we care about is if the delete -// was successful. If it wasn't, an error will be returned. -func (da *deleteResultAdapter) Result(client compute.VirtualMachineScaleSetsClient) (compute.VirtualMachineScaleSet, error) { - _, err := da.VirtualMachineScaleSetsDeleteFuture.Result(client) - return compute.VirtualMachineScaleSet{}, err +// IsDone returns true if the long-running operation has completed. +func (ac *AzureClient) IsDone(ctx context.Context, future azureautorest.FutureAPI) (bool, error) { + ctx, _, done := tele.StartSpanWithLogger(ctx, "scalesets.AzureClient.IsDone") + defer done() + + return future.DoneWithContext(ctx, ac.scalesets) } -// Result returns the Result so that we can treat it generically. -func (g *genericScaleSetFutureImpl) Result(client compute.VirtualMachineScaleSetsClient) (compute.VirtualMachineScaleSet, error) { - return g.result(client) +// Result fetches the result of a long-running operation future. +func (ac *AzureClient) Result(ctx context.Context, future azureautorest.FutureAPI, futureType string) (result interface{}, err error) { + _, _, done := tele.StartSpanWithLogger(ctx, "scalesets.AzureClient.Result") + defer done() + + if future == nil { + return nil, errors.Errorf("cannot get result from nil future") + } + + switch futureType { + case infrav1.PatchFuture: + // Marshal and Unmarshal the future to put it into the correct future type so we can access the Result function. + // Unfortunately the FutureAPI can't be casted directly to VirtualMachineScaleSetsUpdateFuture because it is a azureautorest.Future, which doesn't implement the Result function. See PR #1686 for discussion on alternatives. + // It was converted back to a generic azureautorest.Future from the CAPZ infrav1.Future type stored in Status: https://github.com/kubernetes-sigs/cluster-api-provider-azure/blob/main/azure/converters/futures.go#L49. + var updateFuture *compute.VirtualMachineScaleSetsUpdateFuture + jsonData, err := future.MarshalJSON() + if err != nil { + return nil, errors.Wrap(err, "failed to marshal future") + } + if err := json.Unmarshal(jsonData, &updateFuture); err != nil { + return nil, errors.Wrap(err, "failed to unmarshal future data") + } + return updateFuture.Result(ac.scalesets) + + case infrav1.PutFuture: + // Marshal and Unmarshal the future to put it into the correct future type so we can access the Result function. + // Unfortunately the FutureAPI can't be casted directly to VirtualMachineScaleSetsCreateOrUpdateFuture because it is a azureautorest.Future, which doesn't implement the Result function. See PR #1686 for discussion on alternatives. + // It was converted back to a generic azureautorest.Future from the CAPZ infrav1.Future type stored in Status: https://github.com/kubernetes-sigs/cluster-api-provider-azure/blob/main/azure/converters/futures.go#L49. + var createFuture *compute.VirtualMachineScaleSetsCreateOrUpdateFuture + jsonData, err := future.MarshalJSON() + if err != nil { + return nil, errors.Wrap(err, "failed to marshal future") + } + if err := json.Unmarshal(jsonData, &createFuture); err != nil { + return nil, errors.Wrap(err, "failed to unmarshal future data") + } + return createFuture.Result(ac.scalesets) + + case infrav1.DeleteFuture: + // Delete does not return a result compute.VirtualMachineScaleSet + return nil, nil + default: + return nil, errors.Errorf("unknown future type %q", futureType) + } } diff --git a/azure/services/scalesets/mock_scalesets/client_mock.go b/azure/services/scalesets/mock_scalesets/client_mock.go index 60c5304bdc6..258aa53d4f8 100644 --- a/azure/services/scalesets/mock_scalesets/client_mock.go +++ b/azure/services/scalesets/mock_scalesets/client_mock.go @@ -25,9 +25,9 @@ import ( reflect "reflect" compute "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute" - autorest "github.com/Azure/go-autorest/autorest" + azure "github.com/Azure/go-autorest/autorest/azure" gomock "go.uber.org/mock/gomock" - v1beta1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + azure0 "sigs.k8s.io/cluster-api-provider-azure/azure" ) // MockClient is a mock of Client interface. @@ -54,63 +54,64 @@ func (m *MockClient) EXPECT() *MockClientMockRecorder { } // CreateOrUpdateAsync mocks base method. -func (m *MockClient) CreateOrUpdateAsync(arg0 context.Context, arg1, arg2 string, arg3 compute.VirtualMachineScaleSet) (*v1beta1.Future, error) { +func (m *MockClient) CreateOrUpdateAsync(ctx context.Context, spec azure0.ResourceSpecGetter, parameters interface{}) (interface{}, azure.FutureAPI, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateOrUpdateAsync", arg0, arg1, arg2, arg3) - ret0, _ := ret[0].(*v1beta1.Future) - ret1, _ := ret[1].(error) - return ret0, ret1 + ret := m.ctrl.Call(m, "CreateOrUpdateAsync", ctx, spec, parameters) + ret0, _ := ret[0].(interface{}) + ret1, _ := ret[1].(azure.FutureAPI) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 } // CreateOrUpdateAsync indicates an expected call of CreateOrUpdateAsync. -func (mr *MockClientMockRecorder) CreateOrUpdateAsync(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { +func (mr *MockClientMockRecorder) CreateOrUpdateAsync(ctx, spec, parameters interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdateAsync", reflect.TypeOf((*MockClient)(nil).CreateOrUpdateAsync), arg0, arg1, arg2, arg3) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdateAsync", reflect.TypeOf((*MockClient)(nil).CreateOrUpdateAsync), ctx, spec, parameters) } // DeleteAsync mocks base method. -func (m *MockClient) DeleteAsync(arg0 context.Context, arg1, arg2 string) (*v1beta1.Future, error) { +func (m *MockClient) DeleteAsync(ctx context.Context, spec azure0.ResourceSpecGetter) (azure.FutureAPI, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteAsync", arg0, arg1, arg2) - ret0, _ := ret[0].(*v1beta1.Future) + ret := m.ctrl.Call(m, "DeleteAsync", ctx, spec) + ret0, _ := ret[0].(azure.FutureAPI) ret1, _ := ret[1].(error) return ret0, ret1 } // DeleteAsync indicates an expected call of DeleteAsync. -func (mr *MockClientMockRecorder) DeleteAsync(arg0, arg1, arg2 interface{}) *gomock.Call { +func (mr *MockClientMockRecorder) DeleteAsync(ctx, spec interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAsync", reflect.TypeOf((*MockClient)(nil).DeleteAsync), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteAsync", reflect.TypeOf((*MockClient)(nil).DeleteAsync), ctx, spec) } // Get mocks base method. -func (m *MockClient) Get(arg0 context.Context, arg1, arg2 string) (compute.VirtualMachineScaleSet, error) { +func (m *MockClient) Get(arg0 context.Context, arg1 azure0.ResourceSpecGetter) (interface{}, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Get", arg0, arg1, arg2) - ret0, _ := ret[0].(compute.VirtualMachineScaleSet) + ret := m.ctrl.Call(m, "Get", arg0, arg1) + ret0, _ := ret[0].(interface{}) ret1, _ := ret[1].(error) return ret0, ret1 } // Get indicates an expected call of Get. -func (mr *MockClientMockRecorder) Get(arg0, arg1, arg2 interface{}) *gomock.Call { +func (mr *MockClientMockRecorder) Get(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockClient)(nil).Get), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockClient)(nil).Get), arg0, arg1) } -// GetResultIfDone mocks base method. -func (m *MockClient) GetResultIfDone(ctx context.Context, future *v1beta1.Future) (compute.VirtualMachineScaleSet, error) { +// IsDone mocks base method. +func (m *MockClient) IsDone(ctx context.Context, future azure.FutureAPI) (bool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetResultIfDone", ctx, future) - ret0, _ := ret[0].(compute.VirtualMachineScaleSet) + ret := m.ctrl.Call(m, "IsDone", ctx, future) + ret0, _ := ret[0].(bool) ret1, _ := ret[1].(error) return ret0, ret1 } -// GetResultIfDone indicates an expected call of GetResultIfDone. -func (mr *MockClientMockRecorder) GetResultIfDone(ctx, future interface{}) *gomock.Call { +// IsDone indicates an expected call of IsDone. +func (mr *MockClientMockRecorder) IsDone(ctx, future interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetResultIfDone", reflect.TypeOf((*MockClient)(nil).GetResultIfDone), ctx, future) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsDone", reflect.TypeOf((*MockClient)(nil).IsDone), ctx, future) } // List mocks base method. @@ -143,19 +144,19 @@ func (mr *MockClientMockRecorder) ListInstances(arg0, arg1, arg2 interface{}) *g return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListInstances", reflect.TypeOf((*MockClient)(nil).ListInstances), arg0, arg1, arg2) } -// UpdateAsync mocks base method. -func (m *MockClient) UpdateAsync(arg0 context.Context, arg1, arg2 string, arg3 compute.VirtualMachineScaleSetUpdate) (*v1beta1.Future, error) { +// Result mocks base method. +func (m *MockClient) Result(ctx context.Context, future azure.FutureAPI, futureType string) (interface{}, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateAsync", arg0, arg1, arg2, arg3) - ret0, _ := ret[0].(*v1beta1.Future) + ret := m.ctrl.Call(m, "Result", ctx, future, futureType) + ret0, _ := ret[0].(interface{}) ret1, _ := ret[1].(error) return ret0, ret1 } -// UpdateAsync indicates an expected call of UpdateAsync. -func (mr *MockClientMockRecorder) UpdateAsync(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { +// Result indicates an expected call of Result. +func (mr *MockClientMockRecorder) Result(ctx, future, futureType interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateAsync", reflect.TypeOf((*MockClient)(nil).UpdateAsync), arg0, arg1, arg2, arg3) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Result", reflect.TypeOf((*MockClient)(nil).Result), ctx, future, futureType) } // UpdateInstances mocks base method. @@ -171,56 +172,3 @@ func (mr *MockClientMockRecorder) UpdateInstances(arg0, arg1, arg2, arg3 interfa mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateInstances", reflect.TypeOf((*MockClient)(nil).UpdateInstances), arg0, arg1, arg2, arg3) } - -// MockgenericScaleSetFuture is a mock of genericScaleSetFuture interface. -type MockgenericScaleSetFuture struct { - ctrl *gomock.Controller - recorder *MockgenericScaleSetFutureMockRecorder -} - -// MockgenericScaleSetFutureMockRecorder is the mock recorder for MockgenericScaleSetFuture. -type MockgenericScaleSetFutureMockRecorder struct { - mock *MockgenericScaleSetFuture -} - -// NewMockgenericScaleSetFuture creates a new mock instance. -func NewMockgenericScaleSetFuture(ctrl *gomock.Controller) *MockgenericScaleSetFuture { - mock := &MockgenericScaleSetFuture{ctrl: ctrl} - mock.recorder = &MockgenericScaleSetFutureMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockgenericScaleSetFuture) EXPECT() *MockgenericScaleSetFutureMockRecorder { - return m.recorder -} - -// DoneWithContext mocks base method. -func (m *MockgenericScaleSetFuture) DoneWithContext(ctx context.Context, sender autorest.Sender) (bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DoneWithContext", ctx, sender) - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// DoneWithContext indicates an expected call of DoneWithContext. -func (mr *MockgenericScaleSetFutureMockRecorder) DoneWithContext(ctx, sender interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DoneWithContext", reflect.TypeOf((*MockgenericScaleSetFuture)(nil).DoneWithContext), ctx, sender) -} - -// Result mocks base method. -func (m *MockgenericScaleSetFuture) Result(client compute.VirtualMachineScaleSetsClient) (compute.VirtualMachineScaleSet, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Result", client) - ret0, _ := ret[0].(compute.VirtualMachineScaleSet) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Result indicates an expected call of Result. -func (mr *MockgenericScaleSetFutureMockRecorder) Result(client interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Result", reflect.TypeOf((*MockgenericScaleSetFuture)(nil).Result), client) -} diff --git a/azure/services/scalesets/mock_scalesets/scalesets_mock.go b/azure/services/scalesets/mock_scalesets/scalesets_mock.go index 561113bc15f..a83dd4ab03c 100644 --- a/azure/services/scalesets/mock_scalesets/scalesets_mock.go +++ b/azure/services/scalesets/mock_scalesets/scalesets_mock.go @@ -248,21 +248,6 @@ func (mr *MockScaleSetScopeMockRecorder) FailureDomains() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FailureDomains", reflect.TypeOf((*MockScaleSetScope)(nil).FailureDomains)) } -// GetBootstrapData mocks base method. -func (m *MockScaleSetScope) GetBootstrapData(arg0 context.Context) (string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetBootstrapData", arg0) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetBootstrapData indicates an expected call of GetBootstrapData. -func (mr *MockScaleSetScopeMockRecorder) GetBootstrapData(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBootstrapData", reflect.TypeOf((*MockScaleSetScope)(nil).GetBootstrapData), arg0) -} - // GetLongRunningOperationState mocks base method. func (m *MockScaleSetScope) GetLongRunningOperationState(arg0, arg1, arg2 string) *v1beta1.Future { m.ctrl.T.Helper() @@ -277,50 +262,6 @@ func (mr *MockScaleSetScopeMockRecorder) GetLongRunningOperationState(arg0, arg1 return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLongRunningOperationState", reflect.TypeOf((*MockScaleSetScope)(nil).GetLongRunningOperationState), arg0, arg1, arg2) } -// GetVMImage mocks base method. -func (m *MockScaleSetScope) GetVMImage(arg0 context.Context) (*v1beta1.Image, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetVMImage", arg0) - ret0, _ := ret[0].(*v1beta1.Image) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetVMImage indicates an expected call of GetVMImage. -func (mr *MockScaleSetScopeMockRecorder) GetVMImage(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVMImage", reflect.TypeOf((*MockScaleSetScope)(nil).GetVMImage), arg0) -} - -// HasBootstrapDataChanges mocks base method. -func (m *MockScaleSetScope) HasBootstrapDataChanges(arg0 context.Context) (bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "HasBootstrapDataChanges", arg0) - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// HasBootstrapDataChanges indicates an expected call of HasBootstrapDataChanges. -func (mr *MockScaleSetScopeMockRecorder) HasBootstrapDataChanges(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HasBootstrapDataChanges", reflect.TypeOf((*MockScaleSetScope)(nil).HasBootstrapDataChanges), arg0) -} - -// HasReplicasExternallyManaged mocks base method. -func (m *MockScaleSetScope) HasReplicasExternallyManaged(arg0 context.Context) bool { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "HasReplicasExternallyManaged", arg0) - ret0, _ := ret[0].(bool) - return ret0 -} - -// HasReplicasExternallyManaged indicates an expected call of HasReplicasExternallyManaged. -func (mr *MockScaleSetScopeMockRecorder) HasReplicasExternallyManaged(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HasReplicasExternallyManaged", reflect.TypeOf((*MockScaleSetScope)(nil).HasReplicasExternallyManaged), arg0) -} - // HashKey mocks base method. func (m *MockScaleSetScope) HashKey() string { m.ctrl.T.Helper() @@ -349,21 +290,6 @@ func (mr *MockScaleSetScopeMockRecorder) Location() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Location", reflect.TypeOf((*MockScaleSetScope)(nil).Location)) } -// MaxSurge mocks base method. -func (m *MockScaleSetScope) MaxSurge() (int, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "MaxSurge") - ret0, _ := ret[0].(int) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// MaxSurge indicates an expected call of MaxSurge. -func (mr *MockScaleSetScopeMockRecorder) MaxSurge() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MaxSurge", reflect.TypeOf((*MockScaleSetScope)(nil).MaxSurge)) -} - // ReconcileReplicas mocks base method. func (m *MockScaleSetScope) ReconcileReplicas(arg0 context.Context, arg1 *azure.VMSS) error { m.ctrl.T.Helper() @@ -392,30 +318,18 @@ func (mr *MockScaleSetScopeMockRecorder) ResourceGroup() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResourceGroup", reflect.TypeOf((*MockScaleSetScope)(nil).ResourceGroup)) } -// SaveVMImageToStatus mocks base method. -func (m *MockScaleSetScope) SaveVMImageToStatus(arg0 *v1beta1.Image) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "SaveVMImageToStatus", arg0) -} - -// SaveVMImageToStatus indicates an expected call of SaveVMImageToStatus. -func (mr *MockScaleSetScopeMockRecorder) SaveVMImageToStatus(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveVMImageToStatus", reflect.TypeOf((*MockScaleSetScope)(nil).SaveVMImageToStatus), arg0) -} - // ScaleSetSpec mocks base method. -func (m *MockScaleSetScope) ScaleSetSpec() azure.ScaleSetSpec { +func (m *MockScaleSetScope) ScaleSetSpec(arg0 context.Context) azure.ResourceSpecGetter { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ScaleSetSpec") - ret0, _ := ret[0].(azure.ScaleSetSpec) + ret := m.ctrl.Call(m, "ScaleSetSpec", arg0) + ret0, _ := ret[0].(azure.ResourceSpecGetter) return ret0 } // ScaleSetSpec indicates an expected call of ScaleSetSpec. -func (mr *MockScaleSetScopeMockRecorder) ScaleSetSpec() *gomock.Call { +func (mr *MockScaleSetScopeMockRecorder) ScaleSetSpec(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ScaleSetSpec", reflect.TypeOf((*MockScaleSetScope)(nil).ScaleSetSpec)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ScaleSetSpec", reflect.TypeOf((*MockScaleSetScope)(nil).ScaleSetSpec), arg0) } // SetAnnotation mocks base method. diff --git a/azure/services/scalesets/scalesets.go b/azure/services/scalesets/scalesets.go index cc982ff7e9b..d3eb38eeacd 100644 --- a/azure/services/scalesets/scalesets.go +++ b/azure/services/scalesets/scalesets.go @@ -18,21 +18,18 @@ package scalesets import ( "context" - "encoding/base64" "fmt" - "strconv" - "time" "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute" "github.com/pkg/errors" - "k8s.io/utils/ptr" azprovider "sigs.k8s.io/cloud-provider-azure/pkg/provider" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/converters" + "sigs.k8s.io/cluster-api-provider-azure/azure/services/async" "sigs.k8s.io/cluster-api-provider-azure/azure/services/resourceskus" azureutil "sigs.k8s.io/cluster-api-provider-azure/util/azure" - "sigs.k8s.io/cluster-api-provider-azure/util/generators" + "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" "sigs.k8s.io/cluster-api-provider-azure/util/slice" "sigs.k8s.io/cluster-api-provider-azure/util/tele" ) @@ -44,18 +41,12 @@ type ( ScaleSetScope interface { azure.ClusterDescriber azure.AsyncStatusUpdater - GetBootstrapData(context.Context) (string, error) - GetVMImage(context.Context) (*infrav1.Image, error) - SaveVMImageToStatus(*infrav1.Image) - MaxSurge() (int, error) - ScaleSetSpec() azure.ScaleSetSpec + ScaleSetSpec(context.Context) azure.ResourceSpecGetter VMSSExtensionSpecs() []azure.ResourceSpecGetter SetAnnotation(string, string) SetProviderID(string) SetVMSSState(*azure.VMSS) ReconcileReplicas(context.Context, *azure.VMSS) error - HasReplicasExternallyManaged(context.Context) bool - HasBootstrapDataChanges(context.Context) (bool, error) } // Service provides operations on Azure resources. @@ -63,13 +54,16 @@ type ( Scope ScaleSetScope Client resourceSKUCache *resourceskus.Cache + async.Reconciler } ) // New creates a new service. func New(scope ScaleSetScope, skuCache *resourceskus.Cache) *Service { + client := NewClient(scope) return &Service{ - Client: NewClient(scope), + Reconciler: async.New(scope, client, client), + Client: client, Scope: scope, resourceSKUCache: skuCache, } @@ -82,89 +76,60 @@ func (s *Service) Name() string { // Reconcile idempotently gets, creates, and updates a scale set. func (s *Service) Reconcile(ctx context.Context) (retErr error) { - ctx, log, done := tele.StartSpanWithLogger(ctx, "scalesets.Service.Reconcile") + ctx, _, done := tele.StartSpanWithLogger(ctx, "scalesets.Service.Reconcile") defer done() + ctx, cancel := context.WithTimeout(ctx, reconciler.DefaultAzureServiceReconcileTimeout) + defer cancel() + if err := s.validateSpec(ctx); err != nil { // do as much early validation as possible to limit calls to Azure return err } - var err error - - scaleSetSpec := s.Scope.ScaleSetSpec() - - // check if there is an ongoing long running operation - var fetchedVMSS *azure.VMSS - future := s.Scope.GetLongRunningOperationState(s.Scope.ScaleSetSpec().Name, serviceName, infrav1.PutFuture) - if future == nil { - future = s.Scope.GetLongRunningOperationState(s.Scope.ScaleSetSpec().Name, serviceName, infrav1.PatchFuture) + spec := s.Scope.ScaleSetSpec(ctx) + scaleSetSpec, ok := spec.(*ScaleSetSpec) + if !ok { + return errors.Errorf("%T is not of type ScaleSetSpec", spec) } - defer func() { - // save the updated state of the VMSS for the MachinePoolScope to use for updating K8s state - if fetchedVMSS == nil { - fetchedVMSS, err = s.getVirtualMachineScaleSet(ctx, scaleSetSpec.Name) - if err != nil && !azure.ResourceNotFound(err) { - log.Error(err, "failed to get vmss in deferred update") - } - } - - if fetchedVMSS != nil { - // Transform the VMSS resource representation to conform to the cloud-provider-azure representation - providerID, err := azprovider.ConvertResourceGroupNameToLower(azureutil.ProviderIDPrefix + fetchedVMSS.ID) - if err != nil { - log.Error(err, "failed to parse VMSS ID", "ID", fetchedVMSS.ID) - } - s.Scope.SetProviderID(providerID) - s.Scope.SetVMSSState(fetchedVMSS) + _, err := s.Client.Get(ctx, spec) + if err == nil { + // We can only get the existing instances if the VMSS already exists + scaleSetSpec.VMSSInstances, err = s.Client.ListInstances(ctx, spec.ResourceGroupName(), spec.ResourceName()) + if err != nil { + err = errors.Wrapf(err, "failed to get existing VMSS instances") + s.Scope.UpdatePutStatus(infrav1.BootstrapSucceededCondition, serviceName, err) + return err } - }() - - if future == nil { - fetchedVMSS, err = s.getVirtualMachineScaleSet(ctx, scaleSetSpec.Name) - } else { - fetchedVMSS, err = s.getVirtualMachineScaleSetIfDone(ctx, future) + } else if !azure.ResourceNotFound(err) { + return errors.Wrapf(err, "failed to get existing VMSS") } - switch { - case err != nil && !azure.ResourceNotFound(err): - // There was an error and it was not an HTTP 404 not found. This is either a transient error, like long running operation not done, or an Azure service error. - return errors.Wrapf(err, "failed to get VMSS %s", scaleSetSpec.Name) - case err != nil && azure.ResourceNotFound(err): - // HTTP(404) resource was not found, so we need to create it with a PUT - future, err = s.createVMSS(ctx) - if err != nil { - return errors.Wrap(err, "failed to start creating VMSS") + result, err := s.CreateOrUpdateResource(ctx, scaleSetSpec, serviceName) + s.Scope.UpdatePutStatus(infrav1.BootstrapSucceededCondition, serviceName, err) + + if err == nil && result != nil { + vmss, ok := result.(compute.VirtualMachineScaleSet) + if !ok { + return errors.Errorf("%T is not a compute.VirtualMachineScaleSet", result) } - case err == nil: - // HTTP(200) - // VMSS already exists and may have changes; update it with a PATCH - // we do this to avoid overwriting fields in networkProfile modified by cloud-provider - future, err = s.patchVMSSIfNeeded(ctx, fetchedVMSS) - if err != nil { - return errors.Wrap(err, "failed to start updating VMSS") + + fetchedVMSS := converters.SDKToVMSS(vmss, scaleSetSpec.VMSSInstances) + if err := s.Scope.ReconcileReplicas(ctx, &fetchedVMSS); err != nil { + return errors.Wrap(err, "unable to reconcile VMSS replicas") } - } - // Try to get the VMSS to update status if we have created a long running operation. If the VMSS is still in a long - // running operation, getVirtualMachineScaleSetIfDone will return an azure.WithTransientError and requeue. - if future != nil { - fetchedVMSS, err = s.getVirtualMachineScaleSetIfDone(ctx, future) + // Transform the VMSS resource representation to conform to the cloud-provider-azure representation + providerID, err := azprovider.ConvertResourceGroupNameToLower(azureutil.ProviderIDPrefix + fetchedVMSS.ID) if err != nil { - return errors.Wrapf(err, "failed to get VMSS %s after create or update", scaleSetSpec.Name) + return errors.Wrapf(err, "failed to parse VMSS ID %s", fetchedVMSS.ID) } + s.Scope.SetProviderID(providerID) + s.Scope.SetVMSSState(&fetchedVMSS) } - // If we get to here, we have completed any long running VMSS operations (creates / updates) - s.Scope.DeleteLongRunningOperationState(s.Scope.ScaleSetSpec().Name, serviceName, infrav1.PutFuture) - s.Scope.DeleteLongRunningOperationState(s.Scope.ScaleSetSpec().Name, serviceName, infrav1.PatchFuture) - - // This also means that the VMSS extensions were successfully installed - // Note: we want to handle UpdatePutStatus when VMSSExtensions have an error when scalesets become an async service - s.Scope.UpdatePutStatus(infrav1.BootstrapSucceededCondition, serviceName, nil) - - return nil + return err } // Delete deletes a scale set asynchronously. Delete sends a DELETE request to Azure and if accepted without error, @@ -173,13 +138,13 @@ func (s *Service) Delete(ctx context.Context) error { ctx, log, done := tele.StartSpanWithLogger(ctx, "scalesets.Service.Delete") defer done() - var err error + ctx, cancel := context.WithTimeout(ctx, reconciler.DefaultAzureServiceReconcileTimeout) + defer cancel() - vmssSpec := s.Scope.ScaleSetSpec() + scaleSetSpec := s.Scope.ScaleSetSpec(ctx) defer func() { - // save the updated state of the VMSS for the MachinePoolScope to use for updating K8s state - fetchedVMSS, err := s.getVirtualMachineScaleSet(ctx, vmssSpec.Name) + fetchedVMSS, err := s.getVirtualMachineScaleSet(ctx, scaleSetSpec) if err != nil && !azure.ResourceNotFound(err) { log.Error(err, "failed to get vmss in deferred update") } @@ -189,159 +154,26 @@ func (s *Service) Delete(ctx context.Context) error { } }() - // check if there is an ongoing long running operation - future := s.Scope.GetLongRunningOperationState(vmssSpec.Name, serviceName, infrav1.DeleteFuture) - if future != nil { - // if the operation is not complete this will return an error - _, err := s.GetResultIfDone(ctx, future) - if err != nil { - return errors.Wrap(err, "failed to get result from future") - } - - // ScaleSet has been deleted - s.Scope.DeleteLongRunningOperationState(vmssSpec.Name, serviceName, infrav1.DeleteFuture) - // Note: we want to handle UpdateDeleteStatus when VMSSExtensions have an error when scalesets become an async service - s.Scope.UpdateDeleteStatus(infrav1.BootstrapSucceededCondition, serviceName, nil) - - return nil - } - - // no long running delete operation is active, so delete the ScaleSet - log.V(2).Info("deleting VMSS", "scale set", vmssSpec.Name) - future, err = s.Client.DeleteAsync(ctx, s.Scope.ResourceGroup(), vmssSpec.Name) - if err != nil { - if azure.ResourceNotFound(err) { - // already deleted - return nil - } - return errors.Wrapf(err, "failed to delete VMSS %s in resource group %s", vmssSpec.Name, s.Scope.ResourceGroup()) - } - - s.Scope.SetLongRunningOperationState(future) - if future != nil { - // if future exists, check state of the future - if _, err = s.GetResultIfDone(ctx, future); err != nil { - return errors.Wrap(err, "not done with long running operation, or failed to get result") - } - } - - // future is either nil, or the result of the future is complete - s.Scope.DeleteLongRunningOperationState(vmssSpec.Name, serviceName, infrav1.DeleteFuture) - // Note: we want to handle UpdateDeleteStatus when VMSSExtensions have an error when scalesets become an async service - s.Scope.UpdateDeleteStatus(infrav1.BootstrapSucceededCondition, serviceName, nil) - - return nil -} - -func (s *Service) createVMSS(ctx context.Context) (*infrav1.Future, error) { - ctx, log, done := tele.StartSpanWithLogger(ctx, "scalesets.Service.createVMSS") - defer done() - - spec := s.Scope.ScaleSetSpec() - - vmss, err := s.buildVMSSFromSpec(ctx, spec) - if err != nil { - return nil, errors.Wrap(err, "failed building VMSS from spec") - } - - future, err := s.Client.CreateOrUpdateAsync(ctx, s.Scope.ResourceGroup(), spec.Name, vmss) - if err != nil { - return nil, errors.Wrap(err, "cannot create VMSS") - } - - log.V(2).Info("starting to create VMSS", "scale set", spec.Name) - s.Scope.SetLongRunningOperationState(future) - return future, err -} - -func (s *Service) patchVMSSIfNeeded(ctx context.Context, infraVMSS *azure.VMSS) (*infrav1.Future, error) { - ctx, log, done := tele.StartSpanWithLogger(ctx, "scalesets.Service.patchVMSSIfNeeded") - defer done() - - if err := s.Scope.ReconcileReplicas(ctx, infraVMSS); err != nil { - return nil, errors.Wrap(err, "unable to reconcile replicas") - } - - spec := s.Scope.ScaleSetSpec() - - vmss, err := s.buildVMSSFromSpec(ctx, spec) - if err != nil { - return nil, errors.Wrapf(err, "failed to generate scale set update parameters for %s", spec.Name) - } - - patch, err := getVMSSUpdateFromVMSS(vmss) - if err != nil { - return nil, errors.Wrapf(err, "failed to generate vmss patch for %s", spec.Name) - } + err := s.DeleteResource(ctx, scaleSetSpec, serviceName) - maxSurge, err := s.Scope.MaxSurge() - if err != nil { - return nil, errors.Wrap(err, "failed to calculate maxSurge") - } + s.Scope.UpdateDeleteStatus(infrav1.BootstrapSucceededCondition, serviceName, err) - // If the VMSS is managed by an external autoscaler, we should patch the VMSS if customData has changed. - shouldPatchCustomData := false - if s.Scope.HasReplicasExternallyManaged(ctx) { - shouldPatchCustomData, err = s.Scope.HasBootstrapDataChanges(ctx) - if err != nil { - return nil, errors.Wrap(err, "unable to calculate custom data hash") - } - if shouldPatchCustomData { - log.V(4).Info("custom data changed") - } else { - log.V(4).Info("custom data unchanged") - } - } - - hasModelChanges := hasModelModifyingDifferences(infraVMSS, vmss) - isFlex := s.Scope.ScaleSetSpec().OrchestrationMode == infrav1.FlexibleOrchestrationMode - updated := true - if !isFlex { - updated = infraVMSS.HasEnoughLatestModelOrNotMixedModel() - } - if maxSurge > 0 && (hasModelChanges || !updated) && !s.Scope.HasReplicasExternallyManaged(ctx) { - // surge capacity with the intention of lowering during instance reconciliation - surge := spec.Capacity + int64(maxSurge) - log.V(4).Info("surging...", "surge", surge, "hasModelChanges", hasModelChanges, "updated", updated) - patch.Sku.Capacity = ptr.To[int64](surge) - } - - // If the VMSS is managed by an external autoscaler, we should patch the VMSS if customData has changed. - // If there are no model changes and no increase in the replica count, do not update the VMSS. - // Decreases in replica count is handled by deleting AzureMachinePoolMachine instances in the MachinePoolScope - if *patch.Sku.Capacity <= infraVMSS.Capacity && !hasModelChanges && !shouldPatchCustomData { - log.V(4).Info("nothing to update on vmss", "scale set", spec.Name, "newReplicas", *patch.Sku.Capacity, "oldReplicas", infraVMSS.Capacity, "hasModelChanges", hasModelChanges, "shouldPatchCustomData", shouldPatchCustomData) - return nil, nil - } - - log.V(4).Info("patching vmss", "scale set", spec.Name, "patch", patch) - future, err := s.UpdateAsync(ctx, s.Scope.ResourceGroup(), spec.Name, patch) - if err != nil { - if azure.ResourceConflict(err) { - return nil, azure.WithTransientError(err, 30*time.Second) - } - return nil, errors.Wrap(err, "failed updating VMSS") - } - - s.Scope.SetLongRunningOperationState(future) - log.V(2).Info("successfully started to update vmss", "scale set", spec.Name) - return future, err -} - -func hasModelModifyingDifferences(infraVMSS *azure.VMSS, vmss compute.VirtualMachineScaleSet) bool { - other := converters.SDKToVMSS(vmss, []compute.VirtualMachineScaleSetVM{}) - return infraVMSS.HasModelChanges(*other) + return err } func (s *Service) validateSpec(ctx context.Context) error { ctx, _, done := tele.StartSpanWithLogger(ctx, "scalesets.Service.validateSpec") defer done() - spec := s.Scope.ScaleSetSpec() + spec := s.Scope.ScaleSetSpec(ctx) + scaleSetSpec, ok := spec.(*ScaleSetSpec) + if !ok { + return errors.Errorf("%T is not a ScaleSetSpec", spec) + } - sku, err := s.resourceSKUCache.Get(ctx, spec.Size, resourceskus.VirtualMachines) + sku, err := s.resourceSKUCache.Get(ctx, scaleSetSpec.Size, resourceskus.VirtualMachines) if err != nil { - return errors.Wrapf(err, "failed to get SKU %s in compute api", spec.Size) + return errors.Wrapf(err, "failed to get SKU %s in compute api", scaleSetSpec.Size) } // Checking if the requested VM size has at least 2 vCPUS @@ -365,27 +197,26 @@ func (s *Service) validateSpec(ctx context.Context) error { } // enable ephemeral OS - if spec.OSDisk.DiffDiskSettings != nil && !sku.HasCapability(resourceskus.EphemeralOSDisk) { - return azure.WithTerminalError(fmt.Errorf("vm size %s does not support ephemeral os. select a different vm size or disable ephemeral os", spec.Size)) + if scaleSetSpec.OSDisk.DiffDiskSettings != nil && !sku.HasCapability(resourceskus.EphemeralOSDisk) { + return azure.WithTerminalError(fmt.Errorf("vm size %s does not support ephemeral os. select a different vm size or disable ephemeral os", scaleSetSpec.Size)) } - if spec.SecurityProfile != nil && !sku.HasCapability(resourceskus.EncryptionAtHost) { - return azure.WithTerminalError(errors.Errorf("encryption at host is not supported for VM type %s", spec.Size)) + if scaleSetSpec.SecurityProfile != nil && !sku.HasCapability(resourceskus.EncryptionAtHost) { + return azure.WithTerminalError(errors.Errorf("encryption at host is not supported for VM type %s", scaleSetSpec.Size)) } // Fetch location and zone to check for their support of ultra disks. - location := s.Scope.Location() - zones, err := s.resourceSKUCache.GetZones(ctx, location) + zones, err := s.resourceSKUCache.GetZones(ctx, scaleSetSpec.Location) if err != nil { - return azure.WithTerminalError(errors.Wrapf(err, "failed to get the zones for location %s", location)) + return azure.WithTerminalError(errors.Wrapf(err, "failed to get the zones for location %s", scaleSetSpec.Location)) } for _, zone := range zones { - hasLocationCapability := sku.HasLocationCapability(resourceskus.UltraSSDAvailable, location, zone) - err := fmt.Errorf("vm size %s does not support ultra disks in location %s. select a different vm size or disable ultra disks", spec.Size, location) + hasLocationCapability := sku.HasLocationCapability(resourceskus.UltraSSDAvailable, scaleSetSpec.Location, zone) + err := fmt.Errorf("vm size %s does not support ultra disks in location %s. select a different vm size or disable ultra disks", scaleSetSpec.Size, scaleSetSpec.Location) // Check support for ultra disks as data disks. - for _, disks := range spec.DataDisks { + for _, disks := range scaleSetSpec.DataDisks { if disks.ManagedDisk != nil && disks.ManagedDisk.StorageAccountType == string(compute.StorageAccountTypesUltraSSDLRS) && !hasLocationCapability { @@ -393,8 +224,8 @@ func (s *Service) validateSpec(ctx context.Context) error { } } // Check support for ultra disks as persistent volumes. - if spec.AdditionalCapabilities != nil && spec.AdditionalCapabilities.UltraSSDEnabled != nil { - if *spec.AdditionalCapabilities.UltraSSDEnabled && + if scaleSetSpec.AdditionalCapabilities != nil && scaleSetSpec.AdditionalCapabilities.UltraSSDEnabled != nil { + if *scaleSetSpec.AdditionalCapabilities.UltraSSDEnabled && !hasLocationCapability { return azure.WithTerminalError(err) } @@ -402,11 +233,11 @@ func (s *Service) validateSpec(ctx context.Context) error { } // Validate DiagnosticProfile spec - if spec.DiagnosticsProfile != nil && spec.DiagnosticsProfile.Boot != nil { - if spec.DiagnosticsProfile.Boot.StorageAccountType == infrav1.UserManagedDiagnosticsStorage { - if spec.DiagnosticsProfile.Boot.UserManaged == nil { + if scaleSetSpec.DiagnosticsProfile != nil && scaleSetSpec.DiagnosticsProfile.Boot != nil { + if scaleSetSpec.DiagnosticsProfile.Boot.StorageAccountType == infrav1.UserManagedDiagnosticsStorage { + if scaleSetSpec.DiagnosticsProfile.Boot.UserManaged == nil { return azure.WithTerminalError(fmt.Errorf("userManaged must be specified when storageAccountType is '%s'", infrav1.UserManagedDiagnosticsStorage)) - } else if spec.DiagnosticsProfile.Boot.UserManaged.StorageAccountURI == "" { + } else if scaleSetSpec.DiagnosticsProfile.Boot.UserManaged.StorageAccountURI == "" { return azure.WithTerminalError(fmt.Errorf("storageAccountURI cannot be empty when storageAccountType is '%s'", infrav1.UserManagedDiagnosticsStorage)) } } @@ -417,472 +248,49 @@ func (s *Service) validateSpec(ctx context.Context) error { string(infrav1.UserManagedDiagnosticsStorage), } - if !slice.Contains(possibleStorageAccountTypeValues, string(spec.DiagnosticsProfile.Boot.StorageAccountType)) { + if !slice.Contains(possibleStorageAccountTypeValues, string(scaleSetSpec.DiagnosticsProfile.Boot.StorageAccountType)) { return azure.WithTerminalError(fmt.Errorf("invalid storageAccountType: %s. Allowed values are %v", - spec.DiagnosticsProfile.Boot.StorageAccountType, possibleStorageAccountTypeValues)) + scaleSetSpec.DiagnosticsProfile.Boot.StorageAccountType, possibleStorageAccountTypeValues)) } } // Checking if selected availability zones are available selected VM type in location - azsInLocation, err := s.resourceSKUCache.GetZonesWithVMSize(ctx, spec.Size, s.Scope.Location()) + azsInLocation, err := s.resourceSKUCache.GetZonesWithVMSize(ctx, scaleSetSpec.Size, scaleSetSpec.Location) if err != nil { - return errors.Wrapf(err, "failed to get zones for VM type %s in location %s", spec.Size, s.Scope.Location()) + return errors.Wrapf(err, "failed to get zones for VM type %s in location %s", scaleSetSpec.Size, scaleSetSpec.Location) } - for _, az := range spec.FailureDomains { + for _, az := range scaleSetSpec.FailureDomains { if !slice.Contains(azsInLocation, az) { - return azure.WithTerminalError(errors.Errorf("availability zone %s is not available for VM type %s in location %s", az, spec.Size, s.Scope.Location())) + return azure.WithTerminalError(errors.Errorf("availability zone %s is not available for VM type %s in location %s", az, scaleSetSpec.Size, scaleSetSpec.Location)) } } return nil } -func (s *Service) buildVMSSFromSpec(ctx context.Context, vmssSpec azure.ScaleSetSpec) (compute.VirtualMachineScaleSet, error) { - ctx, _, done := tele.StartSpanWithLogger(ctx, "scalesets.Service.buildVMSSFromSpec") - defer done() - - sku, err := s.resourceSKUCache.Get(ctx, vmssSpec.Size, resourceskus.VirtualMachines) - if err != nil { - return compute.VirtualMachineScaleSet{}, errors.Wrapf(err, "failed to get find SKU %s in compute api", vmssSpec.Size) - } - - if vmssSpec.AcceleratedNetworking == nil { - // set accelerated networking to the capability of the VMSize - accelNet := sku.HasCapability(resourceskus.AcceleratedNetworking) - vmssSpec.AcceleratedNetworking = &accelNet - } - - extensions, err := s.generateExtensions(ctx) - if err != nil { - return compute.VirtualMachineScaleSet{}, err - } - - storageProfile, err := s.generateStorageProfile(ctx, vmssSpec, sku) - if err != nil { - return compute.VirtualMachineScaleSet{}, err - } - - securityProfile, err := getSecurityProfile(vmssSpec, sku) - if err != nil { - return compute.VirtualMachineScaleSet{}, err - } - - priority, evictionPolicy, billingProfile, err := converters.GetSpotVMOptions(vmssSpec.SpotVMOptions, vmssSpec.OSDisk.DiffDiskSettings) - if err != nil { - return compute.VirtualMachineScaleSet{}, errors.Wrapf(err, "failed to get Spot VM options") - } - - diagnosticsProfile := converters.GetDiagnosticsProfile(vmssSpec.DiagnosticsProfile) - - osProfile, err := s.generateOSProfile(ctx, vmssSpec) - if err != nil { - return compute.VirtualMachineScaleSet{}, err - } - - orchestrationMode := converters.GetOrchestrationMode(s.Scope.ScaleSetSpec().OrchestrationMode) - vmss := compute.VirtualMachineScaleSet{ - Location: ptr.To(s.Scope.Location()), - Sku: &compute.Sku{ - Name: ptr.To(vmssSpec.Size), - Tier: ptr.To("Standard"), - Capacity: ptr.To[int64](vmssSpec.Capacity), - }, - Zones: &vmssSpec.FailureDomains, - Plan: s.generateImagePlan(ctx), - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - OrchestrationMode: orchestrationMode, - SinglePlacementGroup: ptr.To(false), - VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{ - OsProfile: osProfile, - StorageProfile: storageProfile, - SecurityProfile: securityProfile, - DiagnosticsProfile: diagnosticsProfile, - NetworkProfile: &compute.VirtualMachineScaleSetNetworkProfile{ - NetworkInterfaceConfigurations: s.getVirtualMachineScaleSetNetworkConfiguration(vmssSpec), - }, - Priority: priority, - EvictionPolicy: evictionPolicy, - BillingProfile: billingProfile, - ExtensionProfile: &compute.VirtualMachineScaleSetExtensionProfile{ - Extensions: &extensions, - }, - }, - }, - } - - // Set properties specific to VMSS orchestration mode - switch orchestrationMode { - case compute.OrchestrationModeUniform: - vmss.VirtualMachineScaleSetProperties.Overprovision = ptr.To(false) - vmss.VirtualMachineScaleSetProperties.UpgradePolicy = &compute.UpgradePolicy{Mode: compute.UpgradeModeManual} - case compute.OrchestrationModeFlexible: - vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.NetworkProfile.NetworkAPIVersion = - compute.NetworkAPIVersionTwoZeroTwoZeroHyphenMinusOneOneHyphenMinusZeroOne - vmss.VirtualMachineScaleSetProperties.PlatformFaultDomainCount = ptr.To[int32](1) - if len(vmssSpec.FailureDomains) > 1 { - vmss.VirtualMachineScaleSetProperties.PlatformFaultDomainCount = ptr.To[int32](int32(len(vmssSpec.FailureDomains))) - } - } - - // Assign Identity to VMSS - if vmssSpec.Identity == infrav1.VMIdentitySystemAssigned { - vmss.Identity = &compute.VirtualMachineScaleSetIdentity{ - Type: compute.ResourceIdentityTypeSystemAssigned, - } - } else if vmssSpec.Identity == infrav1.VMIdentityUserAssigned { - userIdentitiesMap, err := converters.UserAssignedIdentitiesToVMSSSDK(vmssSpec.UserAssignedIdentities) - if err != nil { - return vmss, errors.Wrapf(err, "failed to assign identity %q", vmssSpec.Name) - } - vmss.Identity = &compute.VirtualMachineScaleSetIdentity{ - Type: compute.ResourceIdentityTypeUserAssigned, - UserAssignedIdentities: userIdentitiesMap, - } - } - - // Provisionally detect whether there is any Data Disk defined which uses UltraSSDs. - // If that's the case, enable the UltraSSD capability. - for _, dataDisk := range vmssSpec.DataDisks { - if dataDisk.ManagedDisk != nil && dataDisk.ManagedDisk.StorageAccountType == string(compute.StorageAccountTypesUltraSSDLRS) { - vmss.VirtualMachineScaleSetProperties.AdditionalCapabilities = &compute.AdditionalCapabilities{ - UltraSSDEnabled: ptr.To(true), - } - } - } - - // Set Additional Capabilities if any is present on the spec. - if vmssSpec.AdditionalCapabilities != nil { - // Set UltraSSDEnabled if a specific value is set on the spec for it. - if vmssSpec.AdditionalCapabilities.UltraSSDEnabled != nil { - vmss.AdditionalCapabilities.UltraSSDEnabled = vmssSpec.AdditionalCapabilities.UltraSSDEnabled - } - } - - if vmssSpec.TerminateNotificationTimeout != nil { - vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.ScheduledEventsProfile = &compute.ScheduledEventsProfile{ - TerminateNotificationProfile: &compute.TerminateNotificationProfile{ - NotBeforeTimeout: ptr.To(fmt.Sprintf("PT%dM", *vmssSpec.TerminateNotificationTimeout)), - Enable: ptr.To(true), - }, - } - } - - tags := infrav1.Build(infrav1.BuildParams{ - ClusterName: s.Scope.ClusterName(), - Lifecycle: infrav1.ResourceLifecycleOwned, - Name: ptr.To(vmssSpec.Name), - Role: ptr.To(infrav1.Node), - Additional: s.Scope.AdditionalTags(), - }) - - vmss.Tags = converters.TagsToMap(tags) - return vmss, nil -} - -func (s *Service) getVirtualMachineScaleSetNetworkConfiguration(vmssSpec azure.ScaleSetSpec) *[]compute.VirtualMachineScaleSetNetworkConfiguration { - var backendAddressPools []compute.SubResource - if vmssSpec.PublicLBName != "" { - if vmssSpec.PublicLBAddressPoolName != "" { - backendAddressPools = append(backendAddressPools, - compute.SubResource{ - ID: ptr.To(azure.AddressPoolID(s.Scope.SubscriptionID(), s.Scope.ResourceGroup(), vmssSpec.PublicLBName, vmssSpec.PublicLBAddressPoolName)), - }) - } - } - nicConfigs := []compute.VirtualMachineScaleSetNetworkConfiguration{} - for i, n := range vmssSpec.NetworkInterfaces { - nicConfig := compute.VirtualMachineScaleSetNetworkConfiguration{} - nicConfig.VirtualMachineScaleSetNetworkConfigurationProperties = &compute.VirtualMachineScaleSetNetworkConfigurationProperties{} - nicConfig.Name = ptr.To(vmssSpec.Name + "-nic-" + strconv.Itoa(i)) - nicConfig.EnableIPForwarding = ptr.To(true) - if n.AcceleratedNetworking != nil { - nicConfig.VirtualMachineScaleSetNetworkConfigurationProperties.EnableAcceleratedNetworking = n.AcceleratedNetworking - } else { - // If AcceleratedNetworking is not specified, use the value from the VMSS spec. - // It will be set to true if the VMSS SKU supports it. - nicConfig.VirtualMachineScaleSetNetworkConfigurationProperties.EnableAcceleratedNetworking = vmssSpec.AcceleratedNetworking - } - - // Create IPConfigs - ipconfigs := []compute.VirtualMachineScaleSetIPConfiguration{} - for j := 0; j < n.PrivateIPConfigs; j++ { - ipconfig := compute.VirtualMachineScaleSetIPConfiguration{ - Name: ptr.To(fmt.Sprintf("ipConfig" + strconv.Itoa(j))), - VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ - PrivateIPAddressVersion: compute.IPVersionIPv4, - Subnet: &compute.APIEntityReference{ - ID: ptr.To(azure.SubnetID(s.Scope.SubscriptionID(), vmssSpec.VNetResourceGroup, vmssSpec.VNetName, n.SubnetName)), - }, - }, - } - - if j == 0 { - // Always use the first IPConfig as the Primary - ipconfig.Primary = ptr.To(true) - } - ipconfigs = append(ipconfigs, ipconfig) - } - if vmssSpec.IPv6Enabled { - ipv6Config := compute.VirtualMachineScaleSetIPConfiguration{ - Name: ptr.To("ipConfigv6"), - VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ - PrivateIPAddressVersion: compute.IPVersionIPv6, - Primary: ptr.To(false), - Subnet: &compute.APIEntityReference{ - ID: ptr.To(azure.SubnetID(s.Scope.SubscriptionID(), vmssSpec.VNetResourceGroup, vmssSpec.VNetName, n.SubnetName)), - }, - }, - } - ipconfigs = append(ipconfigs, ipv6Config) - } - if i == 0 { - ipconfigs[0].LoadBalancerBackendAddressPools = &backendAddressPools - nicConfig.VirtualMachineScaleSetNetworkConfigurationProperties.Primary = ptr.To(true) - } - nicConfig.VirtualMachineScaleSetNetworkConfigurationProperties.IPConfigurations = &ipconfigs - nicConfigs = append(nicConfigs, nicConfig) - } - return &nicConfigs -} - // getVirtualMachineScaleSet provides information about a Virtual Machine Scale Set and its instances. -func (s *Service) getVirtualMachineScaleSet(ctx context.Context, vmssName string) (*azure.VMSS, error) { +func (s *Service) getVirtualMachineScaleSet(ctx context.Context, spec azure.ResourceSpecGetter) (*azure.VMSS, error) { ctx, _, done := tele.StartSpanWithLogger(ctx, "scalesets.Service.getVirtualMachineScaleSet") defer done() - vmss, err := s.Client.Get(ctx, s.Scope.ResourceGroup(), vmssName) + vmssResult, err := s.Client.Get(ctx, spec) if err != nil { - return nil, errors.Wrap(err, "failed to get existing vmss") + return nil, errors.Wrap(err, "failed to get existing VMSS") } - - vmssInstances, err := s.Client.ListInstances(ctx, s.Scope.ResourceGroup(), vmssName) - if err != nil { - return nil, errors.Wrap(err, "failed to list instances") - } - - return converters.SDKToVMSS(vmss, vmssInstances), nil -} - -// getVirtualMachineScaleSetIfDone gets a Virtual Machine Scale Set and its instances from Azure if the future is completed. -func (s *Service) getVirtualMachineScaleSetIfDone(ctx context.Context, future *infrav1.Future) (*azure.VMSS, error) { - ctx, _, done := tele.StartSpanWithLogger(ctx, "scalesets.Service.getVirtualMachineScaleSetIfDone") - defer done() - - vmss, err := s.GetResultIfDone(ctx, future) - if err != nil { - return nil, errors.Wrap(err, "failed to get result from future") + vmss, ok := vmssResult.(compute.VirtualMachineScaleSet) + if !ok { + return nil, errors.Errorf("%T is not a compute.VirtualMachineScaleSet", vmssResult) } - vmssInstances, err := s.Client.ListInstances(ctx, future.ResourceGroup, future.Name) + vmssInstances, err := s.Client.ListInstances(ctx, spec.ResourceGroupName(), spec.ResourceName()) if err != nil { return nil, errors.Wrap(err, "failed to list instances") } - return converters.SDKToVMSS(vmss, vmssInstances), nil -} - -func (s *Service) generateExtensions(ctx context.Context) ([]compute.VirtualMachineScaleSetExtension, error) { - extensions := make([]compute.VirtualMachineScaleSetExtension, len(s.Scope.VMSSExtensionSpecs())) - for i, extensionSpec := range s.Scope.VMSSExtensionSpecs() { - extensionSpec := extensionSpec - parameters, err := extensionSpec.Parameters(ctx, nil) - if err != nil { - return nil, err - } - vmssextension, ok := parameters.(compute.VirtualMachineScaleSetExtension) - if !ok { - return nil, errors.Errorf("%T is not a compute.VirtualMachineScaleSetExtension", parameters) - } - extensions[i] = vmssextension - } - - return extensions, nil -} - -// generateStorageProfile generates a pointer to a compute.VirtualMachineScaleSetStorageProfile which can utilized for VM creation. -func (s *Service) generateStorageProfile(ctx context.Context, vmssSpec azure.ScaleSetSpec, sku resourceskus.SKU) (*compute.VirtualMachineScaleSetStorageProfile, error) { - ctx, _, done := tele.StartSpanWithLogger(ctx, "scalesets.Service.generateStorageProfile") - defer done() - - storageProfile := &compute.VirtualMachineScaleSetStorageProfile{ - OsDisk: &compute.VirtualMachineScaleSetOSDisk{ - OsType: compute.OperatingSystemTypes(vmssSpec.OSDisk.OSType), - CreateOption: compute.DiskCreateOptionTypesFromImage, - DiskSizeGB: vmssSpec.OSDisk.DiskSizeGB, - }, - } - - // enable ephemeral OS - if vmssSpec.OSDisk.DiffDiskSettings != nil { - if !sku.HasCapability(resourceskus.EphemeralOSDisk) { - return nil, fmt.Errorf("vm size %s does not support ephemeral os. select a different vm size or disable ephemeral os", vmssSpec.Size) - } - - storageProfile.OsDisk.DiffDiskSettings = &compute.DiffDiskSettings{ - Option: compute.DiffDiskOptions(vmssSpec.OSDisk.DiffDiskSettings.Option), - } - } - - if vmssSpec.OSDisk.ManagedDisk != nil { - storageProfile.OsDisk.ManagedDisk = &compute.VirtualMachineScaleSetManagedDiskParameters{} - if vmssSpec.OSDisk.ManagedDisk.StorageAccountType != "" { - storageProfile.OsDisk.ManagedDisk.StorageAccountType = compute.StorageAccountTypes(vmssSpec.OSDisk.ManagedDisk.StorageAccountType) - } - if vmssSpec.OSDisk.ManagedDisk.DiskEncryptionSet != nil { - storageProfile.OsDisk.ManagedDisk.DiskEncryptionSet = &compute.DiskEncryptionSetParameters{ID: ptr.To(vmssSpec.OSDisk.ManagedDisk.DiskEncryptionSet.ID)} - } - } - - if vmssSpec.OSDisk.CachingType != "" { - storageProfile.OsDisk.Caching = compute.CachingTypes(vmssSpec.OSDisk.CachingType) - } - - dataDisks := make([]compute.VirtualMachineScaleSetDataDisk, len(vmssSpec.DataDisks)) - for i, disk := range vmssSpec.DataDisks { - dataDisks[i] = compute.VirtualMachineScaleSetDataDisk{ - CreateOption: compute.DiskCreateOptionTypesEmpty, - DiskSizeGB: ptr.To[int32](disk.DiskSizeGB), - Lun: disk.Lun, - Name: ptr.To(azure.GenerateDataDiskName(vmssSpec.Name, disk.NameSuffix)), - } - - if disk.ManagedDisk != nil { - dataDisks[i].ManagedDisk = &compute.VirtualMachineScaleSetManagedDiskParameters{ - StorageAccountType: compute.StorageAccountTypes(disk.ManagedDisk.StorageAccountType), - } - - if disk.ManagedDisk.DiskEncryptionSet != nil { - dataDisks[i].ManagedDisk.DiskEncryptionSet = &compute.DiskEncryptionSetParameters{ID: ptr.To(disk.ManagedDisk.DiskEncryptionSet.ID)} - } - } - } - storageProfile.DataDisks = &dataDisks - - image, err := s.Scope.GetVMImage(ctx) - if err != nil { - return nil, errors.Wrap(err, "failed to get VM image") - } - - s.Scope.SaveVMImageToStatus(image) - - imageRef, err := converters.ImageToSDK(image) - if err != nil { - return nil, err - } - - storageProfile.ImageReference = imageRef - - return storageProfile, nil -} - -func (s *Service) generateOSProfile(ctx context.Context, vmssSpec azure.ScaleSetSpec) (*compute.VirtualMachineScaleSetOSProfile, error) { - sshKey, err := base64.StdEncoding.DecodeString(vmssSpec.SSHKeyData) - if err != nil { - return nil, errors.Wrap(err, "failed to decode ssh public key") - } - bootstrapData, err := s.Scope.GetBootstrapData(ctx) - if err != nil { - return nil, errors.Wrap(err, "failed to retrieve bootstrap data") - } - - osProfile := &compute.VirtualMachineScaleSetOSProfile{ - ComputerNamePrefix: ptr.To(vmssSpec.Name), - AdminUsername: ptr.To(azure.DefaultUserName), - CustomData: ptr.To(bootstrapData), - } - - switch vmssSpec.OSDisk.OSType { - case string(compute.OperatingSystemTypesWindows): - // Cloudbase-init is used to generate a password. - // https://cloudbase-init.readthedocs.io/en/latest/plugins.html#setting-password-main - // - // We generate a random password here in case of failure - // but the password on the VM will NOT be the same as created here. - // Access is provided via SSH public key that is set during deployment - // Azure also provides a way to reset user passwords in the case of need. - osProfile.AdminPassword = ptr.To(generators.SudoRandomPassword(123)) - osProfile.WindowsConfiguration = &compute.WindowsConfiguration{ - EnableAutomaticUpdates: ptr.To(false), - } - default: - osProfile.LinuxConfiguration = &compute.LinuxConfiguration{ - DisablePasswordAuthentication: ptr.To(true), - SSH: &compute.SSHConfiguration{ - PublicKeys: &[]compute.SSHPublicKey{ - { - Path: ptr.To(fmt.Sprintf("/home/%s/.ssh/authorized_keys", azure.DefaultUserName)), - KeyData: ptr.To(string(sshKey)), - }, - }, - }, - } - } - - return osProfile, nil -} - -func (s *Service) generateImagePlan(ctx context.Context) *compute.Plan { - ctx, log, done := tele.StartSpanWithLogger(ctx, "scalesets.Service.generateImagePlan") - defer done() - - image, err := s.Scope.GetVMImage(ctx) - if err != nil { - log.Error(err, "failed to get vm image, disabling Plan") - return nil - } - - if image.SharedGallery != nil && image.SharedGallery.Publisher != nil && image.SharedGallery.SKU != nil && image.SharedGallery.Offer != nil { - return &compute.Plan{ - Publisher: image.SharedGallery.Publisher, - Name: image.SharedGallery.SKU, - Product: image.SharedGallery.Offer, - } - } - - if image.Marketplace == nil || !image.Marketplace.ThirdPartyImage { - return nil - } - - if image.Marketplace.Publisher == "" || image.Marketplace.SKU == "" || image.Marketplace.Offer == "" { - return nil - } - - return &compute.Plan{ - Publisher: ptr.To(image.Marketplace.Publisher), - Name: ptr.To(image.Marketplace.SKU), - Product: ptr.To(image.Marketplace.Offer), - } -} - -func getVMSSUpdateFromVMSS(vmss compute.VirtualMachineScaleSet) (compute.VirtualMachineScaleSetUpdate, error) { - jsonData, err := vmss.MarshalJSON() - if err != nil { - return compute.VirtualMachineScaleSetUpdate{}, err - } - - var update compute.VirtualMachineScaleSetUpdate - if err := update.UnmarshalJSON(jsonData); err != nil { - return update, err - } - - // wipe out network profile, so updates won't conflict with Cloud Provider updates - update.VirtualMachineProfile.NetworkProfile = nil - return update, nil -} - -func getSecurityProfile(vmssSpec azure.ScaleSetSpec, sku resourceskus.SKU) (*compute.SecurityProfile, error) { - if vmssSpec.SecurityProfile == nil { - return nil, nil - } - - if !sku.HasCapability(resourceskus.EncryptionAtHost) { - return nil, azure.WithTerminalError(errors.Errorf("encryption at host is not supported for VM type %s", vmssSpec.Size)) - } + result := converters.SDKToVMSS(vmss, vmssInstances) - return &compute.SecurityProfile{ - EncryptionAtHost: ptr.To(*vmssSpec.SecurityProfile.EncryptionAtHost), - }, nil + return &result, nil } // IsManaged returns always returns true as CAPZ does not support BYO scale set. diff --git a/azure/services/scalesets/scalesets_test.go b/azure/services/scalesets/scalesets_test.go index 1cbc56c7c7a..b96b0f36410 100644 --- a/azure/services/scalesets/scalesets_test.go +++ b/azure/services/scalesets/scalesets_test.go @@ -25,11 +25,12 @@ import ( "github.com/Azure/go-autorest/autorest" . "github.com/onsi/gomega" "go.uber.org/mock/gomock" - "k8s.io/apimachinery/pkg/api/resource" "k8s.io/client-go/kubernetes/scheme" "k8s.io/utils/ptr" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" + "sigs.k8s.io/cluster-api-provider-azure/azure/converters" + "sigs.k8s.io/cluster-api-provider-azure/azure/services/async/mock_async" "sigs.k8s.io/cluster-api-provider-azure/azure/services/resourceskus" "sigs.k8s.io/cluster-api-provider-azure/azure/services/scalesets/mock_scalesets" gomockinternal "sigs.k8s.io/cluster-api-provider-azure/internal/test/matchers/gomock" @@ -42,817 +43,244 @@ const ( defaultResourceGroup = "my-rg" defaultVMSSName = "my-vmss" vmSizeEPH = "VM_SIZE_EPH" + vmSizeUSSD = "VM_SIZE_USSD" + defaultVMSSID = "subscriptions/1234/resourceGroups/my_resource_group/providers/Microsoft.Compute/virtualMachines/my-vm-id" + sshKeyData = "ZmFrZXNzaGtleQo=" ) -func init() { - _ = clusterv1.AddToScheme(scheme.Scheme) -} - -func TestGetExistingVMSS(t *testing.T) { - testcases := []struct { - name string - vmssName string - result *azure.VMSS - expectedError string - expect func(s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) - }{ - { - name: "scale set not found", - vmssName: "my-vmss", - result: &azure.VMSS{}, - expectedError: "failed to get existing vmss: #: Not found: StatusCode=404", - expect: func(s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { - s.ResourceGroup().AnyTimes().Return("my-rg") - m.Get(gomockinternal.AContext(), "my-rg", "my-vmss").Return(compute.VirtualMachineScaleSet{}, autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: http.StatusNotFound}, "Not found")) - }, - }, - { - name: "get existing vmss", - vmssName: "my-vmss", - result: &azure.VMSS{ - ID: "my-id", - Name: "my-vmss", - State: "Succeeded", - Sku: "Standard_D2", - Identity: "", - Tags: nil, - Capacity: int64(1), - Zones: []string{"1", "3"}, - Instances: []azure.VMSSVM{ - { - ID: "my-vm-id", - InstanceID: "my-vm-1", - Name: "instance-000001", - State: "Succeeded", - }, - }, - }, - expectedError: "", - expect: func(s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { - s.ResourceGroup().AnyTimes().Return("my-rg") - m.Get(gomockinternal.AContext(), "my-rg", "my-vmss").Return(compute.VirtualMachineScaleSet{ - ID: ptr.To("my-id"), - Name: ptr.To("my-vmss"), - Sku: &compute.Sku{ - Capacity: ptr.To[int64](1), - Name: ptr.To("Standard_D2"), - }, - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - SinglePlacementGroup: ptr.To(false), - ProvisioningState: ptr.To("Succeeded"), - }, - Zones: &[]string{"1", "3"}, - }, nil) - m.ListInstances(gomock.Any(), "my-rg", "my-vmss").Return([]compute.VirtualMachineScaleSetVM{ - { - ID: ptr.To("my-vm-id"), - InstanceID: ptr.To("my-vm-1"), - Name: ptr.To("my-vm"), - VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{ - ProvisioningState: ptr.To("Succeeded"), - OsProfile: &compute.OSProfile{ - ComputerName: ptr.To("instance-000001"), - }, - }, - }, - }, nil) - }, - }, - { - name: "list instances fails", - vmssName: "my-vmss", - result: &azure.VMSS{}, - expectedError: "failed to list instances: #: Not found: StatusCode=404", - expect: func(s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { - s.ResourceGroup().AnyTimes().Return("my-rg") - m.Get(gomockinternal.AContext(), "my-rg", "my-vmss").Return(compute.VirtualMachineScaleSet{ - ID: ptr.To("my-id"), - Name: ptr.To("my-vmss"), - VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ - SinglePlacementGroup: ptr.To(false), - ProvisioningState: ptr.To("Succeeded"), - }, - }, nil) - m.ListInstances(gomockinternal.AContext(), "my-rg", "my-vmss").Return([]compute.VirtualMachineScaleSetVM{}, autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: http.StatusNotFound}, "Not found")) +var ( + defaultImage = infrav1.Image{ + Marketplace: &infrav1.AzureMarketplaceImage{ + ImagePlan: infrav1.ImagePlan{ + Publisher: "fake-publisher", + Offer: "my-offer", + SKU: "sku-id", }, + Version: "1.0", }, } - for _, tc := range testcases { - tc := tc - t.Run(tc.name, func(t *testing.T) { - g := NewWithT(t) - t.Parallel() - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() + internalError = autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: http.StatusInternalServerError}, "Internal Server Error") + notFoundError = autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: http.StatusNotFound}, "Not Found") +) - scopeMock := mock_scalesets.NewMockScaleSetScope(mockCtrl) - clientMock := mock_scalesets.NewMockClient(mockCtrl) +func init() { + _ = clusterv1.AddToScheme(scheme.Scheme) +} - tc.expect(scopeMock.EXPECT(), clientMock.EXPECT()) +func getDefaultVMSSSpec() azure.ResourceSpecGetter { + defaultSpec := newDefaultVMSSSpec() + defaultSpec.DataDisks = append(defaultSpec.DataDisks, infrav1.DataDisk{ + NameSuffix: "my_disk_with_ultra_disks", + DiskSizeGB: 128, + Lun: ptr.To[int32](3), + ManagedDisk: &infrav1.ManagedDiskParameters{ + StorageAccountType: "UltraSSD_LRS", + }, + }) - s := &Service{ - Scope: scopeMock, - Client: clientMock, - } + return &defaultSpec +} - result, err := s.getVirtualMachineScaleSet(context.TODO(), tc.vmssName) - if tc.expectedError != "" { - g.Expect(err).To(HaveOccurred()) - t.Log(err.Error()) - g.Expect(err).To(MatchError(tc.expectedError)) - } else { - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(result).To(BeEquivalentTo(tc.result)) - } - }) - } +func getResultVMSS() compute.VirtualMachineScaleSet { + resultVMSS := newDefaultVMSS("VM_SIZE") + resultVMSS.ID = ptr.To(defaultVMSSID) + + return resultVMSS } func TestReconcileVMSS(t *testing.T) { - var ( - putFuture = &infrav1.Future{ - Type: infrav1.PutFuture, - ResourceGroup: defaultResourceGroup, - Name: defaultVMSSName, - } - - patchFuture = &infrav1.Future{ - Type: infrav1.PatchFuture, - ResourceGroup: defaultResourceGroup, - Name: defaultVMSSName, - } - ) + defaultInstances := newDefaultInstances() + resultVMSS := newDefaultVMSS("VM_SIZE") + resultVMSS.ID = ptr.To(defaultVMSSID) + fetchedVMSS := converters.SDKToVMSS(getResultVMSS(), defaultInstances) testcases := []struct { name string - expect func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) + expect func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder, m *mock_scalesets.MockClientMockRecorder) expectedError string }{ { - name: "should start creating a vmss", - expectedError: "failed to get VMSS my-vmss after create or update: failed to get result from future: operation type PUT on Azure resource my-rg/my-vmss is not done", - expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { - defaultSpec := newDefaultVMSSSpec() - defaultSpec.DataDisks = append(defaultSpec.DataDisks, infrav1.DataDisk{ - NameSuffix: "my_disk_with_ultra_disks", - DiskSizeGB: 128, - Lun: ptr.To[int32](3), - ManagedDisk: &infrav1.ManagedDiskParameters{ - StorageAccountType: "UltraSSD_LRS", - }, - }) - s.ScaleSetSpec().Return(defaultSpec).AnyTimes() - setupDefaultVMSSStartCreatingExpectations(s, m) - vmss := newDefaultVMSS("VM_SIZE") - vmss.VirtualMachineScaleSetProperties.AdditionalCapabilities = &compute.AdditionalCapabilities{UltraSSDEnabled: ptr.To(true)} - m.CreateOrUpdateAsync(gomockinternal.AContext(), defaultResourceGroup, defaultVMSSName, gomockinternal.DiffEq(vmss)). - Return(putFuture, nil) - setupCreatingSucceededExpectations(s, m, newDefaultExistingVMSS("VM_SIZE"), putFuture) - }, - }, - { - name: "should finish creating a vmss when long running operation is done", + name: "update an existing vmss", expectedError: "", - expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { - defaultSpec := newDefaultVMSSSpec() - s.ScaleSetSpec().Return(defaultSpec).AnyTimes() - createdVMSS := newDefaultVMSS("VM_SIZE") - instances := newDefaultInstances() - - setupDefaultVMSSInProgressOperationDoneExpectations(s, m, createdVMSS, instances) - s.DeleteLongRunningOperationState(defaultSpec.Name, serviceName, infrav1.PutFuture) - s.DeleteLongRunningOperationState(defaultSpec.Name, serviceName, infrav1.PatchFuture) + expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder, m *mock_scalesets.MockClientMockRecorder) { + spec := getDefaultVMSSSpec() + // Validate spec + s.ScaleSetSpec(gomockinternal.AContext()).Return(spec).AnyTimes() + m.Get(gomockinternal.AContext(), &defaultSpec).Return(&resultVMSS, nil) + m.ListInstances(gomockinternal.AContext(), defaultSpec.ResourceGroup, defaultSpec.Name).Return(defaultInstances, nil) + r.CreateOrUpdateResource(gomockinternal.AContext(), spec, serviceName).Return(getResultVMSS(), nil) s.UpdatePutStatus(infrav1.BootstrapSucceededCondition, serviceName, nil) - s.HasReplicasExternallyManaged(gomockinternal.AContext()).Return(false) + + s.ReconcileReplicas(gomockinternal.AContext(), &fetchedVMSS).Return(nil) + s.SetProviderID(azureutil.ProviderIDPrefix + defaultVMSSID) + s.SetVMSSState(&fetchedVMSS) }, }, { - name: "Windows VMSS should not get patched", + name: "create a vmss, skip list instances if vmss doesn't exist", expectedError: "", - expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { - defaultSpec := newWindowsVMSSSpec() - s.ScaleSetSpec().Return(defaultSpec).AnyTimes() - createdVMSS := newDefaultWindowsVMSS() - instances := newDefaultInstances() - - setupDefaultVMSSInProgressOperationDoneExpectations(s, m, createdVMSS, instances) - s.DeleteLongRunningOperationState(defaultSpec.Name, serviceName, infrav1.PutFuture) - s.DeleteLongRunningOperationState(defaultSpec.Name, serviceName, infrav1.PatchFuture) + expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder, m *mock_scalesets.MockClientMockRecorder) { + spec := getDefaultVMSSSpec() + // Validate spec + s.ScaleSetSpec(gomockinternal.AContext()).Return(spec).AnyTimes() + m.Get(gomockinternal.AContext(), &defaultSpec).Return(nil, notFoundError) + r.CreateOrUpdateResource(gomockinternal.AContext(), spec, serviceName).Return(getResultVMSS(), nil) s.UpdatePutStatus(infrav1.BootstrapSucceededCondition, serviceName, nil) - s.HasReplicasExternallyManaged(gomockinternal.AContext()).Return(false) - }, - }, - { - name: "should start creating vmss with defaulted accelerated networking when size allows", - expectedError: "failed to get VMSS my-vmss after create or update: failed to get result from future: operation type PUT on Azure resource my-rg/my-vmss is not done", - expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { - spec := newDefaultVMSSSpec() - spec.Size = "VM_SIZE_AN" - s.ScaleSetSpec().Return(spec).AnyTimes() - setupDefaultVMSSStartCreatingExpectations(s, m) - vmss := newDefaultVMSS("VM_SIZE_AN") - netConfigs := vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations - (*netConfigs)[0].EnableAcceleratedNetworking = ptr.To(true) - vmss.Sku.Name = ptr.To(spec.Size) - m.CreateOrUpdateAsync(gomockinternal.AContext(), defaultResourceGroup, defaultVMSSName, gomockinternal.DiffEq(vmss)). - Return(putFuture, nil) - setupCreatingSucceededExpectations(s, m, newDefaultExistingVMSS("VM_SIZE_AN"), putFuture) + + s.ReconcileReplicas(gomockinternal.AContext(), &fetchedVMSS).Return(nil) + s.SetProviderID(azureutil.ProviderIDPrefix + defaultVMSSID) + s.SetVMSSState(&fetchedVMSS) }, }, { - name: "should start creating vmss with custom subnet when specified", - expectedError: "failed to get VMSS my-vmss after create or update: failed to get result from future: operation type PUT on Azure resource my-rg/my-vmss is not done", - expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { - spec := newDefaultVMSSSpec() - spec.Size = "VM_SIZE_AN" - spec.NetworkInterfaces = []infrav1.NetworkInterface{ - { - SubnetName: "somesubnet", - PrivateIPConfigs: 1, // defaulter sets this to one - }, - } - s.ScaleSetSpec().Return(spec).AnyTimes() - setupDefaultVMSSStartCreatingExpectations(s, m) - vmss := newDefaultVMSS("VM_SIZE_AN") - netConfigs := vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations - (*netConfigs)[0].Name = ptr.To("my-vmss-nic-0") - (*netConfigs)[0].EnableIPForwarding = ptr.To(true) - (*netConfigs)[0].EnableAcceleratedNetworking = ptr.To(true) - nic1IPConfigs := (*netConfigs)[0].IPConfigurations - (*nic1IPConfigs)[0].Name = ptr.To("ipConfig0") - (*nic1IPConfigs)[0].PrivateIPAddressVersion = compute.IPVersionIPv4 - (*nic1IPConfigs)[0].Subnet = &compute.APIEntityReference{ - ID: ptr.To("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/virtualNetworks/my-vnet/subnets/somesubnet"), - } - (*netConfigs)[0].EnableAcceleratedNetworking = ptr.To(true) - (*netConfigs)[0].Primary = ptr.To(true) - m.CreateOrUpdateAsync(gomockinternal.AContext(), defaultResourceGroup, defaultVMSSName, gomockinternal.DiffEq(vmss)). - Return(putFuture, nil) - setupCreatingSucceededExpectations(s, m, newDefaultExistingVMSS("VM_SIZE_AN"), putFuture) + name: "error getting existing vmss", + expectedError: "failed to get existing VMSS: #: Internal Server Error: StatusCode=500", + expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder, m *mock_scalesets.MockClientMockRecorder) { + spec := getDefaultVMSSSpec() + // Validate spec + s.ScaleSetSpec(gomockinternal.AContext()).Return(spec).AnyTimes() + m.Get(gomockinternal.AContext(), &defaultSpec).Return(nil, internalError) }, }, { - name: "should start creating vmss with custom networking when specified", - expectedError: "failed to get VMSS my-vmss after create or update: failed to get result from future: operation type PUT on Azure resource my-rg/my-vmss is not done", - expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { - spec := newDefaultVMSSSpec() - spec.DataDisks = append(spec.DataDisks, infrav1.DataDisk{ - NameSuffix: "my_disk_with_ultra_disks", - DiskSizeGB: 128, - Lun: ptr.To[int32](3), - ManagedDisk: &infrav1.ManagedDiskParameters{ - StorageAccountType: "UltraSSD_LRS", - }, - }) - spec.NetworkInterfaces = []infrav1.NetworkInterface{ - { - SubnetName: "my-subnet", - PrivateIPConfigs: 1, - AcceleratedNetworking: ptr.To(true), - }, - { - SubnetName: "subnet2", - PrivateIPConfigs: 2, - AcceleratedNetworking: ptr.To(true), - }, - } - s.ScaleSetSpec().Return(spec).AnyTimes() - setupDefaultVMSSStartCreatingExpectations(s, m) - vmss := newDefaultVMSS("VM_SIZE") - vmss.VirtualMachineScaleSetProperties.AdditionalCapabilities = &compute.AdditionalCapabilities{UltraSSDEnabled: ptr.To(true)} - netConfigs := vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations - (*netConfigs)[0].Name = ptr.To("my-vmss-nic-0") - (*netConfigs)[0].EnableIPForwarding = ptr.To(true) - nic1IPConfigs := (*netConfigs)[0].IPConfigurations - (*nic1IPConfigs)[0].Name = ptr.To("ipConfig0") - (*nic1IPConfigs)[0].PrivateIPAddressVersion = compute.IPVersionIPv4 - (*netConfigs)[0].EnableAcceleratedNetworking = ptr.To(true) - (*netConfigs)[0].Primary = ptr.To(true) - vmssIPConfigs := []compute.VirtualMachineScaleSetIPConfiguration{ - { - Name: ptr.To("ipConfig0"), - VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ - Primary: ptr.To(true), - PrivateIPAddressVersion: compute.IPVersionIPv4, - Subnet: &compute.APIEntityReference{ - ID: ptr.To("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/virtualNetworks/my-vnet/subnets/subnet2"), - }, - }, - }, - { - Name: ptr.To("ipConfig1"), - VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ - PrivateIPAddressVersion: compute.IPVersionIPv4, - Subnet: &compute.APIEntityReference{ - ID: ptr.To("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/virtualNetworks/my-vnet/subnets/subnet2"), - }, - }, - }, - } - *netConfigs = append(*netConfigs, compute.VirtualMachineScaleSetNetworkConfiguration{ - Name: ptr.To("my-vmss-nic-1"), - VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{ - EnableAcceleratedNetworking: ptr.To(true), - IPConfigurations: &vmssIPConfigs, - EnableIPForwarding: ptr.To(true), - }, - }) - m.CreateOrUpdateAsync(gomockinternal.AContext(), defaultResourceGroup, defaultVMSSName, gomockinternal.DiffEq(vmss)). - Return(putFuture, nil) - setupCreatingSucceededExpectations(s, m, newDefaultExistingVMSS("VM_SIZE"), putFuture) + name: "failed to list instances", + expectedError: "failed to get existing instances: #: Internal Server Error: StatusCode=500", + expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder, m *mock_scalesets.MockClientMockRecorder) { + spec := getDefaultVMSSSpec() + // Validate spec + s.ScaleSetSpec(gomockinternal.AContext()).Return(spec).AnyTimes() + m.Get(gomockinternal.AContext(), &defaultSpec).Return(&resultVMSS, nil) + m.ListInstances(gomockinternal.AContext(), defaultSpec.ResourceGroup, defaultSpec.Name).Return(defaultInstances, internalError) + s.UpdatePutStatus(infrav1.BootstrapSucceededCondition, serviceName, gomockinternal.ErrStrEq("failed to get existing instances: #: Internal Server Error: StatusCode=500")) }, }, { - name: "should start creating a vmss with spot vm", - expectedError: "failed to get VMSS my-vmss after create or update: failed to get result from future: operation type PUT on Azure resource my-rg/my-vmss is not done", - expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { - spec := newDefaultVMSSSpec() - spec.SpotVMOptions = &infrav1.SpotVMOptions{} - spec.DataDisks = append(spec.DataDisks, infrav1.DataDisk{ - NameSuffix: "my_disk_with_ultra_disks", - DiskSizeGB: 128, - Lun: ptr.To[int32](3), - ManagedDisk: &infrav1.ManagedDiskParameters{ - StorageAccountType: "UltraSSD_LRS", - }, - }) - s.ScaleSetSpec().Return(spec).AnyTimes() - setupDefaultVMSSStartCreatingExpectations(s, m) - vmss := newDefaultVMSS("VM_SIZE") - vmss.VirtualMachineScaleSetProperties.AdditionalCapabilities = &compute.AdditionalCapabilities{UltraSSDEnabled: ptr.To(true)} - vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.Priority = compute.VirtualMachinePriorityTypesSpot - m.CreateOrUpdateAsync(gomockinternal.AContext(), defaultResourceGroup, defaultVMSSName, gomockinternal.DiffEq(vmss)). - Return(putFuture, nil) - setupCreatingSucceededExpectations(s, m, newDefaultExistingVMSS("VM_SIZE"), putFuture) + name: "failed to create a vmss", + expectedError: "#: Internal Server Error: StatusCode=500", + expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder, m *mock_scalesets.MockClientMockRecorder) { + spec := getDefaultVMSSSpec() + s.ScaleSetSpec(gomockinternal.AContext()).Return(spec).AnyTimes() + m.Get(gomockinternal.AContext(), &defaultSpec).Return(&resultVMSS, nil) + m.ListInstances(gomockinternal.AContext(), defaultSpec.ResourceGroup, defaultSpec.Name).Return(defaultInstances, nil) + + r.CreateOrUpdateResource(gomockinternal.AContext(), spec, serviceName). + Return(nil, internalError) + s.UpdatePutStatus(infrav1.BootstrapSucceededCondition, serviceName, internalError) }, }, { - name: "should start creating a vmss with spot vm and ephemeral disk", - expectedError: "failed to get VMSS my-vmss after create or update: failed to get result from future: operation type PUT on Azure resource my-rg/my-vmss is not done", - expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { - spec := newDefaultVMSSSpec() - spec.Size = vmSizeEPH - spec.SpotVMOptions = &infrav1.SpotVMOptions{} - spec.OSDisk.DiffDiskSettings = &infrav1.DiffDiskSettings{ - Option: string(compute.DiffDiskOptionsLocal), - } - s.ScaleSetSpec().Return(spec).AnyTimes() - setupDefaultVMSSStartCreatingExpectations(s, m) - vmss := newDefaultVMSS(vmSizeEPH) - vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.StorageProfile.OsDisk.DiffDiskSettings = &compute.DiffDiskSettings{ - Option: compute.DiffDiskOptionsLocal, - } - vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.Priority = compute.VirtualMachinePriorityTypesSpot - m.CreateOrUpdateAsync(gomockinternal.AContext(), defaultResourceGroup, defaultVMSSName, gomockinternal.DiffEq(vmss)). - Return(putFuture, nil) - setupCreatingSucceededExpectations(s, m, newDefaultExistingVMSS(vmSizeEPH), putFuture) + name: "failed to reconcile replicas", + expectedError: "unable to reconcile replicas: #: Internal Server Error: StatusCode=500", + expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder, m *mock_scalesets.MockClientMockRecorder) { + spec := getDefaultVMSSSpec() + s.ScaleSetSpec(gomockinternal.AContext()).Return(spec).AnyTimes() + m.Get(gomockinternal.AContext(), &defaultSpec).Return(&resultVMSS, nil) + m.ListInstances(gomockinternal.AContext(), defaultSpec.ResourceGroup, defaultSpec.Name).Return(defaultInstances, nil) + + r.CreateOrUpdateResource(gomockinternal.AContext(), spec, serviceName).Return(getResultVMSS(), nil) + s.UpdatePutStatus(infrav1.BootstrapSucceededCondition, serviceName, nil) + + s.ReconcileReplicas(gomockinternal.AContext(), &fetchedVMSS).Return(internalError) }, }, { - name: "should start creating a vmss with spot vm and a defined delete evictionPolicy", - expectedError: "failed to get VMSS my-vmss after create or update: failed to get result from future: operation type PUT on Azure resource my-rg/my-vmss is not done", - expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { + name: "validate spec failure: less than 2 vCPUs", + expectedError: "reconcile error that cannot be recovered occurred: vm size should be bigger or equal to at least 2 vCPUs. Object will not be requeued", + expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder, m *mock_scalesets.MockClientMockRecorder) { spec := newDefaultVMSSSpec() - spec.Size = vmSizeEPH - deletePolicy := infrav1.SpotEvictionPolicyDelete - spec.SpotVMOptions = &infrav1.SpotVMOptions{EvictionPolicy: &deletePolicy} - s.ScaleSetSpec().Return(spec).AnyTimes() - setupDefaultVMSSStartCreatingExpectations(s, m) - vmss := newDefaultVMSS(vmSizeEPH) - vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.Priority = compute.VirtualMachinePriorityTypesSpot - vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.EvictionPolicy = compute.VirtualMachineEvictionPolicyTypesDelete - m.CreateOrUpdateAsync(gomockinternal.AContext(), defaultResourceGroup, defaultVMSSName, gomockinternal.DiffEq(vmss)). - Return(putFuture, nil) - setupCreatingSucceededExpectations(s, m, newDefaultExistingVMSS(vmSizeEPH), putFuture) + spec.Size = "VM_SIZE_1_CPU" + spec.Capacity = 2 + spec.SSHKeyData = sshKeyData + s.ScaleSetSpec(gomockinternal.AContext()).Return(&spec).AnyTimes() }, }, { - name: "should start creating a vmss with spot vm and a maximum price", - expectedError: "failed to get VMSS my-vmss after create or update: failed to get result from future: operation type PUT on Azure resource my-rg/my-vmss is not done", - expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { + name: "validate spec failure: Memory is less than 2Gi", + expectedError: "reconcile error that cannot be recovered occurred: vm memory should be bigger or equal to at least 2Gi. Object will not be requeued", + expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder, m *mock_scalesets.MockClientMockRecorder) { spec := newDefaultVMSSSpec() - maxPrice := resource.MustParse("0.001") - spec.SpotVMOptions = &infrav1.SpotVMOptions{ - MaxPrice: &maxPrice, - } - spec.DataDisks = append(spec.DataDisks, infrav1.DataDisk{ - NameSuffix: "my_disk_with_ultra_disks", - DiskSizeGB: 128, - Lun: ptr.To[int32](3), - ManagedDisk: &infrav1.ManagedDiskParameters{ - StorageAccountType: "UltraSSD_LRS", - }, - }) - s.ScaleSetSpec().Return(spec).AnyTimes() - setupDefaultVMSSStartCreatingExpectations(s, m) - vmss := newDefaultVMSS("VM_SIZE") - vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.Priority = compute.VirtualMachinePriorityTypesSpot - vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.BillingProfile = &compute.BillingProfile{ - MaxPrice: ptr.To[float64](0.001), - } - vmss.VirtualMachineScaleSetProperties.AdditionalCapabilities = &compute.AdditionalCapabilities{UltraSSDEnabled: ptr.To(true)} - m.CreateOrUpdateAsync(gomockinternal.AContext(), defaultResourceGroup, defaultVMSSName, gomockinternal.DiffEq(vmss)). - Return(putFuture, nil) - setupCreatingSucceededExpectations(s, m, newDefaultExistingVMSS("VM_SIZE"), putFuture) + spec.Size = "VM_SIZE_1_MEM" + spec.Capacity = 2 + spec.SSHKeyData = sshKeyData + s.ScaleSetSpec(gomockinternal.AContext()).Return(&spec).AnyTimes() }, }, { - name: "should start creating a vmss with encryption", - expectedError: "failed to get VMSS my-vmss after create or update: failed to get result from future: operation type PUT on Azure resource my-rg/my-vmss is not done", - expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { + name: "validate spec failure: failed to get SKU", + expectedError: "failed to get SKU INVALID_VM_SIZE in compute api: reconcile error that cannot be recovered occurred: resource sku with name 'INVALID_VM_SIZE' and category 'virtualMachines' not found in location 'test-location'. Object will not be requeued", + expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder, m *mock_scalesets.MockClientMockRecorder) { spec := newDefaultVMSSSpec() - spec.OSDisk.ManagedDisk.DiskEncryptionSet = &infrav1.DiskEncryptionSetParameters{ - ID: "my-diskencryptionset-id", - } - spec.DataDisks = append(spec.DataDisks, infrav1.DataDisk{ - NameSuffix: "my_disk_with_ultra_disks", - DiskSizeGB: 128, - Lun: ptr.To[int32](3), - ManagedDisk: &infrav1.ManagedDiskParameters{ - StorageAccountType: "UltraSSD_LRS", - }, - }) - s.ScaleSetSpec().Return(spec).AnyTimes() - setupDefaultVMSSStartCreatingExpectations(s, m) - vmss := newDefaultVMSS("VM_SIZE") - vmss.VirtualMachineScaleSetProperties.AdditionalCapabilities = &compute.AdditionalCapabilities{UltraSSDEnabled: ptr.To(true)} - osdisk := vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.StorageProfile.OsDisk - osdisk.ManagedDisk = &compute.VirtualMachineScaleSetManagedDiskParameters{ - StorageAccountType: "Premium_LRS", - DiskEncryptionSet: &compute.DiskEncryptionSetParameters{ - ID: ptr.To("my-diskencryptionset-id"), - }, - } - m.CreateOrUpdateAsync(gomockinternal.AContext(), defaultResourceGroup, defaultVMSSName, gomockinternal.DiffEq(vmss)). - Return(putFuture, nil) - setupCreatingSucceededExpectations(s, m, newDefaultExistingVMSS("VM_SIZE"), putFuture) + spec.Size = "INVALID_VM_SIZE" + spec.Capacity = 2 + spec.SSHKeyData = sshKeyData + s.ScaleSetSpec(gomockinternal.AContext()).Return(&spec).AnyTimes() }, }, { - name: "can start creating a vmss with user assigned identity", - expectedError: "failed to get VMSS my-vmss after create or update: failed to get result from future: operation type PUT on Azure resource my-rg/my-vmss is not done", - expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { + name: "validate spec failure: fail to create a vm with ultra disk implicitly enabled by data disk, when location not supported", + expectedError: "reconcile error that cannot be recovered occurred: vm size VM_SIZE_USSD does not support ultra disks in location test-location. select a different vm size or disable ultra disks. Object will not be requeued", + expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder, m *mock_scalesets.MockClientMockRecorder) { spec := newDefaultVMSSSpec() + spec.Size = vmSizeUSSD + spec.Capacity = 2 + spec.SSHKeyData = sshKeyData spec.DataDisks = append(spec.DataDisks, infrav1.DataDisk{ - NameSuffix: "my_disk_with_ultra_disks", - DiskSizeGB: 128, - Lun: ptr.To[int32](3), ManagedDisk: &infrav1.ManagedDiskParameters{ StorageAccountType: "UltraSSD_LRS", }, }) - spec.Identity = infrav1.VMIdentityUserAssigned - spec.UserAssignedIdentities = []infrav1.UserAssignedIdentity{ - { - ProviderID: "azure:///subscriptions/123/resourcegroups/456/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id1", - }, - } - s.ScaleSetSpec().Return(spec).AnyTimes() - setupDefaultVMSSStartCreatingExpectations(s, m) - vmss := newDefaultVMSS("VM_SIZE") - vmss.VirtualMachineScaleSetProperties.AdditionalCapabilities = &compute.AdditionalCapabilities{UltraSSDEnabled: ptr.To(true)} - vmss.Identity = &compute.VirtualMachineScaleSetIdentity{ - Type: compute.ResourceIdentityTypeUserAssigned, - UserAssignedIdentities: map[string]*compute.VirtualMachineScaleSetIdentityUserAssignedIdentitiesValue{ - "/subscriptions/123/resourcegroups/456/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id1": {}, - }, - } - m.CreateOrUpdateAsync(gomockinternal.AContext(), defaultResourceGroup, defaultVMSSName, gomockinternal.DiffEq(vmss)). - Return(putFuture, nil) - setupCreatingSucceededExpectations(s, m, newDefaultExistingVMSS("VM_SIZE"), putFuture) + s.ScaleSetSpec(gomockinternal.AContext()).Return(&spec).AnyTimes() }, }, { - name: "should start creating a vmss with encryption at host enabled", - expectedError: "failed to get VMSS my-vmss after create or update: failed to get result from future: operation type PUT on Azure resource my-rg/my-vmss is not done", - expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { + name: "validate spec failure: fail to create a vm with ultra disk explicitly enabled via additional capabilities, when location not supported", + expectedError: "reconcile error that cannot be recovered occurred: vm size VM_SIZE_USSD does not support ultra disks in location test-location. select a different vm size or disable ultra disks. Object will not be requeued", + expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder, m *mock_scalesets.MockClientMockRecorder) { spec := newDefaultVMSSSpec() - spec.Size = "VM_SIZE_EAH" - spec.SecurityProfile = &infrav1.SecurityProfile{EncryptionAtHost: ptr.To(true)} - s.ScaleSetSpec().Return(spec).AnyTimes() - setupDefaultVMSSStartCreatingExpectations(s, m) - vmss := newDefaultVMSS("VM_SIZE_EAH") - vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.SecurityProfile = &compute.SecurityProfile{ - EncryptionAtHost: ptr.To(true), - } - vmss.Sku.Name = ptr.To(spec.Size) - m.CreateOrUpdateAsync(gomockinternal.AContext(), defaultResourceGroup, defaultVMSSName, gomockinternal.DiffEq(vmss)). - Return(putFuture, nil) - setupCreatingSucceededExpectations(s, m, newDefaultExistingVMSS("VM_SIZE_EAH"), putFuture) - }, - }, - { - name: "creating a vmss with encryption at host enabled for unsupported VM type fails", - expectedError: "reconcile error that cannot be recovered occurred: encryption at host is not supported for VM type VM_SIZE. Object will not be requeued", - expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { - s.ScaleSetSpec().Return(azure.ScaleSetSpec{ - Name: defaultVMSSName, - Size: "VM_SIZE", - Capacity: 2, - SSHKeyData: "ZmFrZXNzaGtleQo=", - SecurityProfile: &infrav1.SecurityProfile{EncryptionAtHost: ptr.To(true)}, - }) - }, - }, - { - name: "should start creating a vmss with ephemeral osdisk", - expectedError: "failed to get VMSS my-vmss after create or update: failed to get result from future: operation type PUT on Azure resource my-rg/my-vmss is not done", - expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { - defaultSpec := newDefaultVMSSSpec() - defaultSpec.Size = "VM_SIZE_EPH" - defaultSpec.OSDisk.DiffDiskSettings = &infrav1.DiffDiskSettings{ - Option: "Local", - } - defaultSpec.OSDisk.CachingType = "ReadOnly" - - s.ScaleSetSpec().Return(defaultSpec).AnyTimes() - setupDefaultVMSSStartCreatingExpectations(s, m) - vmss := newDefaultVMSS("VM_SIZE_EPH") - vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.StorageProfile.OsDisk.DiffDiskSettings = &compute.DiffDiskSettings{ - Option: compute.DiffDiskOptionsLocal, + spec.Size = vmSizeUSSD + spec.Capacity = 2 + spec.SSHKeyData = sshKeyData + spec.AdditionalCapabilities = &infrav1.AdditionalCapabilities{ + UltraSSDEnabled: ptr.To(true), } - vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.StorageProfile.OsDisk.Caching = compute.CachingTypesReadOnly - - m.CreateOrUpdateAsync(gomockinternal.AContext(), defaultResourceGroup, defaultVMSSName, gomockinternal.DiffEq(vmss)). - Return(putFuture, nil) - setupCreatingSucceededExpectations(s, m, newDefaultExistingVMSS("VM_SIZE_EPH"), putFuture) + s.ScaleSetSpec(gomockinternal.AContext()).Return(&spec).AnyTimes() }, }, { - name: "should start updating when scale set already exists and not currently in a long running operation", - expectedError: "failed to get VMSS my-vmss after create or update: failed to get result from future: operation type PATCH on Azure resource my-rg/my-vmss is not done", - expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { + name: "validate spec failure: fail to create a vm with ultra disk explicitly enabled via additional capabilities, when location not supported", + expectedError: "reconcile error that cannot be recovered occurred: vm size VM_SIZE_USSD does not support ultra disks in location test-location. select a different vm size or disable ultra disks. Object will not be requeued", + expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder, m *mock_scalesets.MockClientMockRecorder) { spec := newDefaultVMSSSpec() + spec.Size = vmSizeUSSD spec.Capacity = 2 + spec.SSHKeyData = sshKeyData spec.DataDisks = append(spec.DataDisks, infrav1.DataDisk{ - NameSuffix: "my_disk_with_ultra_disks", - DiskSizeGB: 128, - Lun: ptr.To[int32](3), ManagedDisk: &infrav1.ManagedDiskParameters{ StorageAccountType: "UltraSSD_LRS", }, }) - s.ScaleSetSpec().Return(spec).AnyTimes() - - setupDefaultVMSSUpdateExpectations(s) - existingVMSS := newDefaultExistingVMSS("VM_SIZE") - existingVMSS.VirtualMachineScaleSetProperties.AdditionalCapabilities = &compute.AdditionalCapabilities{UltraSSDEnabled: ptr.To(true)} - existingVMSS.Sku.Capacity = ptr.To[int64](2) - existingVMSS.VirtualMachineScaleSetProperties.AdditionalCapabilities = &compute.AdditionalCapabilities{UltraSSDEnabled: ptr.To(true)} - instances := newDefaultInstances() - m.Get(gomockinternal.AContext(), defaultResourceGroup, defaultVMSSName).Return(existingVMSS, nil) - m.ListInstances(gomockinternal.AContext(), defaultResourceGroup, defaultVMSSName).Return(instances, nil) - - clone := newDefaultExistingVMSS("VM_SIZE") - clone.Sku.Capacity = ptr.To[int64](3) - clone.VirtualMachineScaleSetProperties.AdditionalCapabilities = &compute.AdditionalCapabilities{UltraSSDEnabled: ptr.To(true)} - - patchVMSS, err := getVMSSUpdateFromVMSS(clone) - g.Expect(err).NotTo(HaveOccurred()) - patchVMSS.VirtualMachineProfile.StorageProfile.ImageReference.Version = ptr.To("2.0") - patchVMSS.VirtualMachineProfile.NetworkProfile = nil - m.UpdateAsync(gomockinternal.AContext(), defaultResourceGroup, defaultVMSSName, gomockinternal.DiffEq(patchVMSS)). - Return(patchFuture, nil) - s.SetLongRunningOperationState(patchFuture) - m.GetResultIfDone(gomockinternal.AContext(), patchFuture).Return(compute.VirtualMachineScaleSet{}, azure.NewOperationNotDoneError(patchFuture)) - m.Get(gomockinternal.AContext(), defaultResourceGroup, defaultVMSSName).Return(clone, nil) - m.ListInstances(gomockinternal.AContext(), defaultResourceGroup, defaultVMSSName).Return(instances, nil) - s.HasReplicasExternallyManaged(gomockinternal.AContext()).Times(2).Return(false) - }, - }, - { - name: "less than 2 vCPUs", - expectedError: "reconcile error that cannot be recovered occurred: vm size should be bigger or equal to at least 2 vCPUs. Object will not be requeued", - expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { - s.ScaleSetSpec().Return(azure.ScaleSetSpec{ - Name: defaultVMSSName, - Size: "VM_SIZE_1_CPU", - Capacity: 2, - SSHKeyData: "ZmFrZXNzaGtleQo=", - }) - }, - }, - { - name: "Memory is less than 2Gi", - expectedError: "reconcile error that cannot be recovered occurred: vm memory should be bigger or equal to at least 2Gi. Object will not be requeued", - expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { - s.ScaleSetSpec().Return(azure.ScaleSetSpec{ - Name: defaultVMSSName, - Size: "VM_SIZE_1_MEM", - Capacity: 2, - SSHKeyData: "ZmFrZXNzaGtleQo=", - }) - }, - }, - { - name: "failed to get SKU", - expectedError: "failed to get SKU INVALID_VM_SIZE in compute api: reconcile error that cannot be recovered occurred: resource sku with name 'INVALID_VM_SIZE' and category 'virtualMachines' not found in location 'test-location'. Object will not be requeued", - expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { - s.ScaleSetSpec().Return(azure.ScaleSetSpec{ - Name: defaultVMSSName, - Size: "INVALID_VM_SIZE", - Capacity: 2, - SSHKeyData: "ZmFrZXNzaGtleQo=", - }) - }, - }, - { - name: "fails with internal error", - expectedError: "failed to start creating VMSS: cannot create VMSS: #: Internal error: StatusCode=500", - expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { - spec := newDefaultVMSSSpec() - s.ScaleSetSpec().Return(spec).AnyTimes() - setupDefaultVMSSStartCreatingExpectations(s, m) - m.CreateOrUpdateAsync(gomockinternal.AContext(), defaultResourceGroup, defaultVMSSName, gomock.AssignableToTypeOf(compute.VirtualMachineScaleSet{})). - Return(nil, autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: http.StatusInternalServerError}, "Internal error")) - m.Get(gomockinternal.AContext(), defaultResourceGroup, defaultVMSSName). - Return(compute.VirtualMachineScaleSet{}, autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: http.StatusNotFound}, "Not found")) - }, - }, - { - name: "fail to create a vm with ultra disk implicitly enabled by data disk, when location not supported", - expectedError: "reconcile error that cannot be recovered occurred: vm size VM_SIZE_USSD does not support ultra disks in location test-location. select a different vm size or disable ultra disks. Object will not be requeued", - expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { - s.ScaleSetSpec().Return(azure.ScaleSetSpec{ - Name: defaultVMSSName, - Size: "VM_SIZE_USSD", - Capacity: 2, - SSHKeyData: "ZmFrZXNzaGtleQo=", - DataDisks: []infrav1.DataDisk{ - { - ManagedDisk: &infrav1.ManagedDiskParameters{ - StorageAccountType: "UltraSSD_LRS", - }, - }, - }, - }) - s.Location().AnyTimes().Return("test-location") - }, - }, - { - name: "fail to create a vm with ultra disk explicitly enabled via additional capabilities, when location not supported", - expectedError: "reconcile error that cannot be recovered occurred: vm size VM_SIZE_USSD does not support ultra disks in location test-location. select a different vm size or disable ultra disks. Object will not be requeued", - expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { - s.ScaleSetSpec().Return(azure.ScaleSetSpec{ - Name: defaultVMSSName, - Size: "VM_SIZE_USSD", - Capacity: 2, - SSHKeyData: "ZmFrZXNzaGtleQo=", - AdditionalCapabilities: &infrav1.AdditionalCapabilities{ - UltraSSDEnabled: ptr.To(true), - }, - }) - s.Location().AnyTimes().Return("test-location") - }, - }, - { - name: "fail to create a vm with ultra disk explicitly enabled via additional capabilities, when location not supported", - expectedError: "reconcile error that cannot be recovered occurred: vm size VM_SIZE_USSD does not support ultra disks in location test-location. select a different vm size or disable ultra disks. Object will not be requeued", - expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { - s.ScaleSetSpec().Return(azure.ScaleSetSpec{ - Name: defaultVMSSName, - Size: "VM_SIZE_USSD", - Capacity: 2, - SSHKeyData: "ZmFrZXNzaGtleQo=", - DataDisks: []infrav1.DataDisk{ - { - ManagedDisk: &infrav1.ManagedDiskParameters{ - StorageAccountType: "UltraSSD_LRS", - }, - }, - }, - AdditionalCapabilities: &infrav1.AdditionalCapabilities{ - UltraSSDEnabled: ptr.To(false), - }, - }) - s.Location().AnyTimes().Return("test-location") + spec.AdditionalCapabilities = &infrav1.AdditionalCapabilities{ + UltraSSDEnabled: ptr.To(true), + } + s.ScaleSetSpec(gomockinternal.AContext()).Return(&spec).AnyTimes() }, }, { - name: "fail to create a vm with diagnostics set to User Managed but empty StorageAccountURI", + name: "validate spec failure: fail to create a vm with diagnostics set to User Managed but empty StorageAccountURI", expectedError: "reconcile error that cannot be recovered occurred: userManaged must be specified when storageAccountType is 'UserManaged'. Object will not be requeued", - expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { - s.ScaleSetSpec().Return(azure.ScaleSetSpec{ - Name: defaultVMSSName, - Size: "VM_SIZE", - Capacity: 2, - SSHKeyData: "ZmFrZXNzaGtleQo=", - DiagnosticsProfile: &infrav1.Diagnostics{ - Boot: &infrav1.BootDiagnostics{ - StorageAccountType: infrav1.UserManagedDiagnosticsStorage, - UserManaged: nil, - }, - }, - }) - s.Location().AnyTimes().Return("test-location") - }, - }, - { - name: "successfully create a vm with diagnostics set to User Managed and StorageAccountURI set", - expectedError: "", - expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { - storageURI := "https://fakeurl" - + expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder, m *mock_scalesets.MockClientMockRecorder) { spec := newDefaultVMSSSpec() + spec.Size = vmSizeUSSD + spec.Capacity = 2 + spec.SSHKeyData = sshKeyData spec.DiagnosticsProfile = &infrav1.Diagnostics{ Boot: &infrav1.BootDiagnostics{ StorageAccountType: infrav1.UserManagedDiagnosticsStorage, - UserManaged: &infrav1.UserManagedBootDiagnostics{ - StorageAccountURI: storageURI, - }, - }, - } - s.ScaleSetSpec().Return(spec).AnyTimes() - - vmss := newDefaultVMSS("VM_SIZE") - vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.DiagnosticsProfile = &compute.DiagnosticsProfile{BootDiagnostics: &compute.BootDiagnostics{ - Enabled: ptr.To(true), - StorageURI: &storageURI, - }} - - instances := newDefaultInstances() - - setupDefaultVMSSInProgressOperationDoneExpectations(s, m, vmss, instances) - s.DeleteLongRunningOperationState(spec.Name, serviceName, infrav1.PutFuture) - s.DeleteLongRunningOperationState(spec.Name, serviceName, infrav1.PatchFuture) - s.UpdatePutStatus(infrav1.BootstrapSucceededCondition, serviceName, nil) - s.Location().AnyTimes().Return("test-location") - s.HasReplicasExternallyManaged(gomockinternal.AContext()).Return(false) - }, - }, - { - name: "successfully create a vm with diagnostics set to Managed", - expectedError: "", - expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { - spec := newDefaultVMSSSpec() - spec.DiagnosticsProfile = &infrav1.Diagnostics{ - Boot: &infrav1.BootDiagnostics{ - StorageAccountType: infrav1.ManagedDiagnosticsStorage, + UserManaged: nil, }, } - - s.ScaleSetSpec().Return(spec).AnyTimes() - vmss := newDefaultVMSS("VM_SIZE") - vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.DiagnosticsProfile = &compute.DiagnosticsProfile{BootDiagnostics: &compute.BootDiagnostics{ - Enabled: ptr.To(true), - }} - - instances := newDefaultInstances() - - setupDefaultVMSSInProgressOperationDoneExpectations(s, m, vmss, instances) - s.DeleteLongRunningOperationState(spec.Name, serviceName, infrav1.PutFuture) - s.DeleteLongRunningOperationState(spec.Name, serviceName, infrav1.PatchFuture) - s.UpdatePutStatus(infrav1.BootstrapSucceededCondition, serviceName, nil) - s.Location().AnyTimes().Return("test-location") - s.HasReplicasExternallyManaged(gomockinternal.AContext()).Return(false) - }, - }, - { - name: "successfully create a vm with diagnostics set to Disabled", - expectedError: "", - expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { - spec := newDefaultVMSSSpec() - spec.DiagnosticsProfile = &infrav1.Diagnostics{ - Boot: &infrav1.BootDiagnostics{ - StorageAccountType: infrav1.DisabledDiagnosticsStorage, - }, - } - s.ScaleSetSpec().Return(spec).AnyTimes() - - vmss := newDefaultVMSS("VM_SIZE") - vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.DiagnosticsProfile = &compute.DiagnosticsProfile{BootDiagnostics: &compute.BootDiagnostics{ - Enabled: ptr.To(false), - }} - instances := newDefaultInstances() - - setupDefaultVMSSInProgressOperationDoneExpectations(s, m, vmss, instances) - s.DeleteLongRunningOperationState(spec.Name, serviceName, infrav1.PutFuture) - s.DeleteLongRunningOperationState(spec.Name, serviceName, infrav1.PatchFuture) - s.UpdatePutStatus(infrav1.BootstrapSucceededCondition, serviceName, nil) - s.Location().AnyTimes().Return("test-location") - s.HasReplicasExternallyManaged(gomockinternal.AContext()).Return(false) - }, - }, - { - name: "should not panic when DiagnosticsProfile is nil", - expectedError: "", - expect: func(g *WithT, s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { - spec := newDefaultVMSSSpec() - spec.DiagnosticsProfile = nil - s.ScaleSetSpec().Return(spec).AnyTimes() - - vmss := newDefaultVMSS("VM_SIZE") - vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.DiagnosticsProfile = nil - - instances := newDefaultInstances() - - setupDefaultVMSSInProgressOperationDoneExpectations(s, m, vmss, instances) - s.DeleteLongRunningOperationState(spec.Name, serviceName, infrav1.PutFuture) - s.DeleteLongRunningOperationState(spec.Name, serviceName, infrav1.PatchFuture) - s.UpdatePutStatus(infrav1.BootstrapSucceededCondition, serviceName, nil) - s.Location().AnyTimes().Return("test-location") - s.HasReplicasExternallyManaged(gomockinternal.AContext()).Return(false) + s.ScaleSetSpec(gomockinternal.AContext()).Return(&spec).AnyTimes() }, }, } @@ -866,12 +294,14 @@ func TestReconcileVMSS(t *testing.T) { defer mockCtrl.Finish() scopeMock := mock_scalesets.NewMockScaleSetScope(mockCtrl) + asyncMock := mock_async.NewMockReconciler(mockCtrl) clientMock := mock_scalesets.NewMockClient(mockCtrl) - tc.expect(g, scopeMock.EXPECT(), clientMock.EXPECT()) + tc.expect(g, scopeMock.EXPECT(), asyncMock.EXPECT(), clientMock.EXPECT()) s := &Service{ Scope: scopeMock, + Reconciler: asyncMock, Client: clientMock, resourceSKUCache: resourceskus.NewStaticCache(getFakeSkus(), "test-location"), } @@ -888,69 +318,49 @@ func TestReconcileVMSS(t *testing.T) { } func TestDeleteVMSS(t *testing.T) { - const ( - resourceGroup = "my-rg" - name = "my-vmss" - ) + defaultSpec := newDefaultVMSSSpec() + defaultInstances := newDefaultInstances() + resultVMSS := newDefaultVMSS("VM_SIZE") + resultVMSS.ID = ptr.To(defaultVMSSID) + fetchedVMSS := converters.SDKToVMSS(getResultVMSS(), defaultInstances) + // Be careful about race conditions if you need modify these. testcases := []struct { name string expectedError string - expect func(s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) + expect func(s *mock_scalesets.MockScaleSetScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder, m *mock_scalesets.MockClientMockRecorder) }{ { name: "successfully delete an existing vmss", expectedError: "", - expect: func(s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { - s.ScaleSetSpec().Return(azure.ScaleSetSpec{ - Name: "my-existing-vmss", - Size: "VM_SIZE", - Capacity: 3, - }).AnyTimes() - s.ResourceGroup().AnyTimes().Return("my-existing-rg") - future := &infrav1.Future{} - s.GetLongRunningOperationState("my-existing-vmss", serviceName, infrav1.DeleteFuture).Return(future) - m.GetResultIfDone(gomockinternal.AContext(), future).Return(compute.VirtualMachineScaleSet{}, nil) - m.Get(gomockinternal.AContext(), "my-existing-rg", "my-existing-vmss"). - Return(compute.VirtualMachineScaleSet{}, autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: http.StatusNotFound}, "Not found")) - s.DeleteLongRunningOperationState("my-existing-vmss", serviceName, infrav1.DeleteFuture) + expect: func(s *mock_scalesets.MockScaleSetScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder, m *mock_scalesets.MockClientMockRecorder) { + s.ScaleSetSpec(gomockinternal.AContext()).Return(&defaultSpec).AnyTimes() + r.DeleteResource(gomockinternal.AContext(), &defaultSpec, serviceName).Return(nil) s.UpdateDeleteStatus(infrav1.BootstrapSucceededCondition, serviceName, nil) + + m.Get(gomockinternal.AContext(), &defaultSpec).Return(resultVMSS, nil) + m.ListInstances(gomockinternal.AContext(), defaultSpec.ResourceGroup, defaultSpec.Name).Return(defaultInstances, nil) + s.SetVMSSState(&fetchedVMSS) }, }, { - name: "vmss already deleted", + name: "successfully delete an existing vmss, fetch call returns error", expectedError: "", - expect: func(s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { - s.ScaleSetSpec().Return(azure.ScaleSetSpec{ - Name: name, - Size: "VM_SIZE", - Capacity: 3, - }).AnyTimes() - s.ResourceGroup().AnyTimes().Return(resourceGroup) - s.GetLongRunningOperationState(name, serviceName, infrav1.DeleteFuture).Return(nil) - m.DeleteAsync(gomockinternal.AContext(), resourceGroup, name). - Return(nil, autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: http.StatusNotFound}, "Not found")) - m.Get(gomockinternal.AContext(), resourceGroup, name). - Return(compute.VirtualMachineScaleSet{}, autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: http.StatusNotFound}, "Not found")) + expect: func(s *mock_scalesets.MockScaleSetScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder, m *mock_scalesets.MockClientMockRecorder) { + s.ScaleSetSpec(gomockinternal.AContext()).Return(&defaultSpec).AnyTimes() + r.DeleteResource(gomockinternal.AContext(), &defaultSpec, serviceName).Return(nil) + s.UpdateDeleteStatus(infrav1.BootstrapSucceededCondition, serviceName, nil) + m.Get(gomockinternal.AContext(), &defaultSpec).Return(compute.VirtualMachineScaleSet{}, notFoundError) }, }, { - name: "vmss deletion fails", - expectedError: "failed to delete VMSS my-vmss in resource group my-rg: #: Internal Server Error: StatusCode=500", - expect: func(s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { - s.ScaleSetSpec().Return(azure.ScaleSetSpec{ - Name: name, - Size: "VM_SIZE", - Capacity: 3, - }).AnyTimes() - s.ResourceGroup().AnyTimes().Return(resourceGroup) - s.GetLongRunningOperationState(name, serviceName, infrav1.DeleteFuture).Return(nil) - m.DeleteAsync(gomockinternal.AContext(), resourceGroup, name). - Return(nil, autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: http.StatusInternalServerError}, "Internal Server Error")) - m.Get(gomockinternal.AContext(), resourceGroup, name). - Return(newDefaultVMSS("VM_SIZE"), nil) - m.ListInstances(gomockinternal.AContext(), defaultResourceGroup, defaultVMSSName).Return(newDefaultInstances(), nil).AnyTimes() - s.SetVMSSState(gomock.AssignableToTypeOf(&azure.VMSS{})) + name: "failed to delete an existing vmss", + expectedError: "#: Internal Server Error: StatusCode=500", + expect: func(s *mock_scalesets.MockScaleSetScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder, m *mock_scalesets.MockClientMockRecorder) { + s.ScaleSetSpec(gomockinternal.AContext()).Return(&defaultSpec).AnyTimes() + r.DeleteResource(gomockinternal.AContext(), &defaultSpec, serviceName).Return(internalError) + s.UpdateDeleteStatus(infrav1.BootstrapSucceededCondition, serviceName, internalError) + m.Get(gomockinternal.AContext(), &defaultSpec).Return(compute.VirtualMachineScaleSet{}, notFoundError) }, }, } @@ -963,13 +373,15 @@ func TestDeleteVMSS(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() scopeMock := mock_scalesets.NewMockScaleSetScope(mockCtrl) - clientMock := mock_scalesets.NewMockClient(mockCtrl) + asyncMock := mock_async.NewMockReconciler(mockCtrl) + mockClient := mock_scalesets.NewMockClient(mockCtrl) - tc.expect(scopeMock.EXPECT(), clientMock.EXPECT()) + tc.expect(scopeMock.EXPECT(), asyncMock.EXPECT(), mockClient.EXPECT()) s := &Service{ - Scope: scopeMock, - Client: clientMock, + Scope: scopeMock, + Reconciler: asyncMock, + Client: mockClient, } err := s.Delete(context.TODO()) @@ -1035,16 +447,6 @@ func getFakeSkus() []compute.ResourceSku { { Location: ptr.To("test-location"), Zones: &[]string{"1", "3"}, - // ZoneDetails: &[]compute.ResourceSkuZoneDetails{ - // { - // Capabilities: &[]compute.ResourceSkuCapabilities{ - // { - // Name: ptr.To("UltraSSDAvailable"), - // Value: ptr.To("True"), - // }, - // }, - // }, - // }, }, }, Capabilities: &[]compute.ResourceSkuCapabilities{ @@ -1129,16 +531,6 @@ func getFakeSkus() []compute.ResourceSku { { Location: ptr.To("test-location"), Zones: &[]string{"1", "3"}, - // ZoneDetails: &[]compute.ResourceSkuZoneDetails{ - // { - // Capabilities: &[]compute.ResourceSkuCapabilities{ - // { - // Name: ptr.To("UltraSSDAvailable"), - // Value: ptr.To("True"), - // }, - // }, - // }, - // }, }, }, Capabilities: &[]compute.ResourceSkuCapabilities{ @@ -1157,7 +549,7 @@ func getFakeSkus() []compute.ResourceSku { }, }, { - Name: ptr.To("VM_SIZE_USSD"), + Name: ptr.To(vmSizeUSSD), ResourceType: ptr.To(string(resourceskus.VirtualMachines)), Kind: ptr.To(string(resourceskus.VirtualMachines)), Locations: &[]string{ @@ -1230,12 +622,12 @@ func getFakeSkus() []compute.ResourceSku { } } -func newDefaultVMSSSpec() azure.ScaleSetSpec { - return azure.ScaleSetSpec{ +func newDefaultVMSSSpec() ScaleSetSpec { + return ScaleSetSpec{ Name: defaultVMSSName, Size: "VM_SIZE", Capacity: 2, - SSHKeyData: "ZmFrZXNzaGtleQo=", + SSHKeyData: sshKeyData, OSDisk: infrav1.OSDisk{ OSType: "Linux", DiskSizeGB: ptr.To[int32](120), @@ -1288,10 +680,34 @@ func newDefaultVMSSSpec() azure.ScaleSetSpec { PrivateIPConfigs: 1, }, }, + SubscriptionID: "123", + ResourceGroup: "my-rg", + Location: "test-location", + ClusterName: "my-cluster", + VMSSInstances: newDefaultInstances(), + VMImage: &defaultImage, + BootstrapData: "fake-bootstrap-data", + VMSSExtensionSpecs: []azure.ResourceSpecGetter{ + &VMSSExtensionSpec{ + ExtensionSpec: azure.ExtensionSpec{ + Name: "someExtension", + VMName: "my-vmss", + Publisher: "somePublisher", + Version: "someVersion", + Settings: map[string]string{ + "someSetting": "someValue", + }, + ProtectedSettings: map[string]string{ + "commandToExecute": "echo hello", + }, + }, + ResourceGroup: "my-rg", + }, + }, } } -func newWindowsVMSSSpec() azure.ScaleSetSpec { +func newWindowsVMSSSpec() ScaleSetSpec { vmss := newDefaultVMSSSpec() vmss.OSDisk.OSType = azure.WindowsOS return vmss @@ -1424,6 +840,7 @@ func newDefaultVMSS(vmSize string) compute.VirtualMachineScaleSet { }, }, }, + // AdditionalCapabilities: &compute.AdditionalCapabilities{UltraSSDEnabled: ptr.To(true)}, }, } } @@ -1545,106 +962,3 @@ func newDefaultInstances() []compute.VirtualMachineScaleSetVM { }, } } - -func setupDefaultVMSSInProgressOperationDoneExpectations(s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder, createdVMSS compute.VirtualMachineScaleSet, instances []compute.VirtualMachineScaleSetVM) { - createdVMSS.ID = ptr.To("subscriptions/1234/resourceGroups/my_resource_group/providers/Microsoft.Compute/virtualMachines/my-vm") - createdVMSS.ProvisioningState = ptr.To(string(infrav1.Succeeded)) - setupDefaultVMSSExpectations(s) - future := &infrav1.Future{ - Type: infrav1.PutFuture, - ResourceGroup: defaultResourceGroup, - Name: defaultVMSSName, - Data: "", - } - s.GetLongRunningOperationState(defaultVMSSName, serviceName, infrav1.PutFuture).Return(future) - m.GetResultIfDone(gomockinternal.AContext(), future).Return(createdVMSS, nil).AnyTimes() - m.ListInstances(gomockinternal.AContext(), defaultResourceGroup, defaultVMSSName).Return(instances, nil).AnyTimes() - s.MaxSurge().Return(1, nil) - s.SetVMSSState(gomock.Any()) - s.SetProviderID(azureutil.ProviderIDPrefix + *createdVMSS.ID) -} - -func setupDefaultVMSSStartCreatingExpectations(s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder) { - setupDefaultVMSSExpectations(s) - s.GetLongRunningOperationState(defaultVMSSName, serviceName, infrav1.PutFuture).Return(nil) - s.GetLongRunningOperationState(defaultVMSSName, serviceName, infrav1.PatchFuture).Return(nil) - m.Get(gomockinternal.AContext(), defaultResourceGroup, defaultVMSSName). - Return(compute.VirtualMachineScaleSet{}, autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: http.StatusNotFound}, "Not found")) -} - -func setupCreatingSucceededExpectations(s *mock_scalesets.MockScaleSetScopeMockRecorder, m *mock_scalesets.MockClientMockRecorder, vmss compute.VirtualMachineScaleSet, future *infrav1.Future) { - s.SetLongRunningOperationState(future) - m.GetResultIfDone(gomockinternal.AContext(), future).Return(compute.VirtualMachineScaleSet{}, azure.NewOperationNotDoneError(future)) - m.Get(gomockinternal.AContext(), defaultResourceGroup, defaultVMSSName).Return(vmss, nil) - m.ListInstances(gomockinternal.AContext(), defaultResourceGroup, defaultVMSSName).Return(newDefaultInstances(), nil).AnyTimes() - s.SetVMSSState(gomock.Any()) - s.SetProviderID(azureutil.ProviderIDPrefix + *vmss.ID) -} - -func setupDefaultVMSSExpectations(s *mock_scalesets.MockScaleSetScopeMockRecorder) { - setupVMSSExpectationsWithoutVMImage(s) - image := &infrav1.Image{ - Marketplace: &infrav1.AzureMarketplaceImage{ - ImagePlan: infrav1.ImagePlan{ - Publisher: "fake-publisher", - Offer: "my-offer", - SKU: "sku-id", - }, - Version: "1.0", - }, - } - s.GetVMImage(gomockinternal.AContext()).Return(image, nil).AnyTimes() - s.SaveVMImageToStatus(image) -} - -func setupUpdateVMSSExpectations(s *mock_scalesets.MockScaleSetScopeMockRecorder) { - setupVMSSExpectationsWithoutVMImage(s) - image := &infrav1.Image{ - Marketplace: &infrav1.AzureMarketplaceImage{ - ImagePlan: infrav1.ImagePlan{ - Publisher: "fake-publisher", - Offer: "my-offer", - SKU: "sku-id", - }, - Version: "2.0", - }, - } - s.GetVMImage(gomockinternal.AContext()).Return(image, nil).AnyTimes() - s.SaveVMImageToStatus(image) -} - -func setupVMSSExpectationsWithoutVMImage(s *mock_scalesets.MockScaleSetScopeMockRecorder) { - s.SubscriptionID().AnyTimes().Return(defaultSubscriptionID) - s.ResourceGroup().AnyTimes().Return(defaultResourceGroup) - s.AdditionalTags() - s.Location().AnyTimes().Return("test-location") - s.ClusterName().Return("my-cluster") - s.GetBootstrapData(gomockinternal.AContext()).Return("fake-bootstrap-data", nil) - s.VMSSExtensionSpecs().Return([]azure.ResourceSpecGetter{ - &VMSSExtensionSpec{ - ExtensionSpec: azure.ExtensionSpec{ - Name: "someExtension", - VMName: "my-vmss", - Publisher: "somePublisher", - Version: "someVersion", - Settings: map[string]string{ - "someSetting": "someValue", - }, - ProtectedSettings: map[string]string{ - "commandToExecute": "echo hello", - }, - }, - ResourceGroup: "my-rg", - }, - }).AnyTimes() - s.ReconcileReplicas(gomock.Any(), gomock.Any()).AnyTimes() -} - -func setupDefaultVMSSUpdateExpectations(s *mock_scalesets.MockScaleSetScopeMockRecorder) { - setupUpdateVMSSExpectations(s) - s.SetProviderID(azureutil.ProviderIDPrefix + "subscriptions/1234/resourceGroups/my_resource_group/providers/Microsoft.Compute/virtualMachines/my-vm") - s.GetLongRunningOperationState(defaultVMSSName, serviceName, infrav1.PutFuture).Return(nil) - s.GetLongRunningOperationState(defaultVMSSName, serviceName, infrav1.PatchFuture).Return(nil) - s.MaxSurge().Return(1, nil) - s.SetVMSSState(gomock.Any()) -} diff --git a/azure/services/scalesets/spec.go b/azure/services/scalesets/spec.go new file mode 100644 index 00000000000..a2f94ed1dc6 --- /dev/null +++ b/azure/services/scalesets/spec.go @@ -0,0 +1,523 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scalesets + +import ( + "context" + "encoding/base64" + "fmt" + "strconv" + + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute" + "github.com/pkg/errors" + "k8s.io/utils/ptr" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-azure/azure" + "sigs.k8s.io/cluster-api-provider-azure/azure/converters" + "sigs.k8s.io/cluster-api-provider-azure/azure/services/resourceskus" + "sigs.k8s.io/cluster-api-provider-azure/util/generators" + "sigs.k8s.io/cluster-api-provider-azure/util/tele" +) + +// ScaleSetSpec defines the specification for a Scale Set. +type ScaleSetSpec struct { + Name string + ResourceGroup string + Size string + Capacity int64 + SSHKeyData string + OSDisk infrav1.OSDisk + DataDisks []infrav1.DataDisk + SubnetName string + VNetName string + VNetResourceGroup string + PublicLBName string + PublicLBAddressPoolName string + AcceleratedNetworking *bool + TerminateNotificationTimeout *int + Identity infrav1.VMIdentity + UserAssignedIdentities []infrav1.UserAssignedIdentity + SecurityProfile *infrav1.SecurityProfile + SpotVMOptions *infrav1.SpotVMOptions + AdditionalCapabilities *infrav1.AdditionalCapabilities + DiagnosticsProfile *infrav1.Diagnostics + FailureDomains []string + VMExtensions []infrav1.VMExtension + NetworkInterfaces []infrav1.NetworkInterface + IPv6Enabled bool + OrchestrationMode infrav1.OrchestrationModeType + Location string + SubscriptionID string + SKU resourceskus.SKU + VMSSExtensionSpecs []azure.ResourceSpecGetter + VMImage *infrav1.Image + BootstrapData string + VMSSInstances []compute.VirtualMachineScaleSetVM + MaxSurge int + ClusterName string + ShouldPatchCustomData bool + HasReplicasExternallyManaged bool + AdditionalTags infrav1.Tags +} + +// ResourceName returns the name of the Scale Set. +func (s *ScaleSetSpec) ResourceName() string { + return s.Name +} + +// ResourceGroupName returns the name of the resource group for this Scale Set. +func (s *ScaleSetSpec) ResourceGroupName() string { + return s.ResourceGroup +} + +// OwnerResourceName is a no-op for Scale Sets. +func (s *ScaleSetSpec) OwnerResourceName() string { + return "" +} + +func (s *ScaleSetSpec) existingParameters(ctx context.Context, existing interface{}) (parameters interface{}, err error) { + existingVMSS, ok := existing.(compute.VirtualMachineScaleSet) + if !ok { + return nil, errors.Errorf("%T is not a compute.VirtualMachineScaleSet", existing) + } + + existingInfraVMSS := converters.SDKToVMSS(existingVMSS, s.VMSSInstances) + + params, err := s.Parameters(ctx, nil) + if err != nil { + return nil, errors.Wrapf(err, "failed to generate scale set update parameters for %s", s.Name) + } + + vmss, ok := params.(compute.VirtualMachineScaleSet) + if !ok { + return nil, errors.Errorf("%T is not a compute.VirtualMachineScaleSet", existing) + } + + vmss.VirtualMachineProfile.NetworkProfile = nil + vmss.ID = existingVMSS.ID + + hasModelChanges := hasModelModifyingDifferences(&existingInfraVMSS, vmss) + isFlex := s.OrchestrationMode == infrav1.FlexibleOrchestrationMode + updated := true + if !isFlex { + updated = existingInfraVMSS.HasEnoughLatestModelOrNotMixedModel() + } + if s.MaxSurge > 0 && (hasModelChanges || !updated) && !s.HasReplicasExternallyManaged { + // surge capacity with the intention of lowering during instance reconciliation + surge := s.Capacity + int64(s.MaxSurge) + vmss.Sku.Capacity = ptr.To[int64](surge) + } + + // If there are no model changes and no increase in the replica count, do not update the VMSS. + // Decreases in replica count is handled by deleting AzureMachinePoolMachine instances in the MachinePoolScope + if *vmss.Sku.Capacity <= existingInfraVMSS.Capacity && !hasModelChanges && !s.ShouldPatchCustomData { + // up to date, nothing to do + return nil, nil + } + + return vmss, nil +} + +// Parameters returns the parameters for the Scale Set. +func (s *ScaleSetSpec) Parameters(ctx context.Context, existing interface{}) (parameters interface{}, err error) { + if existing != nil { + return s.existingParameters(ctx, existing) + } + + if s.AcceleratedNetworking == nil { + // set accelerated networking to the capability of the VMSize + accelNet := s.SKU.HasCapability(resourceskus.AcceleratedNetworking) + s.AcceleratedNetworking = &accelNet + } + + extensions, err := s.generateExtensions(ctx) + if err != nil { + return compute.VirtualMachineScaleSet{}, err + } + + storageProfile, err := s.generateStorageProfile(ctx) + if err != nil { + return compute.VirtualMachineScaleSet{}, err + } + + securityProfile, err := s.getSecurityProfile() + if err != nil { + return compute.VirtualMachineScaleSet{}, err + } + + priority, evictionPolicy, billingProfile, err := converters.GetSpotVMOptions(s.SpotVMOptions, s.OSDisk.DiffDiskSettings) + if err != nil { + return compute.VirtualMachineScaleSet{}, errors.Wrapf(err, "failed to get Spot VM options") + } + + diagnosticsProfile := converters.GetDiagnosticsProfile(s.DiagnosticsProfile) + + osProfile, err := s.generateOSProfile(ctx) + if err != nil { + return compute.VirtualMachineScaleSet{}, err + } + + orchestrationMode := converters.GetOrchestrationMode(s.OrchestrationMode) + + vmss := compute.VirtualMachineScaleSet{ + Location: ptr.To(s.Location), + Sku: &compute.Sku{ + Name: ptr.To(s.Size), + Tier: ptr.To("Standard"), + Capacity: ptr.To[int64](s.Capacity), + }, + Zones: &s.FailureDomains, + Plan: s.generateImagePlan(ctx), + VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{ + OrchestrationMode: orchestrationMode, + SinglePlacementGroup: ptr.To(false), + VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{ + OsProfile: osProfile, + StorageProfile: storageProfile, + SecurityProfile: securityProfile, + DiagnosticsProfile: diagnosticsProfile, + NetworkProfile: &compute.VirtualMachineScaleSetNetworkProfile{ + NetworkInterfaceConfigurations: s.getVirtualMachineScaleSetNetworkConfiguration(), + }, + Priority: priority, + EvictionPolicy: evictionPolicy, + BillingProfile: billingProfile, + ExtensionProfile: &compute.VirtualMachineScaleSetExtensionProfile{ + Extensions: &extensions, + }, + }, + }, + } + + // Set properties specific to VMSS orchestration mode + // See https://learn.microsoft.com/en-us/azure/virtual-machine-scale-sets/virtual-machine-scale-sets-orchestration-modes for more details + switch orchestrationMode { + case compute.OrchestrationModeUniform: // Uniform VMSS + vmss.VirtualMachineScaleSetProperties.Overprovision = ptr.To(false) + vmss.VirtualMachineScaleSetProperties.UpgradePolicy = &compute.UpgradePolicy{Mode: compute.UpgradeModeManual} + case compute.OrchestrationModeFlexible: // VMSS Flex, VMs are treated as individual virtual machines + vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.NetworkProfile.NetworkAPIVersion = + compute.NetworkAPIVersionTwoZeroTwoZeroHyphenMinusOneOneHyphenMinusZeroOne + vmss.VirtualMachineScaleSetProperties.PlatformFaultDomainCount = ptr.To[int32](1) + if len(s.FailureDomains) > 1 { + vmss.VirtualMachineScaleSetProperties.PlatformFaultDomainCount = ptr.To[int32](int32(len(s.FailureDomains))) + } + } + + // Assign Identity to VMSS + if s.Identity == infrav1.VMIdentitySystemAssigned { + vmss.Identity = &compute.VirtualMachineScaleSetIdentity{ + Type: compute.ResourceIdentityTypeSystemAssigned, + } + } else if s.Identity == infrav1.VMIdentityUserAssigned { + userIdentitiesMap, err := converters.UserAssignedIdentitiesToVMSSSDK(s.UserAssignedIdentities) + if err != nil { + return vmss, errors.Wrapf(err, "failed to assign identity %q", s.Name) + } + vmss.Identity = &compute.VirtualMachineScaleSetIdentity{ + Type: compute.ResourceIdentityTypeUserAssigned, + UserAssignedIdentities: userIdentitiesMap, + } + } + + // Provisionally detect whether there is any Data Disk defined which uses UltraSSDs. + // If that's the case, enable the UltraSSD capability. + for _, dataDisk := range s.DataDisks { + if dataDisk.ManagedDisk != nil && dataDisk.ManagedDisk.StorageAccountType == string(compute.StorageAccountTypesUltraSSDLRS) { + vmss.VirtualMachineScaleSetProperties.AdditionalCapabilities = &compute.AdditionalCapabilities{ + UltraSSDEnabled: ptr.To(true), + } + } + } + + // Set Additional Capabilities if any is present on the spec. + if s.AdditionalCapabilities != nil { + // Set UltraSSDEnabled if a specific value is set on the spec for it. + if s.AdditionalCapabilities.UltraSSDEnabled != nil { + vmss.AdditionalCapabilities.UltraSSDEnabled = s.AdditionalCapabilities.UltraSSDEnabled + } + } + + if s.TerminateNotificationTimeout != nil { + vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.ScheduledEventsProfile = &compute.ScheduledEventsProfile{ + TerminateNotificationProfile: &compute.TerminateNotificationProfile{ + NotBeforeTimeout: ptr.To(fmt.Sprintf("PT%dM", *s.TerminateNotificationTimeout)), + Enable: ptr.To(true), + }, + } + } + + tags := infrav1.Build(infrav1.BuildParams{ + ClusterName: s.ClusterName, + Lifecycle: infrav1.ResourceLifecycleOwned, + Name: ptr.To(s.Name), + Role: ptr.To(infrav1.Node), + Additional: s.AdditionalTags, + }) + + vmss.Tags = converters.TagsToMap(tags) + return vmss, nil +} + +func hasModelModifyingDifferences(infraVMSS *azure.VMSS, vmss compute.VirtualMachineScaleSet) bool { + other := converters.SDKToVMSS(vmss, []compute.VirtualMachineScaleSetVM{}) + return infraVMSS.HasModelChanges(other) +} + +func (s *ScaleSetSpec) generateExtensions(ctx context.Context) ([]compute.VirtualMachineScaleSetExtension, error) { + extensions := make([]compute.VirtualMachineScaleSetExtension, len(s.VMSSExtensionSpecs)) + for i, extensionSpec := range s.VMSSExtensionSpecs { + extensionSpec := extensionSpec + parameters, err := extensionSpec.Parameters(ctx, nil) + if err != nil { + return nil, err + } + vmssextension, ok := parameters.(compute.VirtualMachineScaleSetExtension) + if !ok { + return nil, errors.Errorf("%T is not a compute.VirtualMachineScaleSetExtension", parameters) + } + extensions[i] = vmssextension + } + + return extensions, nil +} + +func (s *ScaleSetSpec) getVirtualMachineScaleSetNetworkConfiguration() *[]compute.VirtualMachineScaleSetNetworkConfiguration { + var backendAddressPools []compute.SubResource + if s.PublicLBName != "" { + if s.PublicLBAddressPoolName != "" { + backendAddressPools = append(backendAddressPools, + compute.SubResource{ + ID: ptr.To(azure.AddressPoolID(s.SubscriptionID, s.ResourceGroup, s.PublicLBName, s.PublicLBAddressPoolName)), + }) + } + } + nicConfigs := []compute.VirtualMachineScaleSetNetworkConfiguration{} + for i, n := range s.NetworkInterfaces { + nicConfig := compute.VirtualMachineScaleSetNetworkConfiguration{} + nicConfig.VirtualMachineScaleSetNetworkConfigurationProperties = &compute.VirtualMachineScaleSetNetworkConfigurationProperties{} + nicConfig.Name = ptr.To(s.Name + "-nic-" + strconv.Itoa(i)) + nicConfig.EnableIPForwarding = ptr.To(true) + if n.AcceleratedNetworking != nil { + nicConfig.VirtualMachineScaleSetNetworkConfigurationProperties.EnableAcceleratedNetworking = n.AcceleratedNetworking + } else { + // If AcceleratedNetworking is not specified, use the value from the VMSS spec. + // It will be set to true if the VMSS SKU supports it. + nicConfig.VirtualMachineScaleSetNetworkConfigurationProperties.EnableAcceleratedNetworking = s.AcceleratedNetworking + } + + // Create IPConfigs + ipconfigs := []compute.VirtualMachineScaleSetIPConfiguration{} + for j := 0; j < n.PrivateIPConfigs; j++ { + ipconfig := compute.VirtualMachineScaleSetIPConfiguration{ + Name: ptr.To(fmt.Sprintf("ipConfig" + strconv.Itoa(j))), + VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ + PrivateIPAddressVersion: compute.IPVersionIPv4, + Subnet: &compute.APIEntityReference{ + ID: ptr.To(azure.SubnetID(s.SubscriptionID, s.VNetResourceGroup, s.VNetName, n.SubnetName)), + }, + }, + } + + if j == 0 { + // Always use the first IPConfig as the Primary + ipconfig.Primary = ptr.To(true) + } + ipconfigs = append(ipconfigs, ipconfig) + } + if s.IPv6Enabled { + ipv6Config := compute.VirtualMachineScaleSetIPConfiguration{ + Name: ptr.To("ipConfigv6"), + VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ + PrivateIPAddressVersion: compute.IPVersionIPv6, + Primary: ptr.To(false), + Subnet: &compute.APIEntityReference{ + ID: ptr.To(azure.SubnetID(s.SubscriptionID, s.VNetResourceGroup, s.VNetName, n.SubnetName)), + }, + }, + } + ipconfigs = append(ipconfigs, ipv6Config) + } + if i == 0 { + ipconfigs[0].LoadBalancerBackendAddressPools = &backendAddressPools + nicConfig.VirtualMachineScaleSetNetworkConfigurationProperties.Primary = ptr.To(true) + } + nicConfig.VirtualMachineScaleSetNetworkConfigurationProperties.IPConfigurations = &ipconfigs + nicConfigs = append(nicConfigs, nicConfig) + } + return &nicConfigs +} + +// generateStorageProfile generates a pointer to a compute.VirtualMachineScaleSetStorageProfile which can utilized for VM creation. +func (s *ScaleSetSpec) generateStorageProfile(ctx context.Context) (*compute.VirtualMachineScaleSetStorageProfile, error) { + _, _, done := tele.StartSpanWithLogger(ctx, "scalesets.ScaleSetSpec.generateStorageProfile") + defer done() + + storageProfile := &compute.VirtualMachineScaleSetStorageProfile{ + OsDisk: &compute.VirtualMachineScaleSetOSDisk{ + OsType: compute.OperatingSystemTypes(s.OSDisk.OSType), + CreateOption: compute.DiskCreateOptionTypesFromImage, + DiskSizeGB: s.OSDisk.DiskSizeGB, + }, + } + + // enable ephemeral OS + if s.OSDisk.DiffDiskSettings != nil { + if !s.SKU.HasCapability(resourceskus.EphemeralOSDisk) { + return nil, fmt.Errorf("vm size %s does not support ephemeral os. select a different vm size or disable ephemeral os", s.Size) + } + + storageProfile.OsDisk.DiffDiskSettings = &compute.DiffDiskSettings{ + Option: compute.DiffDiskOptions(s.OSDisk.DiffDiskSettings.Option), + } + } + + if s.OSDisk.ManagedDisk != nil { + storageProfile.OsDisk.ManagedDisk = &compute.VirtualMachineScaleSetManagedDiskParameters{} + if s.OSDisk.ManagedDisk.StorageAccountType != "" { + storageProfile.OsDisk.ManagedDisk.StorageAccountType = compute.StorageAccountTypes(s.OSDisk.ManagedDisk.StorageAccountType) + } + if s.OSDisk.ManagedDisk.DiskEncryptionSet != nil { + storageProfile.OsDisk.ManagedDisk.DiskEncryptionSet = &compute.DiskEncryptionSetParameters{ID: ptr.To(s.OSDisk.ManagedDisk.DiskEncryptionSet.ID)} + } + } + + if s.OSDisk.CachingType != "" { + storageProfile.OsDisk.Caching = compute.CachingTypes(s.OSDisk.CachingType) + } + + dataDisks := make([]compute.VirtualMachineScaleSetDataDisk, len(s.DataDisks)) + for i, disk := range s.DataDisks { + dataDisks[i] = compute.VirtualMachineScaleSetDataDisk{ + CreateOption: compute.DiskCreateOptionTypesEmpty, + DiskSizeGB: ptr.To[int32](disk.DiskSizeGB), + Lun: disk.Lun, + Name: ptr.To(azure.GenerateDataDiskName(s.Name, disk.NameSuffix)), + } + + if disk.ManagedDisk != nil { + dataDisks[i].ManagedDisk = &compute.VirtualMachineScaleSetManagedDiskParameters{ + StorageAccountType: compute.StorageAccountTypes(disk.ManagedDisk.StorageAccountType), + } + + if disk.ManagedDisk.DiskEncryptionSet != nil { + dataDisks[i].ManagedDisk.DiskEncryptionSet = &compute.DiskEncryptionSetParameters{ID: ptr.To(disk.ManagedDisk.DiskEncryptionSet.ID)} + } + } + } + storageProfile.DataDisks = &dataDisks + + if s.VMImage == nil { + return nil, errors.Errorf("vm image is nil") + } + imageRef, err := converters.ImageToSDK(s.VMImage) + if err != nil { + return nil, err + } + + storageProfile.ImageReference = imageRef + + return storageProfile, nil +} + +func (s *ScaleSetSpec) generateOSProfile(_ context.Context) (*compute.VirtualMachineScaleSetOSProfile, error) { + sshKey, err := base64.StdEncoding.DecodeString(s.SSHKeyData) + if err != nil { + return nil, errors.Wrap(err, "failed to decode ssh public key") + } + + osProfile := &compute.VirtualMachineScaleSetOSProfile{ + ComputerNamePrefix: ptr.To(s.Name), + AdminUsername: ptr.To(azure.DefaultUserName), + CustomData: ptr.To(s.BootstrapData), + } + + switch s.OSDisk.OSType { + case string(compute.OperatingSystemTypesWindows): + // Cloudbase-init is used to generate a password. + // https://cloudbase-init.readthedocs.io/en/latest/plugins.html#setting-password-main + // + // We generate a random password here in case of failure + // but the password on the VM will NOT be the same as created here. + // Access is provided via SSH public key that is set during deployment + // Azure also provides a way to reset user passwords in the case of need. + osProfile.AdminPassword = ptr.To(generators.SudoRandomPassword(123)) + osProfile.WindowsConfiguration = &compute.WindowsConfiguration{ + EnableAutomaticUpdates: ptr.To(false), + } + default: + osProfile.LinuxConfiguration = &compute.LinuxConfiguration{ + DisablePasswordAuthentication: ptr.To(true), + SSH: &compute.SSHConfiguration{ + PublicKeys: &[]compute.SSHPublicKey{ + { + Path: ptr.To(fmt.Sprintf("/home/%s/.ssh/authorized_keys", azure.DefaultUserName)), + KeyData: ptr.To(string(sshKey)), + }, + }, + }, + } + } + + return osProfile, nil +} + +func (s *ScaleSetSpec) generateImagePlan(ctx context.Context) *compute.Plan { + _, log, done := tele.StartSpanWithLogger(ctx, "scalesets.ScaleSetSpec.generateImagePlan") + defer done() + + if s.VMImage == nil { + log.V(2).Info("no vm image found, disabling plan") + return nil + } + + if s.VMImage.SharedGallery != nil && s.VMImage.SharedGallery.Publisher != nil && s.VMImage.SharedGallery.SKU != nil && s.VMImage.SharedGallery.Offer != nil { + return &compute.Plan{ + Publisher: s.VMImage.SharedGallery.Publisher, + Name: s.VMImage.SharedGallery.SKU, + Product: s.VMImage.SharedGallery.Offer, + } + } + + if s.VMImage.Marketplace == nil || !s.VMImage.Marketplace.ThirdPartyImage { + return nil + } + + if s.VMImage.Marketplace.Publisher == "" || s.VMImage.Marketplace.SKU == "" || s.VMImage.Marketplace.Offer == "" { + return nil + } + + return &compute.Plan{ + Publisher: ptr.To(s.VMImage.Marketplace.Publisher), + Name: ptr.To(s.VMImage.Marketplace.SKU), + Product: ptr.To(s.VMImage.Marketplace.Offer), + } +} + +func (s *ScaleSetSpec) getSecurityProfile() (*compute.SecurityProfile, error) { + if s.SecurityProfile == nil { + return nil, nil + } + + if !s.SKU.HasCapability(resourceskus.EncryptionAtHost) { + return nil, azure.WithTerminalError(errors.Errorf("encryption at host is not supported for VM type %s", s.Size)) + } + + return &compute.SecurityProfile{ + EncryptionAtHost: ptr.To(*s.SecurityProfile.EncryptionAtHost), + }, nil +} diff --git a/azure/services/scalesets/spec_test.go b/azure/services/scalesets/spec_test.go new file mode 100644 index 00000000000..7f76ad4027a --- /dev/null +++ b/azure/services/scalesets/spec_test.go @@ -0,0 +1,707 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scalesets + +import ( + "context" + "reflect" + "testing" + + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute" + "github.com/google/go-cmp/cmp" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/utils/ptr" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-azure/azure/services/resourceskus" +) + +var ( + defaultSpec, defaultVMSS = getDefaultVMSS() + windowsSpec, windowsVMSS = getDefaultWindowsVMSS() + acceleratedNetworkingSpec, acceleratedNetworkingVMSS = getAcceleratedNetworkingVMSS() + customSubnetSpec, customSubnetVMSS = getCustomSubnetVMSS() + customNetworkingSpec, customNetworkingVMSS = getCustomNetworkingVMSS() + spotVMSpec, spotVMVMSS = getSpotVMVMSS() + ephemeralSpec, ephemeralVMSS = getEPHVMSSS() + evictionSpec, evictionVMSS = getEvictionPolicyVMSS() + maxPriceSpec, maxPriceVMSS = getMaxPriceVMSS() + encryptionSpec, encryptionVMSS = getEncryptionVMSS() + userIdentitySpec, userIdentityVMSS = getUserIdentityVMSS() + hostEncryptionSpec, hostEncryptionVMSS = getHostEncryptionVMSS() + hostEncryptionUnsupportedSpec = getHostEncryptionUnsupportedSpec() + ephemeralReadSpec, ephemeralReadVMSS = getEphemeralReadOnlyVMSS() + defaultExistingSpec, defaultExistingVMSS, defaultExistingVMSSClone = getExistingDefaultVMSS() + userManagedStorageAccountDiagnosticsSpec, userManagedStorageAccountDiagnosticsVMSS = getUserManagedAndStorageAcccountDiagnosticsVMSS() + managedDiagnosticsSpec, managedDiagnoisticsVMSS = getManagedDiagnosticsVMSS() + disabledDiagnosticsSpec, disabledDiagnosticsVMSS = getDisabledDiagnosticsVMSS() + nilDiagnosticsProfileSpec, nilDiagnosticsProfileVMSS = getNilDiagnosticsProfileVMSS() +) + +func getDefaultVMSS() (ScaleSetSpec, compute.VirtualMachineScaleSet) { + spec := newDefaultVMSSSpec() + spec.DataDisks = append(spec.DataDisks, infrav1.DataDisk{ + NameSuffix: "my_disk_with_ultra_disks", + DiskSizeGB: 128, + Lun: ptr.To[int32](3), + ManagedDisk: &infrav1.ManagedDiskParameters{ + StorageAccountType: "UltraSSD_LRS", + }, + }) + + vmss := newDefaultVMSS("VM_SIZE") + vmss.VirtualMachineScaleSetProperties.AdditionalCapabilities = &compute.AdditionalCapabilities{UltraSSDEnabled: ptr.To(true)} + + return spec, vmss +} + +func getDefaultWindowsVMSS() (ScaleSetSpec, compute.VirtualMachineScaleSet) { + spec := newWindowsVMSSSpec() + // Do we want this here? + spec.DataDisks = append(spec.DataDisks, infrav1.DataDisk{ + NameSuffix: "my_disk_with_ultra_disks", + DiskSizeGB: 128, + Lun: ptr.To[int32](3), + ManagedDisk: &infrav1.ManagedDiskParameters{ + StorageAccountType: "UltraSSD_LRS", + }, + }) + vmss := newDefaultWindowsVMSS() + vmss.VirtualMachineScaleSetProperties.AdditionalCapabilities = &compute.AdditionalCapabilities{UltraSSDEnabled: ptr.To(true)} + + return spec, vmss +} + +func getAcceleratedNetworkingVMSS() (ScaleSetSpec, compute.VirtualMachineScaleSet) { + spec := newDefaultVMSSSpec() + spec.Size = "VM_SIZE_AN" + spec.AcceleratedNetworking = ptr.To(true) + spec.NetworkInterfaces[0].AcceleratedNetworking = ptr.To(true) + vmss := newDefaultVMSS("VM_SIZE_AN") + (*vmss.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations)[0].VirtualMachineScaleSetNetworkConfigurationProperties.EnableAcceleratedNetworking = ptr.To(true) + + return spec, vmss +} + +func getCustomSubnetVMSS() (ScaleSetSpec, compute.VirtualMachineScaleSet) { + spec := newDefaultVMSSSpec() + spec.Size = "VM_SIZE_AN" + spec.AcceleratedNetworking = ptr.To(true) + spec.NetworkInterfaces = []infrav1.NetworkInterface{ + { + SubnetName: "somesubnet", + PrivateIPConfigs: 1, // defaulter sets this to one + }, + } + customSubnetVMSS := newDefaultVMSS("VM_SIZE_AN") + netConfigs := customSubnetVMSS.VirtualMachineScaleSetProperties.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations + (*netConfigs)[0].Name = ptr.To("my-vmss-nic-0") + (*netConfigs)[0].EnableIPForwarding = ptr.To(true) + (*netConfigs)[0].EnableAcceleratedNetworking = ptr.To(true) + nic1IPConfigs := (*netConfigs)[0].IPConfigurations + (*nic1IPConfigs)[0].Name = ptr.To("ipConfig0") + (*nic1IPConfigs)[0].PrivateIPAddressVersion = compute.IPVersionIPv4 + (*nic1IPConfigs)[0].Subnet = &compute.APIEntityReference{ + ID: ptr.To("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/virtualNetworks/my-vnet/subnets/somesubnet"), + } + (*netConfigs)[0].EnableAcceleratedNetworking = ptr.To(true) + (*netConfigs)[0].Primary = ptr.To(true) + + return spec, customSubnetVMSS +} + +func getCustomNetworkingVMSS() (ScaleSetSpec, compute.VirtualMachineScaleSet) { + spec := newDefaultVMSSSpec() + spec.NetworkInterfaces = []infrav1.NetworkInterface{ + { + SubnetName: "my-subnet", + PrivateIPConfigs: 1, + AcceleratedNetworking: ptr.To(true), + }, + { + SubnetName: "subnet2", + PrivateIPConfigs: 2, + AcceleratedNetworking: ptr.To(true), + }, + } + spec.DataDisks = append(spec.DataDisks, infrav1.DataDisk{ + NameSuffix: "my_disk_with_ultra_disks", + DiskSizeGB: 128, + Lun: ptr.To[int32](3), + ManagedDisk: &infrav1.ManagedDiskParameters{ + StorageAccountType: "UltraSSD_LRS", + }, + }) + vmss := newDefaultVMSS("VM_SIZE") + vmss.VirtualMachineScaleSetProperties.AdditionalCapabilities = &compute.AdditionalCapabilities{UltraSSDEnabled: ptr.To(true)} + netConfigs := vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations + (*netConfigs)[0].Name = ptr.To("my-vmss-nic-0") + (*netConfigs)[0].EnableIPForwarding = ptr.To(true) + nic1IPConfigs := (*netConfigs)[0].IPConfigurations + (*nic1IPConfigs)[0].Name = ptr.To("ipConfig0") + (*nic1IPConfigs)[0].PrivateIPAddressVersion = compute.IPVersionIPv4 + (*netConfigs)[0].EnableAcceleratedNetworking = ptr.To(true) + (*netConfigs)[0].Primary = ptr.To(true) + vmssIPConfigs := []compute.VirtualMachineScaleSetIPConfiguration{ + { + Name: ptr.To("ipConfig0"), + VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ + Primary: ptr.To(true), + PrivateIPAddressVersion: compute.IPVersionIPv4, + Subnet: &compute.APIEntityReference{ + ID: ptr.To("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/virtualNetworks/my-vnet/subnets/subnet2"), + }, + }, + }, + { + Name: ptr.To("ipConfig1"), + VirtualMachineScaleSetIPConfigurationProperties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ + PrivateIPAddressVersion: compute.IPVersionIPv4, + Subnet: &compute.APIEntityReference{ + ID: ptr.To("/subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/virtualNetworks/my-vnet/subnets/subnet2"), + }, + }, + }, + } + *netConfigs = append(*netConfigs, compute.VirtualMachineScaleSetNetworkConfiguration{ + Name: ptr.To("my-vmss-nic-1"), + VirtualMachineScaleSetNetworkConfigurationProperties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{ + EnableAcceleratedNetworking: ptr.To(true), + IPConfigurations: &vmssIPConfigs, + EnableIPForwarding: ptr.To(true), + }, + }) + + return spec, vmss +} + +func getSpotVMVMSS() (ScaleSetSpec, compute.VirtualMachineScaleSet) { + spec := newDefaultVMSSSpec() + spec.DataDisks = append(spec.DataDisks, infrav1.DataDisk{ + NameSuffix: "my_disk_with_ultra_disks", + DiskSizeGB: 128, + Lun: ptr.To[int32](3), + ManagedDisk: &infrav1.ManagedDiskParameters{ + StorageAccountType: "UltraSSD_LRS", + }, + }) + spec.SpotVMOptions = &infrav1.SpotVMOptions{} + vmss := newDefaultVMSS("VM_SIZE") + vmss.VirtualMachineScaleSetProperties.AdditionalCapabilities = &compute.AdditionalCapabilities{UltraSSDEnabled: ptr.To(true)} + vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.Priority = compute.VirtualMachinePriorityTypesSpot + + return spec, vmss +} + +func getEPHVMSSS() (ScaleSetSpec, compute.VirtualMachineScaleSet) { + spec := newDefaultVMSSSpec() + spec.Size = vmSizeEPH + spec.SKU = resourceskus.SKU{ + Capabilities: &[]compute.ResourceSkuCapabilities{ + { + Name: ptr.To(resourceskus.EphemeralOSDisk), + Value: ptr.To("True"), + }, + }, + } + spec.SpotVMOptions = &infrav1.SpotVMOptions{} + spec.OSDisk.DiffDiskSettings = &infrav1.DiffDiskSettings{ + Option: string(compute.DiffDiskOptionsLocal), + } + vmss := newDefaultVMSS(vmSizeEPH) + vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.StorageProfile.OsDisk.DiffDiskSettings = &compute.DiffDiskSettings{ + Option: compute.DiffDiskOptionsLocal, + } + vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.Priority = compute.VirtualMachinePriorityTypesSpot + + return spec, vmss +} + +func getEvictionPolicyVMSS() (ScaleSetSpec, compute.VirtualMachineScaleSet) { + spec := newDefaultVMSSSpec() + spec.Size = vmSizeEPH + deletePolicy := infrav1.SpotEvictionPolicyDelete + spec.SpotVMOptions = &infrav1.SpotVMOptions{EvictionPolicy: &deletePolicy} + vmss := newDefaultVMSS(vmSizeEPH) + vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.Priority = compute.VirtualMachinePriorityTypesSpot + vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.EvictionPolicy = compute.VirtualMachineEvictionPolicyTypesDelete + + return spec, vmss +} + +func getMaxPriceVMSS() (ScaleSetSpec, compute.VirtualMachineScaleSet) { + spec := newDefaultVMSSSpec() + maxPrice := resource.MustParse("0.001") + spec.SpotVMOptions = &infrav1.SpotVMOptions{ + MaxPrice: &maxPrice, + } + spec.DataDisks = append(spec.DataDisks, infrav1.DataDisk{ + NameSuffix: "my_disk_with_ultra_disks", + DiskSizeGB: 128, + Lun: ptr.To[int32](3), + ManagedDisk: &infrav1.ManagedDiskParameters{ + StorageAccountType: "UltraSSD_LRS", + }, + }) + vmss := newDefaultVMSS("VM_SIZE") + vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.Priority = compute.VirtualMachinePriorityTypesSpot + vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.BillingProfile = &compute.BillingProfile{ + MaxPrice: ptr.To[float64](0.001), + } + vmss.VirtualMachineScaleSetProperties.AdditionalCapabilities = &compute.AdditionalCapabilities{UltraSSDEnabled: ptr.To(true)} + + return spec, vmss +} + +func getEncryptionVMSS() (ScaleSetSpec, compute.VirtualMachineScaleSet) { + spec := newDefaultVMSSSpec() + spec.OSDisk.ManagedDisk.DiskEncryptionSet = &infrav1.DiskEncryptionSetParameters{ + ID: "my-diskencryptionset-id", + } + spec.DataDisks = append(spec.DataDisks, infrav1.DataDisk{ + NameSuffix: "my_disk_with_ultra_disks", + DiskSizeGB: 128, + Lun: ptr.To[int32](3), + ManagedDisk: &infrav1.ManagedDiskParameters{ + StorageAccountType: "UltraSSD_LRS", + }, + }) + vmss := newDefaultVMSS("VM_SIZE") + vmss.VirtualMachineScaleSetProperties.AdditionalCapabilities = &compute.AdditionalCapabilities{UltraSSDEnabled: ptr.To(true)} + osdisk := vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.StorageProfile.OsDisk + osdisk.ManagedDisk = &compute.VirtualMachineScaleSetManagedDiskParameters{ + StorageAccountType: "Premium_LRS", + DiskEncryptionSet: &compute.DiskEncryptionSetParameters{ + ID: ptr.To("my-diskencryptionset-id"), + }, + } + + return spec, vmss +} + +func getUserIdentityVMSS() (ScaleSetSpec, compute.VirtualMachineScaleSet) { + spec := newDefaultVMSSSpec() + spec.DataDisks = append(spec.DataDisks, infrav1.DataDisk{ + NameSuffix: "my_disk_with_ultra_disks", + DiskSizeGB: 128, + Lun: ptr.To[int32](3), + ManagedDisk: &infrav1.ManagedDiskParameters{ + StorageAccountType: "UltraSSD_LRS", + }, + }) + spec.Identity = infrav1.VMIdentityUserAssigned + spec.UserAssignedIdentities = []infrav1.UserAssignedIdentity{ + { + ProviderID: "azure:///subscriptions/123/resourcegroups/456/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id1", + }, + } + vmss := newDefaultVMSS("VM_SIZE") + vmss.VirtualMachineScaleSetProperties.AdditionalCapabilities = &compute.AdditionalCapabilities{UltraSSDEnabled: ptr.To(true)} + vmss.Identity = &compute.VirtualMachineScaleSetIdentity{ + Type: compute.ResourceIdentityTypeUserAssigned, + UserAssignedIdentities: map[string]*compute.VirtualMachineScaleSetIdentityUserAssignedIdentitiesValue{ + "/subscriptions/123/resourcegroups/456/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id1": {}, + }, + } + + return spec, vmss +} + +func getHostEncryptionVMSS() (ScaleSetSpec, compute.VirtualMachineScaleSet) { + spec := newDefaultVMSSSpec() + spec.Size = "VM_SIZE_EAH" + spec.SecurityProfile = &infrav1.SecurityProfile{EncryptionAtHost: ptr.To(true)} + spec.SKU = resourceskus.SKU{ + Capabilities: &[]compute.ResourceSkuCapabilities{ + { + Name: ptr.To(resourceskus.EncryptionAtHost), + Value: ptr.To("True"), + }, + }, + } + vmss := newDefaultVMSS("VM_SIZE_EAH") + vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.SecurityProfile = &compute.SecurityProfile{ + EncryptionAtHost: ptr.To(true), + } + vmss.Sku.Name = ptr.To(spec.Size) + + return spec, vmss +} + +func getHostEncryptionUnsupportedSpec() ScaleSetSpec { + spec, _ := getHostEncryptionVMSS() + spec.SKU = resourceskus.SKU{} + return spec +} + +func getEphemeralReadOnlyVMSS() (ScaleSetSpec, compute.VirtualMachineScaleSet) { + spec := newDefaultVMSSSpec() + spec.Size = "VM_SIZE_EPH" + spec.OSDisk.DiffDiskSettings = &infrav1.DiffDiskSettings{ + Option: "Local", + } + spec.OSDisk.CachingType = "ReadOnly" + spec.SKU = resourceskus.SKU{ + Capabilities: &[]compute.ResourceSkuCapabilities{ + { + Name: ptr.To(resourceskus.EphemeralOSDisk), + Value: ptr.To("True"), + }, + }, + } + + vmss := newDefaultVMSS("VM_SIZE_EPH") + vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.StorageProfile.OsDisk.DiffDiskSettings = &compute.DiffDiskSettings{ + Option: compute.DiffDiskOptionsLocal, + } + vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.StorageProfile.OsDisk.Caching = compute.CachingTypesReadOnly + + return spec, vmss +} + +func getExistingDefaultVMSS() (s ScaleSetSpec, existing compute.VirtualMachineScaleSet, result compute.VirtualMachineScaleSet) { + spec := newDefaultVMSSSpec() + spec.Capacity = 2 + spec.DataDisks = append(spec.DataDisks, infrav1.DataDisk{ + NameSuffix: "my_disk_with_ultra_disks", + DiskSizeGB: 128, + Lun: ptr.To[int32](3), + ManagedDisk: &infrav1.ManagedDiskParameters{ + StorageAccountType: "UltraSSD_LRS", + }, + }) + spec.MaxSurge = 1 + + spec.VMImage = &infrav1.Image{ + Marketplace: &infrav1.AzureMarketplaceImage{ + ImagePlan: infrav1.ImagePlan{ + Publisher: "fake-publisher", + Offer: "my-offer", + SKU: "sku-id", + }, + Version: "2.0", + }, + } + + existingVMSS := newDefaultExistingVMSS("VM_SIZE") + existingVMSS.VirtualMachineScaleSetProperties.AdditionalCapabilities = &compute.AdditionalCapabilities{UltraSSDEnabled: ptr.To(true)} + existingVMSS.Sku.Capacity = ptr.To[int64](2) + existingVMSS.VirtualMachineScaleSetProperties.AdditionalCapabilities = &compute.AdditionalCapabilities{UltraSSDEnabled: ptr.To(true)} + + clone := newDefaultExistingVMSS("VM_SIZE") + clone.Sku.Capacity = ptr.To[int64](3) + clone.VirtualMachineScaleSetProperties.AdditionalCapabilities = &compute.AdditionalCapabilities{UltraSSDEnabled: ptr.To(true)} + clone.VirtualMachineProfile.NetworkProfile = nil + + clone.VirtualMachineProfile.StorageProfile.ImageReference.Version = ptr.To("2.0") + clone.VirtualMachineProfile.NetworkProfile = nil + + return spec, existingVMSS, clone +} + +func getUserManagedAndStorageAcccountDiagnosticsVMSS() (ScaleSetSpec, compute.VirtualMachineScaleSet) { + storageURI := "https://fakeurl" + spec := newDefaultVMSSSpec() + spec.DiagnosticsProfile = &infrav1.Diagnostics{ + Boot: &infrav1.BootDiagnostics{ + StorageAccountType: infrav1.UserManagedDiagnosticsStorage, + UserManaged: &infrav1.UserManagedBootDiagnostics{ + StorageAccountURI: storageURI, + }, + }, + } + spec.DataDisks = append(spec.DataDisks, infrav1.DataDisk{ + NameSuffix: "my_disk_with_ultra_disks", + DiskSizeGB: 128, + Lun: ptr.To[int32](3), + ManagedDisk: &infrav1.ManagedDiskParameters{ + StorageAccountType: "UltraSSD_LRS", + }, + }) + + spec.VMSSInstances = newDefaultInstances() + spec.MaxSurge = 1 + + vmss := newDefaultVMSS("VM_SIZE") + vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.DiagnosticsProfile = &compute.DiagnosticsProfile{BootDiagnostics: &compute.BootDiagnostics{ + Enabled: ptr.To(true), + StorageURI: &storageURI, + }} + vmss.VirtualMachineScaleSetProperties.AdditionalCapabilities = &compute.AdditionalCapabilities{UltraSSDEnabled: ptr.To(true)} + + return spec, vmss +} + +func getManagedDiagnosticsVMSS() (ScaleSetSpec, compute.VirtualMachineScaleSet) { + spec := newDefaultVMSSSpec() + spec.DiagnosticsProfile = &infrav1.Diagnostics{ + Boot: &infrav1.BootDiagnostics{ + StorageAccountType: infrav1.ManagedDiagnosticsStorage, + }, + } + spec.DataDisks = append(spec.DataDisks, infrav1.DataDisk{ + NameSuffix: "my_disk_with_ultra_disks", + DiskSizeGB: 128, + Lun: ptr.To[int32](3), + ManagedDisk: &infrav1.ManagedDiskParameters{ + StorageAccountType: "UltraSSD_LRS", + }, + }) + spec.VMSSInstances = newDefaultInstances() + + vmss := newDefaultVMSS("VM_SIZE") + vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.DiagnosticsProfile = &compute.DiagnosticsProfile{BootDiagnostics: &compute.BootDiagnostics{ + Enabled: ptr.To(true), + }} + vmss.VirtualMachineScaleSetProperties.AdditionalCapabilities = &compute.AdditionalCapabilities{UltraSSDEnabled: ptr.To(true)} + + return spec, vmss +} + +func getDisabledDiagnosticsVMSS() (ScaleSetSpec, compute.VirtualMachineScaleSet) { + spec := newDefaultVMSSSpec() + spec.DiagnosticsProfile = &infrav1.Diagnostics{ + Boot: &infrav1.BootDiagnostics{ + StorageAccountType: infrav1.DisabledDiagnosticsStorage, + }, + } + spec.DataDisks = append(spec.DataDisks, infrav1.DataDisk{ + NameSuffix: "my_disk_with_ultra_disks", + DiskSizeGB: 128, + Lun: ptr.To[int32](3), + ManagedDisk: &infrav1.ManagedDiskParameters{ + StorageAccountType: "UltraSSD_LRS", + }, + }) + spec.VMSSInstances = newDefaultInstances() + + vmss := newDefaultVMSS("VM_SIZE") + vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.DiagnosticsProfile = &compute.DiagnosticsProfile{BootDiagnostics: &compute.BootDiagnostics{ + Enabled: ptr.To(false), + }} + vmss.VirtualMachineScaleSetProperties.AdditionalCapabilities = &compute.AdditionalCapabilities{UltraSSDEnabled: ptr.To(true)} + + return spec, vmss +} + +func getNilDiagnosticsProfileVMSS() (ScaleSetSpec, compute.VirtualMachineScaleSet) { + spec := newDefaultVMSSSpec() + spec.DiagnosticsProfile = nil + + spec.DataDisks = append(spec.DataDisks, infrav1.DataDisk{ + NameSuffix: "my_disk_with_ultra_disks", + DiskSizeGB: 128, + Lun: ptr.To[int32](3), + ManagedDisk: &infrav1.ManagedDiskParameters{ + StorageAccountType: "UltraSSD_LRS", + }, + }) + spec.VMSSInstances = newDefaultInstances() + + vmss := newDefaultVMSS("VM_SIZE") + vmss.VirtualMachineScaleSetProperties.VirtualMachineProfile.DiagnosticsProfile = nil + + vmss.VirtualMachineScaleSetProperties.AdditionalCapabilities = &compute.AdditionalCapabilities{UltraSSDEnabled: ptr.To(true)} + + return spec, vmss +} + +func TestScaleSetParameters(t *testing.T) { + testcases := []struct { + name string + spec ScaleSetSpec + existing interface{} + expected interface{} + expectedError string + }{ + { + name: "get parameters for a vmss", + spec: defaultSpec, + existing: nil, + expected: defaultVMSS, + expectedError: "", + }, + { + name: "get parameters for a windows vmss", + spec: windowsSpec, + existing: nil, + expected: windowsVMSS, + expectedError: "", + }, + { + name: "windows vmss up to date", + spec: windowsSpec, + existing: windowsVMSS, + expected: nil, + expectedError: "", + }, + { + name: "accelerated networking vmss", + spec: acceleratedNetworkingSpec, + existing: nil, + expected: acceleratedNetworkingVMSS, + expectedError: "", + }, + { + name: "custom subnet vmss", + spec: customSubnetSpec, + existing: nil, + expected: customSubnetVMSS, + expectedError: "", + }, + { + name: "custom networking vmss", + spec: customNetworkingSpec, + existing: nil, + expected: customNetworkingVMSS, + expectedError: "", + }, + { + name: "spot vm vmss", + spec: spotVMSpec, + existing: nil, + expected: spotVMVMSS, + expectedError: "", + }, + { + name: "spot vm and ephemeral disk vmss", + spec: ephemeralSpec, + existing: nil, + expected: ephemeralVMSS, + expectedError: "", + }, + { + name: "spot vm and eviction policy vmss", + spec: evictionSpec, + existing: nil, + expected: evictionVMSS, + expectedError: "", + }, + { + name: "spot vm and max price vmss", + spec: maxPriceSpec, + existing: nil, + expected: maxPriceVMSS, + expectedError: "", + }, + { + name: "eviction policy vmss", + spec: evictionSpec, + existing: nil, + expected: evictionVMSS, + expectedError: "", + }, + { + name: "encryption vmss", + spec: encryptionSpec, + existing: nil, + expected: encryptionVMSS, + expectedError: "", + }, + { + name: "user assigned identity vmss", + spec: userIdentitySpec, + existing: nil, + expected: userIdentityVMSS, + expectedError: "", + }, + { + name: "host encryption vmss", + spec: hostEncryptionSpec, + existing: nil, + expected: hostEncryptionVMSS, + expectedError: "", + }, + { + name: "host encryption unsupported vmss", + spec: hostEncryptionUnsupportedSpec, + existing: nil, + expected: nil, + expectedError: "reconcile error that cannot be recovered occurred: encryption at host is not supported for VM type VM_SIZE_EAH. Object will not be requeued", + }, + { + name: "ephemeral os disk read only vmss", + spec: ephemeralReadSpec, + existing: nil, + expected: ephemeralReadVMSS, + expectedError: "", + }, + { + name: "update for existing vmss", + spec: defaultExistingSpec, + existing: defaultExistingVMSS, + expected: defaultExistingVMSSClone, + expectedError: "", + }, + { + name: "vm with diagnostics set to User Managed and StorageAccountURI set", + spec: userManagedStorageAccountDiagnosticsSpec, + existing: nil, + expected: userManagedStorageAccountDiagnosticsVMSS, + expectedError: "", + }, + { + name: "vm with diagnostics set to Managed", + spec: managedDiagnosticsSpec, + existing: nil, + expected: managedDiagnoisticsVMSS, + expectedError: "", + }, + { + name: "vm with diagnostics set to Disabled", + spec: disabledDiagnosticsSpec, + existing: nil, + expected: disabledDiagnosticsVMSS, + expectedError: "", + }, + { + name: "vm with DiagnosticsProfile set to nil, do not panic", + spec: nilDiagnosticsProfileSpec, + existing: nil, + expected: nilDiagnosticsProfileVMSS, + expectedError: "", + }, + } + for _, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + t.Parallel() + + param, err := tc.spec.Parameters(context.TODO(), tc.existing) + if tc.expectedError != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err).To(MatchError(tc.expectedError)) + } else { + g.Expect(err).NotTo(HaveOccurred()) + if tc.expected == nil { + g.Expect(param).To(BeNil()) + } else { + result, ok := param.(compute.VirtualMachineScaleSet) + if !ok { + t.Fatalf("expected type VirtualMachineScaleSet, got %T", param) + } + result.VirtualMachineProfile.OsProfile.AdminPassword = nil // Override this field as it's randomly generated. We can't set anything in tc.expected to match it. + + if !reflect.DeepEqual(tc.expected, result) { + t.Errorf("Diff between actual result and expected result:\n%s", cmp.Diff(result, tc.expected)) + } + } + } + }) + } +} diff --git a/azure/services/scalesets/vmssextension_spec_test.go b/azure/services/scalesets/vmssextension_spec_test.go index 12693330be6..954ea2583b8 100644 --- a/azure/services/scalesets/vmssextension_spec_test.go +++ b/azure/services/scalesets/vmssextension_spec_test.go @@ -51,7 +51,7 @@ var ( } ) -func TestParameters(t *testing.T) { +func TestVMSSExtensionParameters(t *testing.T) { testcases := []struct { name string spec *VMSSExtensionSpec diff --git a/exp/controllers/azuremachinepool_controller.go b/exp/controllers/azuremachinepool_controller.go index 9ea6258b796..7ff3bc25f1c 100644 --- a/exp/controllers/azuremachinepool_controller.go +++ b/exp/controllers/azuremachinepool_controller.go @@ -300,14 +300,14 @@ func (ampr *AzureMachinePoolReconciler) reconcileNormal(ctx context.Context, mac err := machinePoolScope.InitMachinePoolCache(ctx) if err != nil { if errors.As(err, &reconcileError) && reconcileError.IsTerminal() { - ampr.Recorder.Eventf(machinePoolScope.AzureMachinePool, corev1.EventTypeWarning, "SKUNotFound", errors.Wrap(err, "failed to initialize machine cache").Error()) - log.Error(err, "Failed to initialize machine cache") + ampr.Recorder.Eventf(machinePoolScope.AzureMachinePool, corev1.EventTypeWarning, "SKUNotFound", errors.Wrap(err, "failed to initialize machinepool cache").Error()) + log.Error(err, "Failed to initialize machinepool cache") machinePoolScope.SetFailureReason(capierrors.InvalidConfigurationMachineError) machinePoolScope.SetFailureMessage(err) machinePoolScope.SetNotReady() return reconcile.Result{}, nil } - return reconcile.Result{}, errors.Wrap(err, "failed to init machine scope cache") + return reconcile.Result{}, errors.Wrap(err, "failed to init machinepool scope cache") } ams, err := ampr.createAzureMachinePoolService(machinePoolScope)