Skip to content

Commit

Permalink
Make scaleset service async
Browse files Browse the repository at this point in the history
  • Loading branch information
@Jont828
Jont828 committed May 23, 2023
1 parent bee83d3 commit 83aa946
Show file tree
Hide file tree
Showing 8 changed files with 997 additions and 891 deletions.
94 changes: 92 additions & 2 deletions azure/scope/machinepool.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (
infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
"sigs.k8s.io/cluster-api-provider-azure/azure"
machinepool "sigs.k8s.io/cluster-api-provider-azure/azure/scope/strategies/machinepool_deployments"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/resourceskus"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/roleassignments"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/scalesets"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/virtualmachineimages"
Expand Down Expand Up @@ -62,6 +63,7 @@ type (
MachinePool *expv1.MachinePool
AzureMachinePool *infrav1exp.AzureMachinePool
ClusterScope azure.ClusterScoper
Cache *MachinePoolCache
}

// MachinePoolScope defines a scope defined around a machine pool and its cluster.
Expand All @@ -73,13 +75,22 @@ type (
patchHelper *patch.Helper
capiMachinePoolPatchHelper *patch.Helper
vmssState *azure.VMSS
cache *MachinePoolCache
}

// NodeStatus represents the status of a Kubernetes node.
NodeStatus struct {
Ready bool
Version string
}

// MachinePoolCache stores common machine pool information so we don't have to hit the API multiple times within the same reconcile loop.
MachinePoolCache struct {
BootstrapData string
VMImage *infrav1.Image
VMSKU resourceskus.SKU
availabilitySetSKU resourceskus.SKU
}
)

// NewMachinePoolScope creates a new MachinePoolScope from the supplied parameters.
Expand Down Expand Up @@ -117,9 +128,77 @@ func NewMachinePoolScope(params MachinePoolScopeParams) (*MachinePoolScope, erro
}, nil
}

func (m *MachinePoolScope) InitMachinePoolCache(ctx context.Context) error {
ctx, _, done := tele.StartSpanWithLogger(ctx, "azure.MachineScope.InitMachineCache")
defer done()

if m.cache == nil {
var err error
m.cache = &MachinePoolCache{}

m.cache.BootstrapData, err = m.GetBootstrapData(ctx)
if err != nil {
return err
}

m.cache.VMImage, err = m.GetVMImage(ctx)
if err != nil {
return err
}

skuCache, err := resourceskus.GetCache(m, m.Location())
if err != nil {
return err
}

m.cache.VMSKU, err = skuCache.Get(ctx, m.AzureMachinePool.Spec.Template.VMSize, resourceskus.VirtualMachines)
if err != nil {
return errors.Wrapf(err, "failed to get VM SKU %s in compute api", m.AzureMachinePool.Spec.Template.VMSize)
}

}

return nil
}

// ScaleSetSpec returns the scale set spec.
func (m *MachinePoolScope) ScaleSetSpec() azure.ScaleSetSpec {
return azure.ScaleSetSpec{
func (m *MachinePoolScope) ScaleSetSpec(ctx context.Context) azure.ResourceSpecGetter {
ctx, log, done := tele.StartSpanWithLogger(ctx, "scope.MachinePoolScope.ScaleSetSpec")
defer done()

vmImage, err := m.GetVMImage(ctx)
if err != nil {
log.Error(err, "failed to get VM image")
// TODO: do we just leave this nil or return an error?
}

bootstrapData, err := m.GetBootstrapData(ctx)
if err != nil {
log.Error(err, "failed to get bootstrap data")
// TODO: do we return early here?
}

maxSurge, err := m.MaxSurge()
if err != nil {
log.Error(err, "failed to get max surge")
// TODO: do we return early here?
}

shouldPatchCustomData := false
if m.HasReplicasExternallyManaged(ctx) {
shouldPatchCustomData, err := m.HasBootstrapDataChanges(ctx)
if err != nil {
log.Error(err, "failed to check for bootstrap data changes")
// return nil, errors.Wrap(err, "unable to calculate custom data hash")
}
if shouldPatchCustomData {
log.V(4).Info("custom data changed")
} else {
log.V(4).Info("custom data unchanged")
}
}

return &scalesets.ScaleSetSpec{
Name: m.Name(),
Size: m.AzureMachinePool.Spec.Template.VMSize,
Capacity: int64(pointer.Int32Deref(m.MachinePool.Spec.Replicas, 0)),
Expand All @@ -142,6 +221,17 @@ func (m *MachinePoolScope) ScaleSetSpec() azure.ScaleSetSpec {
NetworkInterfaces: m.AzureMachinePool.Spec.Template.NetworkInterfaces,
IPv6Enabled: m.IsIPv6Enabled(),
OrchestrationMode: m.AzureMachinePool.Spec.OrchestrationMode,
Location: m.AzureMachinePool.Spec.Location,
SubscriptionID: m.SubscriptionID(),
VMSSExtensionSpecs: m.VMSSExtensionSpecs(),
VMImage: vmImage,
BootstrapData: bootstrapData,
ClusterName: m.ClusterName(),
AdditionalTags: m.AzureMachinePool.Spec.AdditionalTags,
MaxSurge: maxSurge,
SKU: m.cache.VMSKU,
ShouldPatchCustomData: shouldPatchCustomData,
// VMSSInstances []compute.VirtualMachineScaleSetVM
}
}

Expand Down
16 changes: 13 additions & 3 deletions azure/services/roleassignments/roleassignments.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ type Service struct {
Scope RoleAssignmentScope
virtualMachinesGetter async.Getter
async.Reconciler
virtualMachineScaleSetClient scalesets.Client
virtualMachineScaleSetGetter async.Getter
}

// New creates a new service.
Expand All @@ -56,7 +56,7 @@ func New(scope RoleAssignmentScope) *Service {
return &Service{
Scope: scope,
virtualMachinesGetter: virtualmachines.NewClient(scope),
virtualMachineScaleSetClient: scalesets.NewClient(scope),
virtualMachineScaleSetGetter: scalesets.NewClient(scope),
Reconciler: async.New(scope, client, client),
}
}
Expand Down Expand Up @@ -141,10 +141,20 @@ func (s *Service) getVMSSPrincipalID(ctx context.Context) (*string, error) {
ctx, log, done := tele.StartSpanWithLogger(ctx, "roleassignments.Service.getVMPrincipalID")
defer done()
log.V(2).Info("fetching principal ID for VMSS")
resultVMSS, err := s.virtualMachineScaleSetClient.Get(ctx, s.Scope.ResourceGroup(), s.Scope.Name())
spec := &scalesets.ScaleSetSpec{
Name: s.Scope.Name(),
ResourceGroup: s.Scope.ResourceGroup(),
}

resultVMSSIface, err := s.virtualMachineScaleSetGetter.Get(ctx, spec)
if err != nil {
return nil, errors.Wrap(err, "failed to get principal ID for VMSS")
}
resultVMSS, ok := resultVMSSIface.(compute.VirtualMachineScaleSet)
if !ok {
return nil, errors.Errorf("%T is not a compute.VirtualMachine", resultVMSSIface)
}

return resultVMSS.Identity.PrincipalID, nil
}

Expand Down
13 changes: 9 additions & 4 deletions azure/services/roleassignments/roleassignments_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
"sigs.k8s.io/cluster-api-provider-azure/azure"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/async/mock_async"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/roleassignments/mock_roleassignments"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/scalesets"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/scalesets/mock_scalesets"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/virtualmachines"
gomockinternal "sigs.k8s.io/cluster-api-provider-azure/internal/test/matchers/gomock"
Expand All @@ -55,6 +56,10 @@ var (

emptyRoleAssignmentSpec = RoleAssignmentSpec{}
fakeRoleAssignmentSpecs = []azure.ResourceSpecGetter{&fakeRoleAssignment1, &fakeRoleAssignment2, &emptyRoleAssignmentSpec}
fakeVMSSSpec = scalesets.ScaleSetSpec{
Name: "test-vmss",
ResourceGroup: "my-rg",
}
)

func TestReconcileRoleAssignmentsVM(t *testing.T) {
Expand Down Expand Up @@ -169,7 +174,7 @@ func TestReconcileRoleAssignmentsVMSS(t *testing.T) {
s.RoleAssignmentResourceType().Return(azure.VirtualMachineScaleSet)
s.ResourceGroup().Return("my-rg")
s.Name().Return("test-vmss")
mvmss.Get(gomockinternal.AContext(), "my-rg", "test-vmss").Return(compute.VirtualMachineScaleSet{
mvmss.Get(gomockinternal.AContext(), &fakeVMSSSpec).Return(compute.VirtualMachineScaleSet{
Identity: &compute.VirtualMachineScaleSetIdentity{
PrincipalID: &fakePrincipalID,
},
Expand All @@ -187,7 +192,7 @@ func TestReconcileRoleAssignmentsVMSS(t *testing.T) {
s.ResourceGroup().Return("my-rg")
s.Name().Return("test-vmss")
s.HasSystemAssignedIdentity().Return(true)
mvmss.Get(gomockinternal.AContext(), "my-rg", "test-vmss").Return(compute.VirtualMachineScaleSet{},
mvmss.Get(gomockinternal.AContext(), &fakeVMSSSpec).Return(compute.VirtualMachineScaleSet{},
autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: http.StatusInternalServerError}, "Internal Server Error"))
},
},
Expand All @@ -202,7 +207,7 @@ func TestReconcileRoleAssignmentsVMSS(t *testing.T) {
s.RoleAssignmentResourceType().Return(azure.VirtualMachineScaleSet)
s.ResourceGroup().Return("my-rg")
s.Name().Return("test-vmss")
mvmss.Get(gomockinternal.AContext(), "my-rg", "test-vmss").Return(compute.VirtualMachineScaleSet{
mvmss.Get(gomockinternal.AContext(), &fakeVMSSSpec).Return(compute.VirtualMachineScaleSet{
Identity: &compute.VirtualMachineScaleSetIdentity{
PrincipalID: &fakePrincipalID,
},
Expand All @@ -229,7 +234,7 @@ func TestReconcileRoleAssignmentsVMSS(t *testing.T) {
s := &Service{
Scope: scopeMock,
Reconciler: asyncMock,
virtualMachineScaleSetClient: vmMock,
virtualMachineScaleSetGetter: vmMock,
}

err := s.Reconcile(context.TODO())
Expand Down
Loading

0 comments on commit 83aa946

Please sign in to comment.