Skip to content

Commit

Permalink
Assign role for system-assigned identity in machine pool
Browse files Browse the repository at this point in the history
  • Loading branch information
shysank committed Oct 26, 2020
1 parent a834a90 commit af862a5
Show file tree
Hide file tree
Showing 12 changed files with 291 additions and 46 deletions.
5 changes: 3 additions & 2 deletions cloud/scope/machine.go
Original file line number Diff line number Diff line change
Expand Up @@ -225,8 +225,9 @@ func (m *MachineScope) RoleAssignmentSpecs() []azure.RoleAssignmentSpec {
if m.AzureMachine.Spec.Identity == infrav1.VMIdentitySystemAssigned {
return []azure.RoleAssignmentSpec{
{
MachineName: m.Name(),
Name: m.AzureMachine.Spec.RoleAssignmentName,
MachineName: m.Name(),
Name: m.AzureMachine.Spec.RoleAssignmentName,
ResourceType: azure.VirtualMachine,
},
}
}
Expand Down
14 changes: 14 additions & 0 deletions cloud/scope/machinepool.go
Original file line number Diff line number Diff line change
Expand Up @@ -311,6 +311,20 @@ func (m *MachinePoolScope) GetVMImage() (*infrav1.Image, error) {
return azure.GetDefaultUbuntuImage(to.String(m.MachinePool.Spec.Template.Spec.Version))
}

// RoleAssignmentSpecs returns the role assignment specs.
func (m *MachinePoolScope) RoleAssignmentSpecs() []azure.RoleAssignmentSpec {
if m.AzureMachinePool.Spec.Identity == infrav1.VMIdentitySystemAssigned {
return []azure.RoleAssignmentSpec{
{
MachineName: m.Name(),
Name: m.AzureMachinePool.Spec.RoleAssignmentName,
ResourceType: azure.VirtualMachineScaleSet,
},
}
}
return []azure.RoleAssignmentSpec{}
}

func (m *MachinePoolScope) getNodeStatusByProviderID(ctx context.Context, providerIDList []string) (map[string]*NodeStatus, error) {
nodeStatusMap := map[string]*NodeStatus{}
for _, id := range providerIDList {
Expand Down
72 changes: 55 additions & 17 deletions cloud/services/roleassignments/roleassignments.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,37 +22,75 @@ import (
"github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/authorization/mgmt/authorization"
"github.com/Azure/go-autorest/autorest/to"
"github.com/pkg/errors"
azure "sigs.k8s.io/cluster-api-provider-azure/cloud"
)

const azureBuiltInContributorID = "b24988ac-6180-42a0-ab88-20f7382dd24c"

// Reconcile creates a role assignment.
func (s *Service) Reconcile(ctx context.Context) error {
for _, roleSpec := range s.Scope.RoleAssignmentSpecs() {
resultVM, err := s.VirtualMachinesClient.Get(ctx, s.Scope.ResourceGroup(), roleSpec.MachineName)
if err != nil {
return errors.Wrapf(err, "cannot get VM to assign role to system assigned identity")
}

scope := fmt.Sprintf("/subscriptions/%s/", s.Scope.SubscriptionID())
// Azure built-in roles https://docs.microsoft.com/en-us/azure/role-based-access-control/built-in-roles
contributorRoleDefinitionID := fmt.Sprintf("/subscriptions/%s/providers/Microsoft.Authorization/roleDefinitions/%s", s.Scope.SubscriptionID(), azureBuiltInContributorID)
params := authorization.RoleAssignmentCreateParameters{
Properties: &authorization.RoleAssignmentProperties{
RoleDefinitionID: to.StringPtr(contributorRoleDefinitionID),
PrincipalID: resultVM.Identity.PrincipalID,
},
}
_, err = s.Client.Create(ctx, scope, roleSpec.Name, params)
if err != nil {
return errors.Wrapf(err, "cannot assign role to VM system assigned identity")
switch roleSpec.ResourceType {
case azure.VirtualMachine:
return s.reconcileVM(ctx, roleSpec)
case azure.VirtualMachineScaleSet:
return s.reconcileVMSS(ctx, roleSpec)
default:
return errors.Errorf("unexpected resource type %q. Expected one of [%s, %s]", roleSpec.ResourceType,
azure.VirtualMachine, azure.VirtualMachineScaleSet)
}

s.Scope.V(2).Info("successfully created role assignment for generated Identity for VM", "virtual machine", roleSpec.MachineName)
}
return nil
}

func (s *Service) reconcileVM(ctx context.Context, roleSpec azure.RoleAssignmentSpec) error {
resultVM, err := s.VirtualMachinesClient.Get(ctx, s.Scope.ResourceGroup(), roleSpec.MachineName)
if err != nil {
return errors.Wrapf(err, "cannot get VM to assign role to system assigned identity")
}

err = s.assignRole(ctx, roleSpec.Name, resultVM.Identity.PrincipalID)
if err != nil {
return errors.Wrapf(err, "cannot assign role to VM system assigned identity")
}

s.Scope.V(2).Info("successfully created role assignment for generated Identity for VM", "virtual machine", roleSpec.MachineName)

return nil
}

func (s *Service) reconcileVMSS(ctx context.Context, roleSpec azure.RoleAssignmentSpec) error {
resultVMSS, err := s.VirtualMachineScaleSetClient.Get(ctx, s.Scope.ResourceGroup(), roleSpec.MachineName)
if err != nil {
return errors.Wrapf(err, "cannot get VMSS to assign role to system assigned identity")
}

err = s.assignRole(ctx, roleSpec.Name, resultVMSS.Identity.PrincipalID)
if err != nil {
return errors.Wrapf(err, "cannot assign role to VMSS system assigned identity")
}

s.Scope.V(2).Info("successfully created role assignment for generated Identity for VMSS", "virtual machine scale set", roleSpec.MachineName)

return nil
}

func (s *Service) assignRole(ctx context.Context, roleAssignmentName string, principalID *string) error {
scope := fmt.Sprintf("/subscriptions/%s/", s.Scope.SubscriptionID())
// Azure built-in roles https://docs.microsoft.com/en-us/azure/role-based-access-control/built-in-roles
contributorRoleDefinitionID := fmt.Sprintf("/subscriptions/%s/providers/Microsoft.Authorization/roleDefinitions/%s", s.Scope.SubscriptionID(), azureBuiltInContributorID)
params := authorization.RoleAssignmentCreateParameters{
Properties: &authorization.RoleAssignmentProperties{
RoleDefinitionID: to.StringPtr(contributorRoleDefinitionID),
PrincipalID: principalID,
},
}
_, err := s.Client.Create(ctx, scope, roleAssignmentName, params)
return err
}

// Delete is a no-op as the role assignments get deleted as part of VM deletion.
func (s *Service) Delete(ctx context.Context) error {
return nil
Expand Down
119 changes: 112 additions & 7 deletions cloud/services/roleassignments/roleassignments_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,22 +18,24 @@ package roleassignments

import (
"context"
"net/http"
"testing"

"github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/authorization/mgmt/authorization"
"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-06-01/compute"
"github.com/Azure/go-autorest/autorest"
"github.com/Azure/go-autorest/autorest/to"
"github.com/golang/mock/gomock"
. "github.com/onsi/gomega"

"k8s.io/klog/klogr"
"net/http"
azure "sigs.k8s.io/cluster-api-provider-azure/cloud"
"testing"

"sigs.k8s.io/cluster-api-provider-azure/cloud/services/roleassignments/mock_roleassignments"
"sigs.k8s.io/cluster-api-provider-azure/cloud/services/scalesets/mock_scalesets"
"sigs.k8s.io/cluster-api-provider-azure/cloud/services/virtualmachines/mock_virtualmachines"
)

func TestReconcileRoleAssignments(t *testing.T) {
func TestReconcileRoleAssignmentsVM(t *testing.T) {
testcases := []struct {
name string
expect func(s *mock_roleassignments.MockRoleAssignmentScopeMockRecorder, m *mock_roleassignments.MockClientMockRecorder, v *mock_virtualmachines.MockClientMockRecorder)
Expand All @@ -48,7 +50,8 @@ func TestReconcileRoleAssignments(t *testing.T) {
s.ResourceGroup().Return("my-rg")
s.RoleAssignmentSpecs().Return([]azure.RoleAssignmentSpec{
{
MachineName: "test-vm",
MachineName: "test-vm",
ResourceType: azure.VirtualMachine,
},
})
v.Get(context.TODO(), "my-rg", "test-vm").Return(compute.VirtualMachine{
Expand All @@ -73,7 +76,8 @@ func TestReconcileRoleAssignments(t *testing.T) {
s.ResourceGroup().Return("my-rg")
s.RoleAssignmentSpecs().Return([]azure.RoleAssignmentSpec{
{
MachineName: "test-vm",
MachineName: "test-vm",
ResourceType: azure.VirtualMachine,
},
})
v.Get(context.TODO(), "my-rg", "test-vm").Return(compute.VirtualMachine{}, autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: 500}, "Internal Server Error"))
Expand All @@ -88,7 +92,8 @@ func TestReconcileRoleAssignments(t *testing.T) {
s.ResourceGroup().Return("my-rg")
s.RoleAssignmentSpecs().Return([]azure.RoleAssignmentSpec{
{
MachineName: "test-vm",
MachineName: "test-vm",
ResourceType: azure.VirtualMachine,
},
})
v.Get(context.TODO(), "my-rg", "test-vm").Return(compute.VirtualMachine{
Expand Down Expand Up @@ -130,3 +135,103 @@ func TestReconcileRoleAssignments(t *testing.T) {
})
}
}
func TestReconcileRoleAssignmentsVMSS(t *testing.T) {
testcases := []struct {
name string
expect func(s *mock_roleassignments.MockRoleAssignmentScopeMockRecorder, m *mock_roleassignments.MockClientMockRecorder, v *mock_scalesets.MockClientMockRecorder)
expectedError string
}{
{
name: "create a role assignment",
expectedError: "",
expect: func(s *mock_roleassignments.MockRoleAssignmentScopeMockRecorder, m *mock_roleassignments.MockClientMockRecorder, v *mock_scalesets.MockClientMockRecorder) {
s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New())
s.SubscriptionID().AnyTimes().Return("12345")
s.ResourceGroup().Return("my-rg")
s.RoleAssignmentSpecs().Return([]azure.RoleAssignmentSpec{
{
MachineName: "test-vmss",
ResourceType: azure.VirtualMachineScaleSet,
},
})
v.Get(context.TODO(), "my-rg", "test-vmss").Return(compute.VirtualMachineScaleSet{
Identity: &compute.VirtualMachineScaleSetIdentity{
PrincipalID: to.StringPtr("000"),
},
}, nil)
m.Create(context.TODO(), "/subscriptions/12345/", gomock.AssignableToTypeOf("uuid"), gomock.AssignableToTypeOf(authorization.RoleAssignmentCreateParameters{
Properties: &authorization.RoleAssignmentProperties{
RoleDefinitionID: to.StringPtr("/subscriptions/12345/providers/Microsoft.Authorization/roleDefinitions/b24988ac-6180-42a0-ab88-20f7382dd24c"),
PrincipalID: to.StringPtr("000"),
},
}))
},
},
{
name: "error getting VMSS",
expectedError: "cannot get VMSS to assign role to system assigned identity: #: Internal Server Error: StatusCode=500",
expect: func(s *mock_roleassignments.MockRoleAssignmentScopeMockRecorder, m *mock_roleassignments.MockClientMockRecorder, v *mock_scalesets.MockClientMockRecorder) {
s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New())
s.SubscriptionID().AnyTimes().Return("12345")
s.ResourceGroup().Return("my-rg")
s.RoleAssignmentSpecs().Return([]azure.RoleAssignmentSpec{
{
MachineName: "test-vmss",
ResourceType: azure.VirtualMachineScaleSet,
},
})
v.Get(context.TODO(), "my-rg", "test-vmss").Return(compute.VirtualMachineScaleSet{}, autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: 500}, "Internal Server Error"))
},
},
{
name: "return error when creating a role assignment",
expectedError: "cannot assign role to VMSS system assigned identity: #: Internal Server Error: StatusCode=500",
expect: func(s *mock_roleassignments.MockRoleAssignmentScopeMockRecorder, m *mock_roleassignments.MockClientMockRecorder, v *mock_scalesets.MockClientMockRecorder) {
s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New())
s.SubscriptionID().AnyTimes().Return("12345")
s.ResourceGroup().Return("my-rg")
s.RoleAssignmentSpecs().Return([]azure.RoleAssignmentSpec{
{
MachineName: "test-vmss",
ResourceType: azure.VirtualMachineScaleSet,
},
})
v.Get(context.TODO(), "my-rg", "test-vmss").Return(compute.VirtualMachineScaleSet{
Identity: &compute.VirtualMachineScaleSetIdentity{
PrincipalID: to.StringPtr("000"),
},
}, nil)
m.Create(context.TODO(), "/subscriptions/12345/", gomock.AssignableToTypeOf("uuid"), gomock.AssignableToTypeOf(authorization.RoleAssignmentCreateParameters{})).Return(authorization.RoleAssignment{}, autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: 500}, "Internal Server Error"))
},
},
}

for _, tc := range testcases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
g := NewWithT(t)
t.Parallel()
mockCtrl := gomock.NewController(t)
defer mockCtrl.Finish()
scopeMock := mock_roleassignments.NewMockRoleAssignmentScope(mockCtrl)
clientMock := mock_roleassignments.NewMockClient(mockCtrl)
vmssMock := mock_scalesets.NewMockClient(mockCtrl)

tc.expect(scopeMock.EXPECT(), clientMock.EXPECT(), vmssMock.EXPECT())

s := &Service{
Scope: scopeMock,
Client: clientMock,
VirtualMachineScaleSetClient: vmssMock,
}

err := s.Reconcile(context.TODO())
if tc.expectedError != "" {
g.Expect(err).To(HaveOccurred())
g.Expect(err).To(MatchError(tc.expectedError))
} else {
g.Expect(err).NotTo(HaveOccurred())
}
})
}
}
11 changes: 7 additions & 4 deletions cloud/services/roleassignments/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package roleassignments
import (
"github.com/go-logr/logr"
azure "sigs.k8s.io/cluster-api-provider-azure/cloud"
"sigs.k8s.io/cluster-api-provider-azure/cloud/services/scalesets"
"sigs.k8s.io/cluster-api-provider-azure/cloud/services/virtualmachines"
)

Expand All @@ -33,14 +34,16 @@ type RoleAssignmentScope interface {
type Service struct {
Scope RoleAssignmentScope
Client
VirtualMachinesClient virtualmachines.Client
VirtualMachinesClient virtualmachines.Client
VirtualMachineScaleSetClient scalesets.Client
}

// NewService creates a new service.
func NewService(scope RoleAssignmentScope) *Service {
return &Service{
Scope: scope,
Client: NewClient(scope),
VirtualMachinesClient: virtualmachines.NewClient(scope),
Scope: scope,
Client: NewClient(scope),
VirtualMachinesClient: virtualmachines.NewClient(scope),
VirtualMachineScaleSetClient: scalesets.NewClient(scope),
}
}
18 changes: 16 additions & 2 deletions cloud/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,10 +98,24 @@ type VNetSpec struct {

// RoleAssignmentSpec defines the specification for a Role Assignment.
type RoleAssignmentSpec struct {
MachineName string
Name string
MachineName string
Name string
ResourceType string
}

// ResourceType defines the type azure resource being reconciled.
// Eg. Virtual Machine, Virtual Machine Scale Sets
type ResourceType string

const (

// VirtualMachine ...
VirtualMachine = "VirtualMachine"

// VirtualMachineScaleSet ...
VirtualMachineScaleSet = "VirtualMachineScaleSet"
)

// NSGSpec defines the specification for a Security Group.
type NSGSpec struct {
Name string
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,11 @@ spec:
items:
type: string
type: array
roleAssignmentName:
description: RoleAssignmentName is the name of the role assignment
to create for a system assigned identity. It can be any valid GUID.
If not specified, a random GUID will be generated.
type: string
template:
description: Template contains the details used to build a replica
virtual machine within the Machine Pool
Expand Down
2 changes: 1 addition & 1 deletion exp/api/v1alpha3/azuremachinepool_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ func TestAzureMachinePool_Validate(t *testing.T) {
t.Parallel()
g := gomega.NewGomegaWithT(t)
amp := c.Factory(g)
actualErr := amp.Validate()
actualErr := amp.Validate(nil)
c.Expect(g, actualErr)
})
}
Expand Down
5 changes: 5 additions & 0 deletions exp/api/v1alpha3/azuremachinepool_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,11 @@ type (
// See https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-manage-ua-identity-cli
// +optional
UserAssignedIdentities []infrav1.UserAssignedIdentity `json:"userAssignedIdentities,omitempty"`

// RoleAssignmentName is the name of the role assignment to create for a system assigned identity. It can be any valid GUID.
// If not specified, a random GUID will be generated.
// +optional
RoleAssignmentName string `json:"roleAssignmentName,omitempty"`
}

// AzureMachinePoolStatus defines the observed state of AzureMachinePool
Expand Down
Loading

0 comments on commit af862a5

Please sign in to comment.