From 9e0fad0e60e26a3f35a44df2b44a4fdea7445c7b Mon Sep 17 00:00:00 2001 From: Cecile Robert-Michon Date: Thu, 17 Dec 2020 15:41:06 -0700 Subject: [PATCH] Add Linux VM and VMSS extensions --- cloud/defaults.go | 14 + cloud/scope/machine.go | 16 + cloud/scope/machinepool.go | 16 + cloud/services/scalesets/scalesets.go | 1 + cloud/services/scalesets/scalesets_test.go | 1 + cloud/services/vmextensions/client.go | 86 +++++ .../mock_vmextensions/client_mock.go | 79 +++++ .../vmextensions/mock_vmextensions/doc.go | 22 ++ .../mock_vmextensions/vmextensions_mock.go | 329 ++++++++++++++++++ cloud/services/vmextensions/vmextensions.go | 86 +++++ .../vmextensions/vmextensions_test.go | 120 +++++++ cloud/services/vmssextensions/client.go | 86 +++++ .../mock_vmssextensions/client_mock.go | 79 +++++ .../vmssextensions/mock_vmssextensions/doc.go | 22 ++ .../vmssextensions_mock.go | 329 ++++++++++++++++++ .../services/vmssextensions/vmssextensions.go | 85 +++++ .../vmssextensions/vmssextensions_test.go | 120 +++++++ cloud/types.go | 16 + controllers/azuremachine_reconciler.go | 7 + .../azuremachinepool_reconciler.go | 8 + go.sum | 1 + 21 files changed, 1523 insertions(+) create mode 100644 cloud/services/vmextensions/client.go create mode 100644 cloud/services/vmextensions/mock_vmextensions/client_mock.go create mode 100644 cloud/services/vmextensions/mock_vmextensions/doc.go create mode 100644 cloud/services/vmextensions/mock_vmextensions/vmextensions_mock.go create mode 100644 cloud/services/vmextensions/vmextensions.go create mode 100644 cloud/services/vmextensions/vmextensions_test.go create mode 100644 cloud/services/vmssextensions/client.go create mode 100644 cloud/services/vmssextensions/mock_vmssextensions/client_mock.go create mode 100644 cloud/services/vmssextensions/mock_vmssextensions/doc.go create mode 100644 cloud/services/vmssextensions/mock_vmssextensions/vmssextensions_mock.go create mode 100644 cloud/services/vmssextensions/vmssextensions.go create mode 100644 cloud/services/vmssextensions/vmssextensions_test.go diff --git a/cloud/defaults.go b/cloud/defaults.go index 99a2d5e85e29..8c4610d1f25a 100644 --- a/cloud/defaults.go +++ b/cloud/defaults.go @@ -19,6 +19,8 @@ package azure import ( "fmt" + "github.com/Azure/go-autorest/autorest/azure" + "github.com/Azure/go-autorest/autorest" "github.com/blang/semver" "github.com/pkg/errors" @@ -246,6 +248,18 @@ func GetDefaultWindowsImage(k8sVersion string) (*infrav1.Image, error) { return defaultImage, nil } +// GetBootstrappingVMExtension returns the CAPZ Bootstrapping VM extension. +// The CAPZ Bootstrapping extension is a simple clone of https://github.com/Azure/custom-script-extension-linux which allows running arbitrary scripts on the VM. +// Its role is to detect and report Kubernetes bootstrap failure or success. +func GetBootstrappingVMExtension(osType string, cloud string) (name, publisher, version string) { + // currently, the bootstrap extension is only available for Linux and in AzurePublicCloud. + if osType == "Linux" && cloud == azure.PublicCloud.Name { + return "CAPZ.Linux.Bootstrapping", "Microsoft.Azure.ContainerUpstream", "1.0" + } + + return "", "", "" +} + // UserAgent specifies a string to append to the agent identifier. func UserAgent() string { return fmt.Sprintf("cluster-api-provider-azure/%s", version.Get().String()) diff --git a/cloud/scope/machine.go b/cloud/scope/machine.go index 0b6f06dd057d..651900b9bab7 100644 --- a/cloud/scope/machine.go +++ b/cloud/scope/machine.go @@ -231,6 +231,22 @@ func (m *MachineScope) RoleAssignmentSpecs() []azure.RoleAssignmentSpec { return []azure.RoleAssignmentSpec{} } +// VMExtensionSpecs returns the vm extension specs. +func (m *MachineScope) VMExtensionSpecs() []azure.VMExtensionSpec { + name, publisher, version := azure.GetBootstrappingVMExtension(m.AzureMachine.Spec.OSDisk.OSType, m.CloudEnvironment()) + if name != "" { + return []azure.VMExtensionSpec{ + { + Name: name, + VMName: m.Name(), + Publisher: publisher, + Version: version, + }, + } + } + return []azure.VMExtensionSpec{} +} + // Subnet returns the machine's subnet based on its role func (m *MachineScope) Subnet() *infrav1.SubnetSpec { if m.IsControlPlane() { diff --git a/cloud/scope/machinepool.go b/cloud/scope/machinepool.go index a41d6fb13f0f..6c56edb3939b 100644 --- a/cloud/scope/machinepool.go +++ b/cloud/scope/machinepool.go @@ -356,6 +356,22 @@ func (m *MachinePoolScope) RoleAssignmentSpecs() []azure.RoleAssignmentSpec { return []azure.RoleAssignmentSpec{} } +// VMSSExtensionSpecs returns the vmss extension specs. +func (m *MachinePoolScope) VMSSExtensionSpecs() []azure.VMSSExtensionSpec { + name, publisher, version := azure.GetBootstrappingVMExtension(m.AzureMachinePool.Spec.Template.OSDisk.OSType, m.CloudEnvironment()) + if name != "" { + return []azure.VMSSExtensionSpec{ + { + Name: name, + ScaleSetName: m.Name(), + Publisher: publisher, + Version: version, + }, + } + } + return []azure.VMSSExtensionSpec{} +} + func getAzureMachineTemplate(ctx context.Context, c client.Client, name, namespace string) (*infrav1.AzureMachineTemplate, error) { ctx, span := tele.Tracer().Start(ctx, "scope.MachinePoolScope.getAzureMachineTemplate") defer span.End() diff --git a/cloud/services/scalesets/scalesets.go b/cloud/services/scalesets/scalesets.go index 0358ae1d01a5..a177cb1232da 100644 --- a/cloud/services/scalesets/scalesets.go +++ b/cloud/services/scalesets/scalesets.go @@ -368,6 +368,7 @@ func (s *Service) buildVMSSFromSpec(ctx context.Context, vmssSpec azure.ScaleSet UpgradePolicy: &compute.UpgradePolicy{ Mode: compute.UpgradeModeManual, }, + DoNotRunExtensionsOnOverprovisionedVMs: to.BoolPtr(true), VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{ OsProfile: osProfile, StorageProfile: storageProfile, diff --git a/cloud/services/scalesets/scalesets_test.go b/cloud/services/scalesets/scalesets_test.go index 15e8f4739e16..a8a4a7dfef63 100644 --- a/cloud/services/scalesets/scalesets_test.go +++ b/cloud/services/scalesets/scalesets_test.go @@ -800,6 +800,7 @@ func newDefaultVMSS() compute.VirtualMachineScaleSet { UpgradePolicy: &compute.UpgradePolicy{ Mode: compute.UpgradeModeRolling, }, + DoNotRunExtensionsOnOverprovisionedVMs: to.BoolPtr(true), VirtualMachineProfile: &compute.VirtualMachineScaleSetVMProfile{ OsProfile: &compute.VirtualMachineScaleSetOSProfile{ ComputerNamePrefix: to.StringPtr(defaultVMSSName), diff --git a/cloud/services/vmextensions/client.go b/cloud/services/vmextensions/client.go new file mode 100644 index 000000000000..f4d87f82fb00 --- /dev/null +++ b/cloud/services/vmextensions/client.go @@ -0,0 +1,86 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vmextensions + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-06-30/compute" + "github.com/Azure/go-autorest/autorest" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + "sigs.k8s.io/cluster-api-provider-azure/util/tele" +) + +// Client wraps go-sdk +type client interface { + CreateOrUpdate(context.Context, string, string, string, compute.VirtualMachineExtension) error + Delete(context.Context, string, string, string) error +} + +// AzureClient contains the Azure go-sdk Client +type azureClient struct { + vmextensions compute.VirtualMachineExtensionsClient +} + +var _ client = (*azureClient)(nil) + +// newClient creates a new VM client from subscription ID. +func newClient(auth azure.Authorizer) *azureClient { + c := newVirtualMachineExtensionsClient(auth.SubscriptionID(), auth.BaseURI(), auth.Authorizer()) + return &azureClient{c} +} + +// newVirtualMachineExtensionsClient creates a new vm extension client from subscription ID. +func newVirtualMachineExtensionsClient(subscriptionID string, baseURI string, authorizer autorest.Authorizer) compute.VirtualMachineExtensionsClient { + vmextensionsClient := compute.NewVirtualMachineExtensionsClientWithBaseURI(baseURI, subscriptionID) + azure.SetAutoRestClientDefaults(&vmextensionsClient.Client, authorizer) + return vmextensionsClient +} + +// CreateOrUpdate creates or updates the virtual machine extension +func (ac *azureClient) CreateOrUpdate(ctx context.Context, resourceGroupName, vmName, name string, parameters compute.VirtualMachineExtension) error { + ctx, span := tele.Tracer().Start(ctx, "vmextensions.AzureClient.CreateOrUpdate") + defer span.End() + + future, err := ac.vmextensions.CreateOrUpdate(ctx, resourceGroupName, vmName, name, parameters) + if err != nil { + return err + } + err = future.WaitForCompletionRef(ctx, ac.vmextensions.Client) + if err != nil { + return err + } + _, err = future.Result(ac.vmextensions) + return err +} + +// Delete removes the virtual machine extension. +func (ac *azureClient) Delete(ctx context.Context, resourceGroupName, vmName, name string) error { + ctx, span := tele.Tracer().Start(ctx, "vmextensions.AzureClient.Delete") + defer span.End() + + future, err := ac.vmextensions.Delete(ctx, resourceGroupName, vmName, name) + if err != nil { + return err + } + err = future.WaitForCompletionRef(ctx, ac.vmextensions.Client) + if err != nil { + return err + } + _, err = future.Result(ac.vmextensions) + return err +} diff --git a/cloud/services/vmextensions/mock_vmextensions/client_mock.go b/cloud/services/vmextensions/mock_vmextensions/client_mock.go new file mode 100644 index 000000000000..3c3964d35e2e --- /dev/null +++ b/cloud/services/vmextensions/mock_vmextensions/client_mock.go @@ -0,0 +1,79 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by MockGen. DO NOT EDIT. +// Source: ../client.go + +// Package mock_vmextensions is a generated GoMock package. +package mock_vmextensions + +import ( + context "context" + compute "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-06-30/compute" + gomock "github.com/golang/mock/gomock" + reflect "reflect" +) + +// Mockclient is a mock of client interface. +type Mockclient struct { + ctrl *gomock.Controller + recorder *MockclientMockRecorder +} + +// MockclientMockRecorder is the mock recorder for Mockclient. +type MockclientMockRecorder struct { + mock *Mockclient +} + +// NewMockclient creates a new mock instance. +func NewMockclient(ctrl *gomock.Controller) *Mockclient { + mock := &Mockclient{ctrl: ctrl} + mock.recorder = &MockclientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *Mockclient) EXPECT() *MockclientMockRecorder { + return m.recorder +} + +// CreateOrUpdate mocks base method. +func (m *Mockclient) CreateOrUpdate(arg0 context.Context, arg1, arg2, arg3 string, arg4 compute.VirtualMachineExtension) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateOrUpdate", arg0, arg1, arg2, arg3, arg4) + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateOrUpdate indicates an expected call of CreateOrUpdate. +func (mr *MockclientMockRecorder) CreateOrUpdate(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdate", reflect.TypeOf((*Mockclient)(nil).CreateOrUpdate), arg0, arg1, arg2, arg3, arg4) +} + +// Delete mocks base method. +func (m *Mockclient) Delete(arg0 context.Context, arg1, arg2, arg3 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockclientMockRecorder) Delete(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*Mockclient)(nil).Delete), arg0, arg1, arg2, arg3) +} diff --git a/cloud/services/vmextensions/mock_vmextensions/doc.go b/cloud/services/vmextensions/mock_vmextensions/doc.go new file mode 100644 index 000000000000..13054e2b2f6b --- /dev/null +++ b/cloud/services/vmextensions/mock_vmextensions/doc.go @@ -0,0 +1,22 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Run go generate to regenerate this mock. +//go:generate ../../../../hack/tools/bin/mockgen -destination client_mock.go -package mock_vmextensions -source ../client.go Client +//go:generate ../../../../hack/tools/bin/mockgen -destination vmextensions_mock.go -package mock_vmextensions -source ../vmextensions.go VMExtensionScope +//go:generate /usr/bin/env bash -c "cat ../../../../hack/boilerplate/boilerplate.generatego.txt client_mock.go > _client_mock.go && mv _client_mock.go client_mock.go" +//go:generate /usr/bin/env bash -c "cat ../../../../hack/boilerplate/boilerplate.generatego.txt vmextensions_mock.go > _vmextensions_mock.go && mv _vmextensions_mock.go vmextensions_mock.go" +package mock_vmextensions //nolint diff --git a/cloud/services/vmextensions/mock_vmextensions/vmextensions_mock.go b/cloud/services/vmextensions/mock_vmextensions/vmextensions_mock.go new file mode 100644 index 000000000000..a6c571289b92 --- /dev/null +++ b/cloud/services/vmextensions/mock_vmextensions/vmextensions_mock.go @@ -0,0 +1,329 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by MockGen. DO NOT EDIT. +// Source: ../vmextensions.go + +// Package mock_vmextensions is a generated GoMock package. +package mock_vmextensions + +import ( + autorest "github.com/Azure/go-autorest/autorest" + logr "github.com/go-logr/logr" + gomock "github.com/golang/mock/gomock" + reflect "reflect" + v1alpha3 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" +) + +// MockVMExtensionScope is a mock of VMExtensionScope interface. +type MockVMExtensionScope struct { + ctrl *gomock.Controller + recorder *MockVMExtensionScopeMockRecorder +} + +// MockVMExtensionScopeMockRecorder is the mock recorder for MockVMExtensionScope. +type MockVMExtensionScopeMockRecorder struct { + mock *MockVMExtensionScope +} + +// NewMockVMExtensionScope creates a new mock instance. +func NewMockVMExtensionScope(ctrl *gomock.Controller) *MockVMExtensionScope { + mock := &MockVMExtensionScope{ctrl: ctrl} + mock.recorder = &MockVMExtensionScopeMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockVMExtensionScope) EXPECT() *MockVMExtensionScopeMockRecorder { + return m.recorder +} + +// Info mocks base method. +func (m *MockVMExtensionScope) Info(msg string, keysAndValues ...interface{}) { + m.ctrl.T.Helper() + varargs := []interface{}{msg} + for _, a := range keysAndValues { + varargs = append(varargs, a) + } + m.ctrl.Call(m, "Info", varargs...) +} + +// Info indicates an expected call of Info. +func (mr *MockVMExtensionScopeMockRecorder) Info(msg interface{}, keysAndValues ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{msg}, keysAndValues...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Info", reflect.TypeOf((*MockVMExtensionScope)(nil).Info), varargs...) +} + +// Enabled mocks base method. +func (m *MockVMExtensionScope) Enabled() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Enabled") + ret0, _ := ret[0].(bool) + return ret0 +} + +// Enabled indicates an expected call of Enabled. +func (mr *MockVMExtensionScopeMockRecorder) Enabled() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Enabled", reflect.TypeOf((*MockVMExtensionScope)(nil).Enabled)) +} + +// Error mocks base method. +func (m *MockVMExtensionScope) Error(err error, msg string, keysAndValues ...interface{}) { + m.ctrl.T.Helper() + varargs := []interface{}{err, msg} + for _, a := range keysAndValues { + varargs = append(varargs, a) + } + m.ctrl.Call(m, "Error", varargs...) +} + +// Error indicates an expected call of Error. +func (mr *MockVMExtensionScopeMockRecorder) Error(err, msg interface{}, keysAndValues ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{err, msg}, keysAndValues...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Error", reflect.TypeOf((*MockVMExtensionScope)(nil).Error), varargs...) +} + +// V mocks base method. +func (m *MockVMExtensionScope) V(level int) logr.InfoLogger { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "V", level) + ret0, _ := ret[0].(logr.InfoLogger) + return ret0 +} + +// V indicates an expected call of V. +func (mr *MockVMExtensionScopeMockRecorder) V(level interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "V", reflect.TypeOf((*MockVMExtensionScope)(nil).V), level) +} + +// WithValues mocks base method. +func (m *MockVMExtensionScope) WithValues(keysAndValues ...interface{}) logr.Logger { + m.ctrl.T.Helper() + varargs := []interface{}{} + for _, a := range keysAndValues { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "WithValues", varargs...) + ret0, _ := ret[0].(logr.Logger) + return ret0 +} + +// WithValues indicates an expected call of WithValues. +func (mr *MockVMExtensionScopeMockRecorder) WithValues(keysAndValues ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WithValues", reflect.TypeOf((*MockVMExtensionScope)(nil).WithValues), keysAndValues...) +} + +// WithName mocks base method. +func (m *MockVMExtensionScope) WithName(name string) logr.Logger { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WithName", name) + ret0, _ := ret[0].(logr.Logger) + return ret0 +} + +// WithName indicates an expected call of WithName. +func (mr *MockVMExtensionScopeMockRecorder) WithName(name interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WithName", reflect.TypeOf((*MockVMExtensionScope)(nil).WithName), name) +} + +// SubscriptionID mocks base method. +func (m *MockVMExtensionScope) SubscriptionID() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SubscriptionID") + ret0, _ := ret[0].(string) + return ret0 +} + +// SubscriptionID indicates an expected call of SubscriptionID. +func (mr *MockVMExtensionScopeMockRecorder) SubscriptionID() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SubscriptionID", reflect.TypeOf((*MockVMExtensionScope)(nil).SubscriptionID)) +} + +// ClientID mocks base method. +func (m *MockVMExtensionScope) ClientID() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ClientID") + ret0, _ := ret[0].(string) + return ret0 +} + +// ClientID indicates an expected call of ClientID. +func (mr *MockVMExtensionScopeMockRecorder) ClientID() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientID", reflect.TypeOf((*MockVMExtensionScope)(nil).ClientID)) +} + +// ClientSecret mocks base method. +func (m *MockVMExtensionScope) ClientSecret() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ClientSecret") + ret0, _ := ret[0].(string) + return ret0 +} + +// ClientSecret indicates an expected call of ClientSecret. +func (mr *MockVMExtensionScopeMockRecorder) ClientSecret() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientSecret", reflect.TypeOf((*MockVMExtensionScope)(nil).ClientSecret)) +} + +// CloudEnvironment mocks base method. +func (m *MockVMExtensionScope) CloudEnvironment() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CloudEnvironment") + ret0, _ := ret[0].(string) + return ret0 +} + +// CloudEnvironment indicates an expected call of CloudEnvironment. +func (mr *MockVMExtensionScopeMockRecorder) CloudEnvironment() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CloudEnvironment", reflect.TypeOf((*MockVMExtensionScope)(nil).CloudEnvironment)) +} + +// TenantID mocks base method. +func (m *MockVMExtensionScope) TenantID() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TenantID") + ret0, _ := ret[0].(string) + return ret0 +} + +// TenantID indicates an expected call of TenantID. +func (mr *MockVMExtensionScopeMockRecorder) TenantID() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TenantID", reflect.TypeOf((*MockVMExtensionScope)(nil).TenantID)) +} + +// BaseURI mocks base method. +func (m *MockVMExtensionScope) BaseURI() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "BaseURI") + ret0, _ := ret[0].(string) + return ret0 +} + +// BaseURI indicates an expected call of BaseURI. +func (mr *MockVMExtensionScopeMockRecorder) BaseURI() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BaseURI", reflect.TypeOf((*MockVMExtensionScope)(nil).BaseURI)) +} + +// Authorizer mocks base method. +func (m *MockVMExtensionScope) Authorizer() autorest.Authorizer { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Authorizer") + ret0, _ := ret[0].(autorest.Authorizer) + return ret0 +} + +// Authorizer indicates an expected call of Authorizer. +func (mr *MockVMExtensionScopeMockRecorder) Authorizer() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Authorizer", reflect.TypeOf((*MockVMExtensionScope)(nil).Authorizer)) +} + +// HashKey mocks base method. +func (m *MockVMExtensionScope) HashKey() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "HashKey") + ret0, _ := ret[0].(string) + return ret0 +} + +// HashKey indicates an expected call of HashKey. +func (mr *MockVMExtensionScopeMockRecorder) HashKey() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HashKey", reflect.TypeOf((*MockVMExtensionScope)(nil).HashKey)) +} + +// ResourceGroup mocks base method. +func (m *MockVMExtensionScope) ResourceGroup() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ResourceGroup") + ret0, _ := ret[0].(string) + return ret0 +} + +// ResourceGroup indicates an expected call of ResourceGroup. +func (mr *MockVMExtensionScopeMockRecorder) ResourceGroup() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResourceGroup", reflect.TypeOf((*MockVMExtensionScope)(nil).ResourceGroup)) +} + +// ClusterName mocks base method. +func (m *MockVMExtensionScope) ClusterName() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ClusterName") + ret0, _ := ret[0].(string) + return ret0 +} + +// ClusterName indicates an expected call of ClusterName. +func (mr *MockVMExtensionScopeMockRecorder) ClusterName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClusterName", reflect.TypeOf((*MockVMExtensionScope)(nil).ClusterName)) +} + +// Location mocks base method. +func (m *MockVMExtensionScope) Location() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Location") + ret0, _ := ret[0].(string) + return ret0 +} + +// Location indicates an expected call of Location. +func (mr *MockVMExtensionScopeMockRecorder) Location() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Location", reflect.TypeOf((*MockVMExtensionScope)(nil).Location)) +} + +// AdditionalTags mocks base method. +func (m *MockVMExtensionScope) AdditionalTags() v1alpha3.Tags { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AdditionalTags") + ret0, _ := ret[0].(v1alpha3.Tags) + return ret0 +} + +// AdditionalTags indicates an expected call of AdditionalTags. +func (mr *MockVMExtensionScopeMockRecorder) AdditionalTags() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AdditionalTags", reflect.TypeOf((*MockVMExtensionScope)(nil).AdditionalTags)) +} + +// VMExtensionSpecs mocks base method. +func (m *MockVMExtensionScope) VMExtensionSpecs() []azure.VMExtensionSpec { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "VMExtensionSpecs") + ret0, _ := ret[0].([]azure.VMExtensionSpec) + return ret0 +} + +// VMExtensionSpecs indicates an expected call of VMExtensionSpecs. +func (mr *MockVMExtensionScopeMockRecorder) VMExtensionSpecs() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VMExtensionSpecs", reflect.TypeOf((*MockVMExtensionScope)(nil).VMExtensionSpecs)) +} diff --git a/cloud/services/vmextensions/vmextensions.go b/cloud/services/vmextensions/vmextensions.go new file mode 100644 index 000000000000..e2471db8e1a6 --- /dev/null +++ b/cloud/services/vmextensions/vmextensions.go @@ -0,0 +1,86 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vmextensions + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/profiles/latest/compute/mgmt/compute" + "github.com/Azure/go-autorest/autorest/to" + "github.com/go-logr/logr" + "github.com/pkg/errors" + + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + "sigs.k8s.io/cluster-api-provider-azure/util/tele" +) + +// VMExtensionScope defines the scope interface for a vm extension service. +type VMExtensionScope interface { + logr.Logger + azure.ClusterDescriber + VMExtensionSpecs() []azure.VMExtensionSpec +} + +// Service provides operations on azure resources +type Service struct { + Scope VMExtensionScope + client +} + +// New creates a new vm extension service. +func New(scope VMExtensionScope) *Service { + return &Service{ + Scope: scope, + client: newClient(scope), + } +} + +// Reconcile creates or updates the VM extension. +func (s *Service) Reconcile(ctx context.Context) error { + _, span := tele.Tracer().Start(ctx, "vmextensions.Service.Reconcile") + defer span.End() + + for _, extensionSpec := range s.Scope.VMExtensionSpecs() { + s.Scope.V(2).Info("creating VM extension", "vm extension", extensionSpec.Name) + err := s.client.CreateOrUpdate( + ctx, + s.Scope.ResourceGroup(), + extensionSpec.VMName, + extensionSpec.Name, + compute.VirtualMachineExtension{ + VirtualMachineExtensionProperties: &compute.VirtualMachineExtensionProperties{ + Publisher: to.StringPtr(extensionSpec.Publisher), + Type: to.StringPtr(extensionSpec.Name), + TypeHandlerVersion: to.StringPtr(extensionSpec.Version), + Settings: nil, + ProtectedSettings: nil, + }, + Location: to.StringPtr(s.Scope.Location()), + }, + ) + if err != nil { + return errors.Wrapf(err, "failed to create VM extension %s on VM %s in resource group %s", extensionSpec.Name, extensionSpec.VMName, s.Scope.ResourceGroup()) + } + s.Scope.V(2).Info("successfully created VM extension", "vm extension", extensionSpec.Name) + } + return nil +} + +// Delete is a no-op. Extensions will be deleted as part of VM deletion. +func (s *Service) Delete(ctx context.Context) error { + return nil +} diff --git a/cloud/services/vmextensions/vmextensions_test.go b/cloud/services/vmextensions/vmextensions_test.go new file mode 100644 index 000000000000..fddab690d242 --- /dev/null +++ b/cloud/services/vmextensions/vmextensions_test.go @@ -0,0 +1,120 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vmextensions + +import ( + "context" + "net/http" + "testing" + + "github.com/Azure/azure-sdk-for-go/profiles/latest/compute/mgmt/compute" + "github.com/Azure/go-autorest/autorest" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/vmextensions/mock_vmextensions" + + "github.com/golang/mock/gomock" + . "github.com/onsi/gomega" + "k8s.io/klog/klogr" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + gomockinternal "sigs.k8s.io/cluster-api-provider-azure/internal/test/matchers/gomock" +) + +func TestReconcileVMExtension(t *testing.T) { + testcases := []struct { + name string + expectedError string + expect func(s *mock_vmextensions.MockVMExtensionScopeMockRecorder, m *mock_vmextensions.MockclientMockRecorder) + }{ + { + name: "reconcile multiple extensions", + expectedError: "", + expect: func(s *mock_vmextensions.MockVMExtensionScopeMockRecorder, m *mock_vmextensions.MockclientMockRecorder) { + s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New()) + s.VMExtensionSpecs().Return([]azure.VMExtensionSpec{ + { + Name: "my-extension-1", + VMName: "my-vm", + Publisher: "some-publisher", + Version: "1.0", + }, + { + Name: "other-extension", + VMName: "my-vm", + Publisher: "other-publisher", + Version: "2.0", + }, + }) + s.ResourceGroup().AnyTimes().Return("my-rg") + s.Location().AnyTimes().Return("test-location") + m.CreateOrUpdate(gomockinternal.AContext(), "my-rg", "my-vm", "my-extension-1", gomock.AssignableToTypeOf(compute.VirtualMachineExtension{})) + m.CreateOrUpdate(gomockinternal.AContext(), "my-rg", "my-vm", "other-extension", gomock.AssignableToTypeOf(compute.VirtualMachineExtension{})) + }, + }, + { + name: "error creating the extension", + expectedError: "failed to create VM extension my-extension-1 on VM my-vm in resource group my-rg: #: Internal Server Error: StatusCode=500", + expect: func(s *mock_vmextensions.MockVMExtensionScopeMockRecorder, m *mock_vmextensions.MockclientMockRecorder) { + s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New()) + s.VMExtensionSpecs().Return([]azure.VMExtensionSpec{ + { + Name: "my-extension-1", + VMName: "my-vm", + Publisher: "some-publisher", + Version: "1.0", + }, + { + Name: "other-extension", + VMName: "my-vm", + Publisher: "other-publisher", + Version: "2.0", + }, + }) + s.ResourceGroup().AnyTimes().Return("my-rg") + s.Location().AnyTimes().Return("test-location") + m.CreateOrUpdate(gomockinternal.AContext(), "my-rg", "my-vm", "my-extension-1", gomock.AssignableToTypeOf(compute.VirtualMachineExtension{})).Return(autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: 500}, "Internal Server Error")) + + }, + }, + } + + for _, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + t.Parallel() + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + scopeMock := mock_vmextensions.NewMockVMExtensionScope(mockCtrl) + clientMock := mock_vmextensions.NewMockclient(mockCtrl) + + tc.expect(scopeMock.EXPECT(), clientMock.EXPECT()) + + s := &Service{ + Scope: scopeMock, + client: clientMock, + } + + err := s.Reconcile(context.TODO()) + if tc.expectedError != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err).To(MatchError(tc.expectedError)) + } else { + g.Expect(err).NotTo(HaveOccurred()) + } + }) + } +} diff --git a/cloud/services/vmssextensions/client.go b/cloud/services/vmssextensions/client.go new file mode 100644 index 000000000000..a9a2e368250d --- /dev/null +++ b/cloud/services/vmssextensions/client.go @@ -0,0 +1,86 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vmssextensions + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-06-30/compute" + "github.com/Azure/go-autorest/autorest" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + "sigs.k8s.io/cluster-api-provider-azure/util/tele" +) + +// Client wraps go-sdk +type client interface { + CreateOrUpdate(context.Context, string, string, string, compute.VirtualMachineScaleSetExtension) error + Delete(context.Context, string, string, string) error +} + +// AzureClient contains the Azure go-sdk Client +type azureClient struct { + vmssextensions compute.VirtualMachineScaleSetExtensionsClient +} + +var _ client = (*azureClient)(nil) + +// newClient creates a new VMSS client from subscription ID. +func newClient(auth azure.Authorizer) *azureClient { + c := newVirtualMachineScaleSetExtensionsClient(auth.SubscriptionID(), auth.BaseURI(), auth.Authorizer()) + return &azureClient{c} +} + +// newVirtualMachineScaleSetExtensionsClient creates a new vmss extension client from subscription ID. +func newVirtualMachineScaleSetExtensionsClient(subscriptionID string, baseURI string, authorizer autorest.Authorizer) compute.VirtualMachineScaleSetExtensionsClient { + vmssextensionsClient := compute.NewVirtualMachineScaleSetExtensionsClientWithBaseURI(baseURI, subscriptionID) + azure.SetAutoRestClientDefaults(&vmssextensionsClient.Client, authorizer) + return vmssextensionsClient +} + +// CreateOrUpdate creates or updates the virtual machine scale set extension +func (ac *azureClient) CreateOrUpdate(ctx context.Context, resourceGroupName, vmName, name string, parameters compute.VirtualMachineScaleSetExtension) error { + ctx, span := tele.Tracer().Start(ctx, "vmssextensions.AzureClient.CreateOrUpdate") + defer span.End() + + future, err := ac.vmssextensions.CreateOrUpdate(ctx, resourceGroupName, vmName, name, parameters) + if err != nil { + return err + } + err = future.WaitForCompletionRef(ctx, ac.vmssextensions.Client) + if err != nil { + return err + } + _, err = future.Result(ac.vmssextensions) + return err +} + +// Delete removes the virtual machine scale set extension. +func (ac *azureClient) Delete(ctx context.Context, resourceGroupName, vmName, name string) error { + ctx, span := tele.Tracer().Start(ctx, "vmssextensions.AzureClient.Delete") + defer span.End() + + future, err := ac.vmssextensions.Delete(ctx, resourceGroupName, vmName, name) + if err != nil { + return err + } + err = future.WaitForCompletionRef(ctx, ac.vmssextensions.Client) + if err != nil { + return err + } + _, err = future.Result(ac.vmssextensions) + return err +} diff --git a/cloud/services/vmssextensions/mock_vmssextensions/client_mock.go b/cloud/services/vmssextensions/mock_vmssextensions/client_mock.go new file mode 100644 index 000000000000..10ff8c1ebfa1 --- /dev/null +++ b/cloud/services/vmssextensions/mock_vmssextensions/client_mock.go @@ -0,0 +1,79 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by MockGen. DO NOT EDIT. +// Source: ../client.go + +// Package mock_vmssextensions is a generated GoMock package. +package mock_vmssextensions + +import ( + context "context" + compute "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-06-30/compute" + gomock "github.com/golang/mock/gomock" + reflect "reflect" +) + +// Mockclient is a mock of client interface. +type Mockclient struct { + ctrl *gomock.Controller + recorder *MockclientMockRecorder +} + +// MockclientMockRecorder is the mock recorder for Mockclient. +type MockclientMockRecorder struct { + mock *Mockclient +} + +// NewMockclient creates a new mock instance. +func NewMockclient(ctrl *gomock.Controller) *Mockclient { + mock := &Mockclient{ctrl: ctrl} + mock.recorder = &MockclientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *Mockclient) EXPECT() *MockclientMockRecorder { + return m.recorder +} + +// CreateOrUpdate mocks base method. +func (m *Mockclient) CreateOrUpdate(arg0 context.Context, arg1, arg2, arg3 string, arg4 compute.VirtualMachineScaleSetExtension) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateOrUpdate", arg0, arg1, arg2, arg3, arg4) + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateOrUpdate indicates an expected call of CreateOrUpdate. +func (mr *MockclientMockRecorder) CreateOrUpdate(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateOrUpdate", reflect.TypeOf((*Mockclient)(nil).CreateOrUpdate), arg0, arg1, arg2, arg3, arg4) +} + +// Delete mocks base method. +func (m *Mockclient) Delete(arg0 context.Context, arg1, arg2, arg3 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockclientMockRecorder) Delete(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*Mockclient)(nil).Delete), arg0, arg1, arg2, arg3) +} diff --git a/cloud/services/vmssextensions/mock_vmssextensions/doc.go b/cloud/services/vmssextensions/mock_vmssextensions/doc.go new file mode 100644 index 000000000000..cf635806416a --- /dev/null +++ b/cloud/services/vmssextensions/mock_vmssextensions/doc.go @@ -0,0 +1,22 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Run go generate to regenerate this mock. +//go:generate ../../../../hack/tools/bin/mockgen -destination client_mock.go -package mock_vmssextensions -source ../client.go Client +//go:generate ../../../../hack/tools/bin/mockgen -destination vmssextensions_mock.go -package mock_vmssextensions -source ../vmssextensions.go VMSSExtensionScope +//go:generate /usr/bin/env bash -c "cat ../../../../hack/boilerplate/boilerplate.generatego.txt client_mock.go > _client_mock.go && mv _client_mock.go client_mock.go" +//go:generate /usr/bin/env bash -c "cat ../../../../hack/boilerplate/boilerplate.generatego.txt vmssextensions_mock.go > _vmssextensions_mock.go && mv _vmssextensions_mock.go vmssextensions_mock.go" +package mock_vmssextensions //nolint diff --git a/cloud/services/vmssextensions/mock_vmssextensions/vmssextensions_mock.go b/cloud/services/vmssextensions/mock_vmssextensions/vmssextensions_mock.go new file mode 100644 index 000000000000..3294d9e96fd4 --- /dev/null +++ b/cloud/services/vmssextensions/mock_vmssextensions/vmssextensions_mock.go @@ -0,0 +1,329 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by MockGen. DO NOT EDIT. +// Source: ../vmssextensions.go + +// Package mock_vmssextensions is a generated GoMock package. +package mock_vmssextensions + +import ( + autorest "github.com/Azure/go-autorest/autorest" + logr "github.com/go-logr/logr" + gomock "github.com/golang/mock/gomock" + reflect "reflect" + v1alpha3 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" +) + +// MockVMSSExtensionScope is a mock of VMSSExtensionScope interface. +type MockVMSSExtensionScope struct { + ctrl *gomock.Controller + recorder *MockVMSSExtensionScopeMockRecorder +} + +// MockVMSSExtensionScopeMockRecorder is the mock recorder for MockVMSSExtensionScope. +type MockVMSSExtensionScopeMockRecorder struct { + mock *MockVMSSExtensionScope +} + +// NewMockVMSSExtensionScope creates a new mock instance. +func NewMockVMSSExtensionScope(ctrl *gomock.Controller) *MockVMSSExtensionScope { + mock := &MockVMSSExtensionScope{ctrl: ctrl} + mock.recorder = &MockVMSSExtensionScopeMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockVMSSExtensionScope) EXPECT() *MockVMSSExtensionScopeMockRecorder { + return m.recorder +} + +// Info mocks base method. +func (m *MockVMSSExtensionScope) Info(msg string, keysAndValues ...interface{}) { + m.ctrl.T.Helper() + varargs := []interface{}{msg} + for _, a := range keysAndValues { + varargs = append(varargs, a) + } + m.ctrl.Call(m, "Info", varargs...) +} + +// Info indicates an expected call of Info. +func (mr *MockVMSSExtensionScopeMockRecorder) Info(msg interface{}, keysAndValues ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{msg}, keysAndValues...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Info", reflect.TypeOf((*MockVMSSExtensionScope)(nil).Info), varargs...) +} + +// Enabled mocks base method. +func (m *MockVMSSExtensionScope) Enabled() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Enabled") + ret0, _ := ret[0].(bool) + return ret0 +} + +// Enabled indicates an expected call of Enabled. +func (mr *MockVMSSExtensionScopeMockRecorder) Enabled() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Enabled", reflect.TypeOf((*MockVMSSExtensionScope)(nil).Enabled)) +} + +// Error mocks base method. +func (m *MockVMSSExtensionScope) Error(err error, msg string, keysAndValues ...interface{}) { + m.ctrl.T.Helper() + varargs := []interface{}{err, msg} + for _, a := range keysAndValues { + varargs = append(varargs, a) + } + m.ctrl.Call(m, "Error", varargs...) +} + +// Error indicates an expected call of Error. +func (mr *MockVMSSExtensionScopeMockRecorder) Error(err, msg interface{}, keysAndValues ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{err, msg}, keysAndValues...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Error", reflect.TypeOf((*MockVMSSExtensionScope)(nil).Error), varargs...) +} + +// V mocks base method. +func (m *MockVMSSExtensionScope) V(level int) logr.InfoLogger { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "V", level) + ret0, _ := ret[0].(logr.InfoLogger) + return ret0 +} + +// V indicates an expected call of V. +func (mr *MockVMSSExtensionScopeMockRecorder) V(level interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "V", reflect.TypeOf((*MockVMSSExtensionScope)(nil).V), level) +} + +// WithValues mocks base method. +func (m *MockVMSSExtensionScope) WithValues(keysAndValues ...interface{}) logr.Logger { + m.ctrl.T.Helper() + varargs := []interface{}{} + for _, a := range keysAndValues { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "WithValues", varargs...) + ret0, _ := ret[0].(logr.Logger) + return ret0 +} + +// WithValues indicates an expected call of WithValues. +func (mr *MockVMSSExtensionScopeMockRecorder) WithValues(keysAndValues ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WithValues", reflect.TypeOf((*MockVMSSExtensionScope)(nil).WithValues), keysAndValues...) +} + +// WithName mocks base method. +func (m *MockVMSSExtensionScope) WithName(name string) logr.Logger { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WithName", name) + ret0, _ := ret[0].(logr.Logger) + return ret0 +} + +// WithName indicates an expected call of WithName. +func (mr *MockVMSSExtensionScopeMockRecorder) WithName(name interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WithName", reflect.TypeOf((*MockVMSSExtensionScope)(nil).WithName), name) +} + +// SubscriptionID mocks base method. +func (m *MockVMSSExtensionScope) SubscriptionID() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SubscriptionID") + ret0, _ := ret[0].(string) + return ret0 +} + +// SubscriptionID indicates an expected call of SubscriptionID. +func (mr *MockVMSSExtensionScopeMockRecorder) SubscriptionID() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SubscriptionID", reflect.TypeOf((*MockVMSSExtensionScope)(nil).SubscriptionID)) +} + +// ClientID mocks base method. +func (m *MockVMSSExtensionScope) ClientID() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ClientID") + ret0, _ := ret[0].(string) + return ret0 +} + +// ClientID indicates an expected call of ClientID. +func (mr *MockVMSSExtensionScopeMockRecorder) ClientID() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientID", reflect.TypeOf((*MockVMSSExtensionScope)(nil).ClientID)) +} + +// ClientSecret mocks base method. +func (m *MockVMSSExtensionScope) ClientSecret() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ClientSecret") + ret0, _ := ret[0].(string) + return ret0 +} + +// ClientSecret indicates an expected call of ClientSecret. +func (mr *MockVMSSExtensionScopeMockRecorder) ClientSecret() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientSecret", reflect.TypeOf((*MockVMSSExtensionScope)(nil).ClientSecret)) +} + +// CloudEnvironment mocks base method. +func (m *MockVMSSExtensionScope) CloudEnvironment() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CloudEnvironment") + ret0, _ := ret[0].(string) + return ret0 +} + +// CloudEnvironment indicates an expected call of CloudEnvironment. +func (mr *MockVMSSExtensionScopeMockRecorder) CloudEnvironment() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CloudEnvironment", reflect.TypeOf((*MockVMSSExtensionScope)(nil).CloudEnvironment)) +} + +// TenantID mocks base method. +func (m *MockVMSSExtensionScope) TenantID() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TenantID") + ret0, _ := ret[0].(string) + return ret0 +} + +// TenantID indicates an expected call of TenantID. +func (mr *MockVMSSExtensionScopeMockRecorder) TenantID() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TenantID", reflect.TypeOf((*MockVMSSExtensionScope)(nil).TenantID)) +} + +// BaseURI mocks base method. +func (m *MockVMSSExtensionScope) BaseURI() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "BaseURI") + ret0, _ := ret[0].(string) + return ret0 +} + +// BaseURI indicates an expected call of BaseURI. +func (mr *MockVMSSExtensionScopeMockRecorder) BaseURI() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BaseURI", reflect.TypeOf((*MockVMSSExtensionScope)(nil).BaseURI)) +} + +// Authorizer mocks base method. +func (m *MockVMSSExtensionScope) Authorizer() autorest.Authorizer { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Authorizer") + ret0, _ := ret[0].(autorest.Authorizer) + return ret0 +} + +// Authorizer indicates an expected call of Authorizer. +func (mr *MockVMSSExtensionScopeMockRecorder) Authorizer() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Authorizer", reflect.TypeOf((*MockVMSSExtensionScope)(nil).Authorizer)) +} + +// HashKey mocks base method. +func (m *MockVMSSExtensionScope) HashKey() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "HashKey") + ret0, _ := ret[0].(string) + return ret0 +} + +// HashKey indicates an expected call of HashKey. +func (mr *MockVMSSExtensionScopeMockRecorder) HashKey() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HashKey", reflect.TypeOf((*MockVMSSExtensionScope)(nil).HashKey)) +} + +// ResourceGroup mocks base method. +func (m *MockVMSSExtensionScope) ResourceGroup() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ResourceGroup") + ret0, _ := ret[0].(string) + return ret0 +} + +// ResourceGroup indicates an expected call of ResourceGroup. +func (mr *MockVMSSExtensionScopeMockRecorder) ResourceGroup() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResourceGroup", reflect.TypeOf((*MockVMSSExtensionScope)(nil).ResourceGroup)) +} + +// ClusterName mocks base method. +func (m *MockVMSSExtensionScope) ClusterName() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ClusterName") + ret0, _ := ret[0].(string) + return ret0 +} + +// ClusterName indicates an expected call of ClusterName. +func (mr *MockVMSSExtensionScopeMockRecorder) ClusterName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClusterName", reflect.TypeOf((*MockVMSSExtensionScope)(nil).ClusterName)) +} + +// Location mocks base method. +func (m *MockVMSSExtensionScope) Location() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Location") + ret0, _ := ret[0].(string) + return ret0 +} + +// Location indicates an expected call of Location. +func (mr *MockVMSSExtensionScopeMockRecorder) Location() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Location", reflect.TypeOf((*MockVMSSExtensionScope)(nil).Location)) +} + +// AdditionalTags mocks base method. +func (m *MockVMSSExtensionScope) AdditionalTags() v1alpha3.Tags { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AdditionalTags") + ret0, _ := ret[0].(v1alpha3.Tags) + return ret0 +} + +// AdditionalTags indicates an expected call of AdditionalTags. +func (mr *MockVMSSExtensionScopeMockRecorder) AdditionalTags() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AdditionalTags", reflect.TypeOf((*MockVMSSExtensionScope)(nil).AdditionalTags)) +} + +// VMSSExtensionSpecs mocks base method. +func (m *MockVMSSExtensionScope) VMSSExtensionSpecs() []azure.VMSSExtensionSpec { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "VMSSExtensionSpecs") + ret0, _ := ret[0].([]azure.VMSSExtensionSpec) + return ret0 +} + +// VMSSExtensionSpecs indicates an expected call of VMSSExtensionSpecs. +func (mr *MockVMSSExtensionScopeMockRecorder) VMSSExtensionSpecs() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VMSSExtensionSpecs", reflect.TypeOf((*MockVMSSExtensionScope)(nil).VMSSExtensionSpecs)) +} diff --git a/cloud/services/vmssextensions/vmssextensions.go b/cloud/services/vmssextensions/vmssextensions.go new file mode 100644 index 000000000000..81c56ce72c13 --- /dev/null +++ b/cloud/services/vmssextensions/vmssextensions.go @@ -0,0 +1,85 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vmssextensions + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/profiles/latest/compute/mgmt/compute" + "github.com/Azure/go-autorest/autorest/to" + "github.com/go-logr/logr" + "github.com/pkg/errors" + + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + "sigs.k8s.io/cluster-api-provider-azure/util/tele" +) + +// VMSSExtensionScope defines the scope interface for a vmss extension service. +type VMSSExtensionScope interface { + logr.Logger + azure.ClusterDescriber + VMSSExtensionSpecs() []azure.VMSSExtensionSpec +} + +// Service provides operations on azure resources +type Service struct { + Scope VMSSExtensionScope + client +} + +// New creates a new vm extension service. +func New(scope VMSSExtensionScope) *Service { + return &Service{ + Scope: scope, + client: newClient(scope), + } +} + +// Reconcile creates or updates the VMSS extension. +func (s *Service) Reconcile(ctx context.Context) error { + _, span := tele.Tracer().Start(ctx, "vmssextensions.Service.Reconcile") + defer span.End() + + for _, extensionSpec := range s.Scope.VMSSExtensionSpecs() { + s.Scope.V(2).Info("creating VM extension", "vm extension", extensionSpec.Name) + err := s.client.CreateOrUpdate( + ctx, + s.Scope.ResourceGroup(), + extensionSpec.ScaleSetName, + extensionSpec.Name, + compute.VirtualMachineScaleSetExtension{ + VirtualMachineScaleSetExtensionProperties: &compute.VirtualMachineScaleSetExtensionProperties{ + Publisher: to.StringPtr(extensionSpec.Publisher), + Type: to.StringPtr(extensionSpec.Name), + TypeHandlerVersion: to.StringPtr(extensionSpec.Version), + Settings: nil, + ProtectedSettings: nil, + }, + }, + ) + if err != nil { + return errors.Wrapf(err, "failed to create VM extension %s on scale set %s in resource group %s", extensionSpec.Name, extensionSpec.ScaleSetName, s.Scope.ResourceGroup()) + } + s.Scope.V(2).Info("successfully created VM extension", "vm extension", extensionSpec.Name) + } + return nil +} + +// Delete is a no-op. Extensions will be deleted as part of VMSS deletion. +func (s *Service) Delete(ctx context.Context) error { + return nil +} diff --git a/cloud/services/vmssextensions/vmssextensions_test.go b/cloud/services/vmssextensions/vmssextensions_test.go new file mode 100644 index 000000000000..7639979535dc --- /dev/null +++ b/cloud/services/vmssextensions/vmssextensions_test.go @@ -0,0 +1,120 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vmssextensions + +import ( + "context" + "net/http" + "testing" + + "github.com/Azure/azure-sdk-for-go/profiles/latest/compute/mgmt/compute" + "github.com/Azure/go-autorest/autorest" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/vmssextensions/mock_vmssextensions" + + "github.com/golang/mock/gomock" + . "github.com/onsi/gomega" + "k8s.io/klog/klogr" + azure "sigs.k8s.io/cluster-api-provider-azure/cloud" + gomockinternal "sigs.k8s.io/cluster-api-provider-azure/internal/test/matchers/gomock" +) + +func TestReconcileVMSSExtension(t *testing.T) { + testcases := []struct { + name string + expectedError string + expect func(s *mock_vmssextensions.MockVMSSExtensionScopeMockRecorder, m *mock_vmssextensions.MockclientMockRecorder) + }{ + { + name: "reconcile multiple extensions", + expectedError: "", + expect: func(s *mock_vmssextensions.MockVMSSExtensionScopeMockRecorder, m *mock_vmssextensions.MockclientMockRecorder) { + s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New()) + s.VMSSExtensionSpecs().Return([]azure.VMSSExtensionSpec{ + { + Name: "my-extension-1", + ScaleSetName: "my-vmss", + Publisher: "some-publisher", + Version: "1.0", + }, + { + Name: "other-extension", + ScaleSetName: "my-vmss", + Publisher: "other-publisher", + Version: "2.0", + }, + }) + s.ResourceGroup().AnyTimes().Return("my-rg") + s.Location().AnyTimes().Return("test-location") + m.CreateOrUpdate(gomockinternal.AContext(), "my-rg", "my-vmss", "my-extension-1", gomock.AssignableToTypeOf(compute.VirtualMachineScaleSetExtension{})) + m.CreateOrUpdate(gomockinternal.AContext(), "my-rg", "my-vmss", "other-extension", gomock.AssignableToTypeOf(compute.VirtualMachineScaleSetExtension{})) + }, + }, + { + name: "error creating the extension", + expectedError: "failed to create VM extension my-extension-1 on scale set my-vmss in resource group my-rg: #: Internal Server Error: StatusCode=500", + expect: func(s *mock_vmssextensions.MockVMSSExtensionScopeMockRecorder, m *mock_vmssextensions.MockclientMockRecorder) { + s.V(gomock.AssignableToTypeOf(2)).AnyTimes().Return(klogr.New()) + s.VMSSExtensionSpecs().Return([]azure.VMSSExtensionSpec{ + { + Name: "my-extension-1", + ScaleSetName: "my-vmss", + Publisher: "some-publisher", + Version: "1.0", + }, + { + Name: "other-extension", + ScaleSetName: "my-vmss", + Publisher: "other-publisher", + Version: "2.0", + }, + }) + s.ResourceGroup().AnyTimes().Return("my-rg") + s.Location().AnyTimes().Return("test-location") + m.CreateOrUpdate(gomockinternal.AContext(), "my-rg", "my-vmss", "my-extension-1", gomock.AssignableToTypeOf(compute.VirtualMachineScaleSetExtension{})).Return(autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: 500}, "Internal Server Error")) + + }, + }, + } + + for _, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + t.Parallel() + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + scopeMock := mock_vmssextensions.NewMockVMSSExtensionScope(mockCtrl) + clientMock := mock_vmssextensions.NewMockclient(mockCtrl) + + tc.expect(scopeMock.EXPECT(), clientMock.EXPECT()) + + s := &Service{ + Scope: scopeMock, + client: clientMock, + } + + err := s.Reconcile(context.TODO()) + if tc.expectedError != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err).To(MatchError(tc.expectedError)) + } else { + g.Expect(err).NotTo(HaveOccurred()) + } + }) + } +} diff --git a/cloud/types.go b/cloud/types.go index b449d1fa4e70..2f6fe527b636 100644 --- a/cloud/types.go +++ b/cloud/types.go @@ -187,3 +187,19 @@ type PrivateDNSSpec struct { type AvailabilitySetSpec struct { Name string } + +// VMExtensionSpec defines the specification for a VM extension. +type VMExtensionSpec struct { + Name string + VMName string + Publisher string + Version string +} + +// VMSSExtensionSpec defines the specification for a VMSS extension. +type VMSSExtensionSpec struct { + Name string + ScaleSetName string + Publisher string + Version string +} diff --git a/controllers/azuremachine_reconciler.go b/controllers/azuremachine_reconciler.go index c48c2711c6ce..3172092358fe 100644 --- a/controllers/azuremachine_reconciler.go +++ b/controllers/azuremachine_reconciler.go @@ -31,6 +31,7 @@ import ( "sigs.k8s.io/cluster-api-provider-azure/cloud/services/roleassignments" "sigs.k8s.io/cluster-api-provider-azure/cloud/services/tags" "sigs.k8s.io/cluster-api-provider-azure/cloud/services/virtualmachines" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/vmextensions" "sigs.k8s.io/cluster-api-provider-azure/util/tele" ) @@ -43,6 +44,7 @@ type azureMachineService struct { disksSvc azure.Service publicIPsSvc azure.Service tagsSvc azure.Service + vmExtensionsSvc azure.Service skuCache *resourceskus.Cache } @@ -63,6 +65,7 @@ func newAzureMachineService(machineScope *scope.MachineScope) (*azureMachineServ disksSvc: disks.New(machineScope), publicIPsSvc: publicips.New(machineScope), tagsSvc: tags.New(machineScope), + vmExtensionsSvc: vmextensions.New(machineScope), skuCache: cache, }, nil } @@ -92,6 +95,10 @@ func (s *azureMachineService) Reconcile(ctx context.Context) error { return errors.Wrap(err, "unable to create role assignment") } + if err := s.vmExtensionsSvc.Reconcile(ctx); err != nil { + return errors.Wrap(err, "unable to create vm extension") + } + if err := s.tagsSvc.Reconcile(ctx); err != nil { return errors.Wrap(err, "unable to update tags") } diff --git a/exp/controllers/azuremachinepool_reconciler.go b/exp/controllers/azuremachinepool_reconciler.go index 2d5d26eeb12e..7ff794d7143e 100644 --- a/exp/controllers/azuremachinepool_reconciler.go +++ b/exp/controllers/azuremachinepool_reconciler.go @@ -19,6 +19,8 @@ package controllers import ( "context" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/vmssextensions" + "github.com/pkg/errors" azure "sigs.k8s.io/cluster-api-provider-azure/cloud" @@ -34,6 +36,7 @@ type azureMachinePoolService struct { virtualMachinesScaleSetSvc azure.Service skuCache *resourceskus.Cache roleAssignmentsSvc azure.Service + vmssExtensionSvc azure.Service } // newAzureMachinePoolService populates all the services based on input scope. @@ -47,6 +50,7 @@ func newAzureMachinePoolService(machinePoolScope *scope.MachinePoolScope) (*azur virtualMachinesScaleSetSvc: scalesets.NewService(machinePoolScope, cache), skuCache: cache, roleAssignmentsSvc: roleassignments.New(machinePoolScope), + vmssExtensionSvc: vmssextensions.New(machinePoolScope), }, nil } @@ -63,6 +67,10 @@ func (s *azureMachinePoolService) Reconcile(ctx context.Context) error { return errors.Wrap(err, "unable to create role assignment") } + if err := s.vmssExtensionSvc.Reconcile(ctx); err != nil { + return errors.Wrap(err, "unable to create vmss extension") + } + return nil } diff --git a/go.sum b/go.sum index 4fb891d85972..2fd941844c15 100644 --- a/go.sum +++ b/go.sum @@ -1264,6 +1264,7 @@ sigs.k8s.io/controller-runtime v0.5.14/go.mod h1:OTqxLuz7gVcrq+BHGUgedRu6b2VIKCE sigs.k8s.io/kind v0.7.1-0.20200303021537-981bd80d3802 h1:L6/8hETA7jvdx3xBcbDifrIN2xaYHE7tA58n+Kdp2Zw= sigs.k8s.io/kind v0.7.1-0.20200303021537-981bd80d3802/go.mod h1:HIZ3PWUezpklcjkqpFbnYOqaqsAE1JeCTEwkgvPLXjk= sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= +sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e h1:4Z09Hglb792X0kfOBBJUPFEyvVfQWrYT/l8h5EKA6JQ= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff/v2 v2.0.1/go.mod h1:Wb7vfKAodbKgf6tn1Kl0VvGj7mRH6DGaRcixXEJXTsE= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=