From fa35b437f782adc96e3de7708c31555cf10ac37f Mon Sep 17 00:00:00 2001 From: Jon Huhn Date: Fri, 2 Dec 2022 17:17:09 -0600 Subject: [PATCH] add filter to ignore heath metrics error --- azure/scope/managedcontrolplane.go | 11 ++++++ .../resourcehealth/mock_resourcehealth/doc.go | 2 +- .../resourcehealth_mock.go | 38 +++++++++++++++++++ .../services/resourcehealth/resourcehealth.go | 11 ++++++ .../resourcehealth/resourcehealth_test.go | 37 +++++++++++++++--- 5 files changed, 93 insertions(+), 6 deletions(-) diff --git a/azure/scope/managedcontrolplane.go b/azure/scope/managedcontrolplane.go index 3e76e8f4534..a1c1d4b4b2b 100644 --- a/azure/scope/managedcontrolplane.go +++ b/azure/scope/managedcontrolplane.go @@ -20,6 +20,7 @@ import ( "context" "encoding/json" "strings" + "time" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/to" @@ -666,3 +667,13 @@ func (s *ManagedControlPlaneScope) AvailabilityStatusResource() conditions.Sette func (s *ManagedControlPlaneScope) AvailabilityStatusResourceURI() string { return azure.ManagedClusterID(s.SubscriptionID(), s.ResourceGroup(), s.ControlPlane.Name) } + +// AvailabilityStatusFilter ignores the health metrics connection error that +// occurs on startup for every AKS cluster. +func (s *ManagedControlPlaneScope) AvailabilityStatusFilter(cond *clusterv1.Condition) *clusterv1.Condition { + if time.Since(s.ControlPlane.CreationTimestamp.Time) < 1*time.Hour && + strings.Contains(cond.Message, "We've temporarily lost connection to the health metrics of this AKS cluster.") { + return conditions.TrueCondition(infrav1.AzureResourceAvailableCondition) + } + return cond +} diff --git a/azure/services/resourcehealth/mock_resourcehealth/doc.go b/azure/services/resourcehealth/mock_resourcehealth/doc.go index df91b732d28..7c5bbd9816f 100644 --- a/azure/services/resourcehealth/mock_resourcehealth/doc.go +++ b/azure/services/resourcehealth/mock_resourcehealth/doc.go @@ -17,7 +17,7 @@ limitations under the License. // Run go generate to regenerate this mock. // //go:generate ../../../../hack/tools/bin/mockgen -destination client_mock.go -package mock_resourcehealth -source ../client.go Client -//go:generate ../../../../hack/tools/bin/mockgen -destination resourcehealth_mock.go -package mock_resourcehealth -source ../resourcehealth.go ResourceHealthScope +//go:generate ../../../../hack/tools/bin/mockgen -destination resourcehealth_mock.go -package mock_resourcehealth -source ../resourcehealth.go ResourceHealthScope,AvailabilityStatusFilterer //go:generate /usr/bin/env bash -c "cat ../../../../hack/boilerplate/boilerplate.generatego.txt client_mock.go > _client_mock.go && mv _client_mock.go client_mock.go" //go:generate /usr/bin/env bash -c "cat ../../../../hack/boilerplate/boilerplate.generatego.txt resourcehealth_mock.go > _resourcehealth_mock.go && mv _resourcehealth_mock.go resourcehealth_mock.go" package mock_resourcehealth diff --git a/azure/services/resourcehealth/mock_resourcehealth/resourcehealth_mock.go b/azure/services/resourcehealth/mock_resourcehealth/resourcehealth_mock.go index 8b5e6b5a3e1..03d95059f8b 100644 --- a/azure/services/resourcehealth/mock_resourcehealth/resourcehealth_mock.go +++ b/azure/services/resourcehealth/mock_resourcehealth/resourcehealth_mock.go @@ -25,6 +25,7 @@ import ( autorest "github.com/Azure/go-autorest/autorest" gomock "github.com/golang/mock/gomock" + v1beta1 "sigs.k8s.io/cluster-api/api/v1beta1" conditions "sigs.k8s.io/cluster-api/util/conditions" ) @@ -190,3 +191,40 @@ func (mr *MockResourceHealthScopeMockRecorder) TenantID() *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TenantID", reflect.TypeOf((*MockResourceHealthScope)(nil).TenantID)) } + +// MockAvailabilityStatusFilterer is a mock of AvailabilityStatusFilterer interface. +type MockAvailabilityStatusFilterer struct { + ctrl *gomock.Controller + recorder *MockAvailabilityStatusFiltererMockRecorder +} + +// MockAvailabilityStatusFiltererMockRecorder is the mock recorder for MockAvailabilityStatusFilterer. +type MockAvailabilityStatusFiltererMockRecorder struct { + mock *MockAvailabilityStatusFilterer +} + +// NewMockAvailabilityStatusFilterer creates a new mock instance. +func NewMockAvailabilityStatusFilterer(ctrl *gomock.Controller) *MockAvailabilityStatusFilterer { + mock := &MockAvailabilityStatusFilterer{ctrl: ctrl} + mock.recorder = &MockAvailabilityStatusFiltererMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockAvailabilityStatusFilterer) EXPECT() *MockAvailabilityStatusFiltererMockRecorder { + return m.recorder +} + +// AvailabilityStatusFilter mocks base method. +func (m *MockAvailabilityStatusFilterer) AvailabilityStatusFilter(cond *v1beta1.Condition) *v1beta1.Condition { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AvailabilityStatusFilter", cond) + ret0, _ := ret[0].(*v1beta1.Condition) + return ret0 +} + +// AvailabilityStatusFilter indicates an expected call of AvailabilityStatusFilter. +func (mr *MockAvailabilityStatusFiltererMockRecorder) AvailabilityStatusFilter(cond interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AvailabilityStatusFilter", reflect.TypeOf((*MockAvailabilityStatusFilterer)(nil).AvailabilityStatusFilter), cond) +} diff --git a/azure/services/resourcehealth/resourcehealth.go b/azure/services/resourcehealth/resourcehealth.go index 3aaf14110b5..f66fb823155 100644 --- a/azure/services/resourcehealth/resourcehealth.go +++ b/azure/services/resourcehealth/resourcehealth.go @@ -39,6 +39,13 @@ type ResourceHealthScope interface { AvailabilityStatusResource() conditions.Setter } +// AvailabilityStatusFilterer transforms the condition derived from the +// availability status to allow the condition to be overridden in specific +// circumstances. +type AvailabilityStatusFilterer interface { + AvailabilityStatusFilter(cond *clusterv1.Condition) *clusterv1.Condition +} + // Service provides operations on Azure resources. type Service struct { Scope ResourceHealthScope @@ -71,6 +78,10 @@ func (s *Service) Reconcile(ctx context.Context) error { log.V(2).Info("got availability status for resource", "resource", resource, "status", avail) cond := azureAvailabilityStatusToCondition(avail) + if filterer, ok := s.Scope.(AvailabilityStatusFilterer); ok { + cond = filterer.AvailabilityStatusFilter(cond) + } + conditions.Set(s.Scope.AvailabilityStatusResource(), cond) if cond.Status == corev1.ConditionFalse { diff --git a/azure/services/resourcehealth/resourcehealth_test.go b/azure/services/resourcehealth/resourcehealth_test.go index 316ba408c19..993860e326b 100644 --- a/azure/services/resourcehealth/resourcehealth_test.go +++ b/azure/services/resourcehealth/resourcehealth_test.go @@ -26,20 +26,23 @@ import ( . "github.com/onsi/gomega" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure/services/resourcehealth/mock_resourcehealth" gomockinternal "sigs.k8s.io/cluster-api-provider-azure/internal/test/matchers/gomock" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/util/conditions" ) func TestReconcileResourceHealth(t *testing.T) { testcases := []struct { name string - expect func(s *mock_resourcehealth.MockResourceHealthScopeMockRecorder, m *mock_resourcehealth.MockclientMockRecorder) + filterEnabled bool + expect func(s *mock_resourcehealth.MockResourceHealthScopeMockRecorder, m *mock_resourcehealth.MockclientMockRecorder, f *mock_resourcehealth.MockAvailabilityStatusFiltererMockRecorder) expectedError string }{ { name: "available resource", - expect: func(s *mock_resourcehealth.MockResourceHealthScopeMockRecorder, m *mock_resourcehealth.MockclientMockRecorder) { + expect: func(s *mock_resourcehealth.MockResourceHealthScopeMockRecorder, m *mock_resourcehealth.MockclientMockRecorder, _ *mock_resourcehealth.MockAvailabilityStatusFiltererMockRecorder) { s.AvailabilityStatusResource().Times(1) s.AvailabilityStatusResourceURI().Times(1) m.GetByResource(gomockinternal.AContext(), gomock.Any()).Times(1).Return(resourcehealth.AvailabilityStatus{ @@ -52,7 +55,7 @@ func TestReconcileResourceHealth(t *testing.T) { }, { name: "unavailable resource", - expect: func(s *mock_resourcehealth.MockResourceHealthScopeMockRecorder, m *mock_resourcehealth.MockclientMockRecorder) { + expect: func(s *mock_resourcehealth.MockResourceHealthScopeMockRecorder, m *mock_resourcehealth.MockclientMockRecorder, _ *mock_resourcehealth.MockAvailabilityStatusFiltererMockRecorder) { s.AvailabilityStatusResource().Times(1) s.AvailabilityStatusResourceURI().Times(1) m.GetByResource(gomockinternal.AContext(), gomock.Any()).Times(1).Return(resourcehealth.AvailabilityStatus{ @@ -66,12 +69,29 @@ func TestReconcileResourceHealth(t *testing.T) { }, { name: "API error", - expect: func(s *mock_resourcehealth.MockResourceHealthScopeMockRecorder, m *mock_resourcehealth.MockclientMockRecorder) { + expect: func(s *mock_resourcehealth.MockResourceHealthScopeMockRecorder, m *mock_resourcehealth.MockclientMockRecorder, _ *mock_resourcehealth.MockAvailabilityStatusFiltererMockRecorder) { s.AvailabilityStatusResourceURI().Times(1).Return("myURI") m.GetByResource(gomockinternal.AContext(), gomock.Any()).Times(1).Return(resourcehealth.AvailabilityStatus{}, errors.New("some API error")) }, expectedError: "failed to get availability status for resource myURI: some API error", }, + { + name: "filter", + filterEnabled: true, + expect: func(s *mock_resourcehealth.MockResourceHealthScopeMockRecorder, m *mock_resourcehealth.MockclientMockRecorder, f *mock_resourcehealth.MockAvailabilityStatusFiltererMockRecorder) { + s.AvailabilityStatusResource().Times(1) + s.AvailabilityStatusResourceURI().Times(1) + m.GetByResource(gomockinternal.AContext(), gomock.Any()).Times(1).Return(resourcehealth.AvailabilityStatus{ + Properties: &resourcehealth.AvailabilityStatusProperties{ + AvailabilityState: resourcehealth.AvailabilityStateValuesUnavailable, + Summary: to.StringPtr("summary"), + }, + }, nil) + // ignore the above status + f.AvailabilityStatusFilter(gomock.Any()).Return(conditions.TrueCondition(infrav1.AzureResourceAvailableCondition)) + }, + expectedError: "", + }, } for _, tc := range testcases { @@ -81,13 +101,20 @@ func TestReconcileResourceHealth(t *testing.T) { defer mockCtrl.Finish() scopeMock := mock_resourcehealth.NewMockResourceHealthScope(mockCtrl) clientMock := mock_resourcehealth.NewMockclient(mockCtrl) + filtererMock := mock_resourcehealth.NewMockAvailabilityStatusFilterer(mockCtrl) - tc.expect(scopeMock.EXPECT(), clientMock.EXPECT()) + tc.expect(scopeMock.EXPECT(), clientMock.EXPECT(), filtererMock.EXPECT()) s := &Service{ Scope: scopeMock, client: clientMock, } + if tc.filterEnabled { + s.Scope = struct { + ResourceHealthScope + AvailabilityStatusFilterer + }{scopeMock, filtererMock} + } err := s.Reconcile(context.TODO())