add filter to ignore heath metrics error

azure/scope/managedcontrolplane.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"encoding/json"
 	"strings"
+	"time"
 
 	"github.com/Azure/go-autorest/autorest"
 	"github.com/Azure/go-autorest/autorest/to"
@@ -666,3 +667,13 @@ func (s *ManagedControlPlaneScope) AvailabilityStatusResource() conditions.Sette
 func (s *ManagedControlPlaneScope) AvailabilityStatusResourceURI() string {
 	return azure.ManagedClusterID(s.SubscriptionID(), s.ResourceGroup(), s.ControlPlane.Name)
 }
+
+// AvailabilityStatusFilter ignores the health metrics connection error that
+// occurs on startup for every AKS cluster.
+func (s *ManagedControlPlaneScope) AvailabilityStatusFilter(cond *clusterv1.Condition) *clusterv1.Condition {
+	if time.Since(s.ControlPlane.CreationTimestamp.Time) < 1*time.Hour &&
+		strings.Contains(cond.Message, "We've temporarily lost connection to the health metrics of this AKS cluster.") {
+		return conditions.TrueCondition(infrav1.AzureResourceAvailableCondition)
+	}
+	return cond
+}

azure/services/resourcehealth/mock_resourcehealth/doc.go

@@ -17,7 +17,7 @@ limitations under the License.
 // Run go generate to regenerate this mock.
 //
 //go:generate ../../../../hack/tools/bin/mockgen -destination client_mock.go -package mock_resourcehealth -source ../client.go Client
-//go:generate ../../../../hack/tools/bin/mockgen -destination resourcehealth_mock.go -package mock_resourcehealth -source ../resourcehealth.go ResourceHealthScope
+//go:generate ../../../../hack/tools/bin/mockgen -destination resourcehealth_mock.go -package mock_resourcehealth -source ../resourcehealth.go ResourceHealthScope,AvailabilityStatusFilterer
 //go:generate /usr/bin/env bash -c "cat ../../../../hack/boilerplate/boilerplate.generatego.txt client_mock.go > _client_mock.go && mv _client_mock.go client_mock.go"
 //go:generate /usr/bin/env bash -c "cat ../../../../hack/boilerplate/boilerplate.generatego.txt resourcehealth_mock.go > _resourcehealth_mock.go && mv _resourcehealth_mock.go resourcehealth_mock.go"
 package mock_resourcehealth

azure/services/resourcehealth/resourcehealth.go

@@ -39,6 +39,13 @@ type ResourceHealthScope interface {
 	AvailabilityStatusResource() conditions.Setter
 }
 
+// AvailabilityStatusFilterer transforms the condition derived from the
+// availability status to allow the condition to be overridden in specific
+// circumstances.
+type AvailabilityStatusFilterer interface {
+	AvailabilityStatusFilter(cond *clusterv1.Condition) *clusterv1.Condition
+}
+
 // Service provides operations on Azure resources.
 type Service struct {
 	Scope ResourceHealthScope
@@ -71,6 +78,10 @@ func (s *Service) Reconcile(ctx context.Context) error {
 	log.V(2).Info("got availability status for resource", "resource", resource, "status", avail)
 
 	cond := azureAvailabilityStatusToCondition(avail)
+	if filterer, ok := s.Scope.(AvailabilityStatusFilterer); ok {
+		cond = filterer.AvailabilityStatusFilter(cond)
+	}
+
 	conditions.Set(s.Scope.AvailabilityStatusResource(), cond)
 
 	if cond.Status == corev1.ConditionFalse {

azure/services/resourcehealth/resourcehealth_test.go

@@ -26,20 +26,23 @@ import (
 	. "github.com/onsi/gomega"
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
+	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
 	"sigs.k8s.io/cluster-api-provider-azure/azure/services/resourcehealth/mock_resourcehealth"
 	gomockinternal "sigs.k8s.io/cluster-api-provider-azure/internal/test/matchers/gomock"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	"sigs.k8s.io/cluster-api/util/conditions"
 )
 
 func TestReconcileResourceHealth(t *testing.T) {
 	testcases := []struct {
 		name          string
-		expect        func(s *mock_resourcehealth.MockResourceHealthScopeMockRecorder, m *mock_resourcehealth.MockclientMockRecorder)
+		filterEnabled bool
+		expect        func(s *mock_resourcehealth.MockResourceHealthScopeMockRecorder, m *mock_resourcehealth.MockclientMockRecorder, f *mock_resourcehealth.MockAvailabilityStatusFiltererMockRecorder)
 		expectedError string
 	}{
 		{
 			name: "available resource",
-			expect: func(s *mock_resourcehealth.MockResourceHealthScopeMockRecorder, m *mock_resourcehealth.MockclientMockRecorder) {
+			expect: func(s *mock_resourcehealth.MockResourceHealthScopeMockRecorder, m *mock_resourcehealth.MockclientMockRecorder, _ *mock_resourcehealth.MockAvailabilityStatusFiltererMockRecorder) {
 				s.AvailabilityStatusResource().Times(1)
 				s.AvailabilityStatusResourceURI().Times(1)
 				m.GetByResource(gomockinternal.AContext(), gomock.Any()).Times(1).Return(resourcehealth.AvailabilityStatus{
@@ -52,7 +55,7 @@ func TestReconcileResourceHealth(t *testing.T) {
 		},
 		{
 			name: "unavailable resource",
-			expect: func(s *mock_resourcehealth.MockResourceHealthScopeMockRecorder, m *mock_resourcehealth.MockclientMockRecorder) {
+			expect: func(s *mock_resourcehealth.MockResourceHealthScopeMockRecorder, m *mock_resourcehealth.MockclientMockRecorder, _ *mock_resourcehealth.MockAvailabilityStatusFiltererMockRecorder) {
 				s.AvailabilityStatusResource().Times(1)
 				s.AvailabilityStatusResourceURI().Times(1)
 				m.GetByResource(gomockinternal.AContext(), gomock.Any()).Times(1).Return(resourcehealth.AvailabilityStatus{
@@ -66,12 +69,29 @@ func TestReconcileResourceHealth(t *testing.T) {
 		},
 		{
 			name: "API error",
-			expect: func(s *mock_resourcehealth.MockResourceHealthScopeMockRecorder, m *mock_resourcehealth.MockclientMockRecorder) {
+			expect: func(s *mock_resourcehealth.MockResourceHealthScopeMockRecorder, m *mock_resourcehealth.MockclientMockRecorder, _ *mock_resourcehealth.MockAvailabilityStatusFiltererMockRecorder) {
 				s.AvailabilityStatusResourceURI().Times(1).Return("myURI")
 				m.GetByResource(gomockinternal.AContext(), gomock.Any()).Times(1).Return(resourcehealth.AvailabilityStatus{}, errors.New("some API error"))
 			},
 			expectedError: "failed to get availability status for resource myURI: some API error",
 		},
+		{
+			name:          "filter",
+			filterEnabled: true,
+			expect: func(s *mock_resourcehealth.MockResourceHealthScopeMockRecorder, m *mock_resourcehealth.MockclientMockRecorder, f *mock_resourcehealth.MockAvailabilityStatusFiltererMockRecorder) {
+				s.AvailabilityStatusResource().Times(1)
+				s.AvailabilityStatusResourceURI().Times(1)
+				m.GetByResource(gomockinternal.AContext(), gomock.Any()).Times(1).Return(resourcehealth.AvailabilityStatus{
+					Properties: &resourcehealth.AvailabilityStatusProperties{
+						AvailabilityState: resourcehealth.AvailabilityStateValuesUnavailable,
+						Summary:           to.StringPtr("summary"),
+					},
+				}, nil)
+				// ignore the above status
+				f.AvailabilityStatusFilter(gomock.Any()).Return(conditions.TrueCondition(infrav1.AzureResourceAvailableCondition))
+			},
+			expectedError: "",
+		},
 	}
 
 	for _, tc := range testcases {
@@ -81,13 +101,20 @@ func TestReconcileResourceHealth(t *testing.T) {
 			defer mockCtrl.Finish()
 			scopeMock := mock_resourcehealth.NewMockResourceHealthScope(mockCtrl)
 			clientMock := mock_resourcehealth.NewMockclient(mockCtrl)
+			filtererMock := mock_resourcehealth.NewMockAvailabilityStatusFilterer(mockCtrl)
 
-			tc.expect(scopeMock.EXPECT(), clientMock.EXPECT())
+			tc.expect(scopeMock.EXPECT(), clientMock.EXPECT(), filtererMock.EXPECT())
 
 			s := &Service{
 				Scope:  scopeMock,
 				client: clientMock,
 			}
+			if tc.filterEnabled {
+				s.Scope = struct {
+					ResourceHealthScope
+					AvailabilityStatusFilterer
+				}{scopeMock, filtererMock}
+			}
 
 			err := s.Reconcile(context.TODO())