X-Smart-Squash: Squashed 11 commits:

b4a141d Add log caches to reduce log noise
35ce2dd WIP
65aedae WIP
1a853d1 reduce logs
15e8fa3 WIP
fd32e77 update cluster logs
4dd5724 WIP
d3f3a68 WIP
ab6c110 WIP
6084ed8 fix log message
8eed701 stylez

Makefile

@@ -69,7 +69,7 @@ internal_image_registry:=image-registry.openshift-image-registry.svc:5000
 test_image:=test/$(IMAGE_NAME)
 
 DOCKER ?= docker
-DOCKER_CONFIG ?= "${PWD}/.docker"
+DOCKER_CONFIG ?= "${HOME}/.docker"
 
 # Default Variables
 ACSMS_NAMESPACE ?= acsms

fleetshard/pkg/central/reconciler/reconciler.go

@@ -115,6 +115,8 @@ func (r *CentralReconciler) Reconcile(ctx context.Context, remoteCentral private
 		return nil, ErrCentralNotChanged
 	}
 
+	glog.Infof("Start reconcile central %s/%s", remoteCentral.Metadata.Namespace, remoteCentral.Metadata.Name)
+
 	remoteCentralName := remoteCentral.Metadata.Name
 	remoteCentralNamespace := remoteCentral.Metadata.Namespace
 

fleetshard/pkg/runtime/runtime.go

@@ -19,6 +19,7 @@ import (
 	"github.com/stackrox/acs-fleet-manager/fleetshard/pkg/k8s"
 	"github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/api/private"
 	"github.com/stackrox/acs-fleet-manager/pkg/client/fleetmanager"
+	"github.com/stackrox/acs-fleet-manager/pkg/logger"
 	"github.com/stackrox/rox/pkg/concurrency"
 	"k8s.io/apimachinery/pkg/util/wait"
 	ctrlClient "sigs.k8s.io/controller-runtime/pkg/client"
@@ -29,6 +30,8 @@ import (
 // TODO(SimonBaeumer): set a unique identifier for the map key, currently the instance name is used
 type reconcilerRegistry map[string]*centralReconciler.CentralReconciler
 
+var reconciledCentralCountCache int32
+
 var backoff = wait.Backoff{
 	Duration: 1 * time.Second,
 	Factor:   1.5,
@@ -135,7 +138,8 @@ func (r *Runtime) Start() error {
 		}
 
 		// Start for each Central its own reconciler which can be triggered by sending a central to the receive channel.
-		glog.Infof("Received %d centrals", len(list.Items))
+		reconciledCentralCountCache = int32(len(list.Items))
+		logger.InfoChangedInt32(&reconciledCentralCountCache, "Received central count changed: received %d centrals", reconciledCentralCountCache)
 		for _, central := range list.Items {
 			if _, ok := r.reconcilers[central.Id]; !ok {
 				r.reconcilers[central.Id] = centralReconciler.NewCentralReconciler(r.k8sClient, central,
@@ -151,7 +155,6 @@ func (r *Runtime) Start() error {
 				ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
 				defer cancel()
 
-				glog.Infof("Start reconcile central %s/%s", central.Metadata.Namespace, central.Metadata.Name)
 				status, err := reconciler.Reconcile(ctx, central)
 				fleetshardmetrics.MetricsInstance().IncCentralReconcilations()
 				r.handleReconcileResult(central, status, err)

internal/dinosaur/pkg/workers/clusters_mgr.go

@@ -3,6 +3,8 @@ package workers
 
 import (
 	"fmt"
+	"strings"
+	"sync"
 
 	dinosaurConstants "github.com/stackrox/acs-fleet-manager/internal/dinosaur/constants"
 	"github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/clusters/types"
@@ -12,9 +14,6 @@ import (
 	"github.com/stackrox/acs-fleet-manager/pkg/constants"
 	"github.com/stackrox/acs-fleet-manager/pkg/logger"
 
-	"strings"
-	"sync"
-
 	"github.com/goava/di"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/uuid"
@@ -59,6 +58,10 @@ const (
 	clusterAdminRoleName             = "cluster-admin"
 )
 
+var (
+	readyClusterCount int32
+)
+
 var clusterMetricsStatuses = []api.ClusterStatus{
 	api.ClusterAccepted,
 	api.ClusterProvisioning,
@@ -322,9 +325,9 @@ func (c *ClusterManager) processReadyClusters() []error {
 		errs = append(errs, errors.Wrap(listErr, "failed to list ready clusters"))
 		return errs
 	}
-	if len(readyClusters) > 0 {
-		glog.V(10).Infof("ready clusters count = %d", len(readyClusters))
-	}
+
+	readyClusterCount = int32(len(readyClusters))
+	logger.InfoChangedInt32(&readyClusterCount, "ready clusters count = %d", readyClusterCount)
 
 	for _, readyCluster := range readyClusters {
 		emptyClusterReconciled := false
@@ -472,9 +475,9 @@ func (c *ClusterManager) reconcileClusterInstanceType(cluster api.Cluster) error
 		if err != nil {
 			return errors.Wrapf(err, "failed to update instance type in database for cluster %s", cluster.ClusterID)
 		}
+		logger.Logger.V(10).Infof("supported instance type for cluster = %s successful updated", cluster.ClusterID)
 	}
 
-	logger.Logger.V(10).Infof("supported instance type for cluster = %s successful updated", cluster.ClusterID)
 	return nil
 }
 

internal/dinosaur/pkg/workers/dinosaurmgrs/accepted_centrals_mgr.go

@@ -59,6 +59,8 @@ func (k *AcceptedCentralManager) Stop() {
 	k.StopWorker(k)
 }
 
+var acceptedCentralCount int32
+
 // Reconcile ...
 func (k *AcceptedCentralManager) Reconcile() []error {
 	var encounteredErrors []error
@@ -68,9 +70,9 @@ func (k *AcceptedCentralManager) Reconcile() []error {
 	if serviceErr != nil {
 		encounteredErrors = append(encounteredErrors, errors.Wrap(serviceErr, "failed to list accepted centrals"))
 	}
-	if len(acceptedCentralRequests) > 0 {
-		glog.Infof("accepted centrals count = %d", len(acceptedCentralRequests))
-	}
+
+	acceptedCentralCount = int32(len(acceptedCentralRequests))
+	logger.InfoChangedInt32(&acceptedCentralCount, "accepted centrals count = %d", acceptedCentralCount)
 
 	for _, centralRequest := range acceptedCentralRequests {
 		glog.V(10).Infof("accepted central id = %s", centralRequest.ID)

internal/dinosaur/pkg/workers/dinosaurmgrs/ready_dinosaurs_mgr.go

@@ -1,19 +1,21 @@
 package dinosaurmgrs
 
 import (
-	"github.com/golang/glog"
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
 	constants2 "github.com/stackrox/acs-fleet-manager/internal/dinosaur/constants"
 	"github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/services"
 	"github.com/stackrox/acs-fleet-manager/pkg/client/iam"
+	"github.com/stackrox/acs-fleet-manager/pkg/logger"
 	"github.com/stackrox/acs-fleet-manager/pkg/metrics"
 	"github.com/stackrox/acs-fleet-manager/pkg/services/sso"
 	"github.com/stackrox/acs-fleet-manager/pkg/workers"
 )
 
 const readyCentralWorkerType = "ready_dinosaur"
 
+var readyCentralCountCache int32
+
 // ReadyDinosaurManager represents a dinosaur manager that periodically reconciles dinosaur requests
 type ReadyDinosaurManager struct {
 	workers.BaseWorker
@@ -49,21 +51,14 @@ func (k *ReadyDinosaurManager) Stop() {
 
 // Reconcile ...
 func (k *ReadyDinosaurManager) Reconcile() []error {
-
 	var encounteredErrors []error
 
-	readyDinosaurs, serviceErr := k.dinosaurService.ListByStatus(constants2.CentralRequestStatusReady)
+	readyCentrals, serviceErr := k.dinosaurService.ListByStatus(constants2.CentralRequestStatusReady)
 	if serviceErr != nil {
 		encounteredErrors = append(encounteredErrors, errors.Wrap(serviceErr, "failed to list ready centrals"))
 	}
-	if len(readyDinosaurs) > 0 {
-		glog.Infof("ready centrals count = %d", len(readyDinosaurs))
-	}
-
-	for _, dinosaur := range readyDinosaurs {
-		glog.V(10).Infof("ready central id = %s", dinosaur.ID)
-		// TODO implement reconciliation logic for ready dinosaurs
-	}
+	readyCentralCountCache = int32(len(readyCentrals))
+	logger.InfoChangedInt32(&readyCentralCountCache, "ready centrals count = %d", readyCentralCountCache)
 
 	return encounteredErrors
 }

pkg/logger/logger.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 	"strings"
+	"sync"
 
 	sentry "github.com/getsentry/sentry-go"
 
@@ -189,6 +190,18 @@ func (l *logger) InfoDepth(depth int, args ...interface{}) {
 	glog.InfoDepth(logDepth+depth, args...)
 }
 
+var cache = sync.Map{}
+
+// InfoChangedInt32 logs whenever the value of the passed counter reference changes when it is called.
+func InfoChangedInt32(counter *int32, msg string, args ...interface{}) {
+	val, loaded := cache.LoadOrStore(counter, *counter)
+	if loaded && val == *counter {
+		return // counter not changed, do not log
+	}
+
+	glog.InfoDepth(logDepth, fmt.Sprintf(msg, args...))
+}
+
 // Warningf ...
 func (l *logger) Warningf(format string, args ...interface{}) {
 	prefixed := l.prepareLogPrefix(format, args...)