From d9860c09e4f219068baecbe88784d32be1f9da4f Mon Sep 17 00:00:00 2001 From: Simon Baeumer Date: Wed, 1 Mar 2023 16:10:11 +0100 Subject: [PATCH] X-Smart-Squash: Squashed 11 commits: b4a141d8 Add log caches to reduce log noise 35ce2dd5 WIP 65aedaed WIP 1a853d1a reduce logs 15e8fa3d WIP fd32e77a update cluster logs 4dd57244 WIP d3f3a686 WIP ab6c110d WIP 6084ed82 fix log message 8eed7016 stylez --- Makefile | 2 +- fleetshard/pkg/central/reconciler/reconciler.go | 2 ++ fleetshard/pkg/runtime/runtime.go | 7 +++++-- internal/dinosaur/pkg/workers/clusters_mgr.go | 17 ++++++++++------- .../dinosaurmgrs/accepted_centrals_mgr.go | 8 +++++--- .../workers/dinosaurmgrs/ready_dinosaurs_mgr.go | 17 ++++++----------- pkg/logger/logger.go | 13 +++++++++++++ 7 files changed, 42 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index 233e65b686..2bedbc3eef 100644 --- a/Makefile +++ b/Makefile @@ -69,7 +69,7 @@ internal_image_registry:=image-registry.openshift-image-registry.svc:5000 test_image:=test/$(IMAGE_NAME) DOCKER ?= docker -DOCKER_CONFIG ?= "${PWD}/.docker" +DOCKER_CONFIG ?= "${HOME}/.docker" # Default Variables ACSMS_NAMESPACE ?= acsms diff --git a/fleetshard/pkg/central/reconciler/reconciler.go b/fleetshard/pkg/central/reconciler/reconciler.go index 0586679aba..db6119a15f 100644 --- a/fleetshard/pkg/central/reconciler/reconciler.go +++ b/fleetshard/pkg/central/reconciler/reconciler.go @@ -104,6 +104,8 @@ func (r *CentralReconciler) Reconcile(ctx context.Context, remoteCentral private return nil, errors.Wrapf(err, "checking if central changed") } + glog.Infof("Start reconcile central %s/%s", remoteCentral.Metadata.Namespace, remoteCentral.Metadata.Name) + remoteCentralName := remoteCentral.Metadata.Name remoteCentralNamespace := remoteCentral.Metadata.Namespace if !changed && r.wantsAuthProvider == r.hasAuthProvider && isRemoteCentralReady(remoteCentral) { diff --git a/fleetshard/pkg/runtime/runtime.go b/fleetshard/pkg/runtime/runtime.go index e42aee08a3..7fdc117f73 100644 --- a/fleetshard/pkg/runtime/runtime.go +++ b/fleetshard/pkg/runtime/runtime.go @@ -17,6 +17,7 @@ import ( "github.com/stackrox/acs-fleet-manager/fleetshard/pkg/k8s" "github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/api/private" "github.com/stackrox/acs-fleet-manager/pkg/client/fleetmanager" + "github.com/stackrox/acs-fleet-manager/pkg/logger" "github.com/stackrox/rox/pkg/concurrency" "k8s.io/apimachinery/pkg/util/wait" ctrlClient "sigs.k8s.io/controller-runtime/pkg/client" @@ -27,6 +28,8 @@ import ( // TODO(SimonBaeumer): set a unique identifier for the map key, currently the instance name is used type reconcilerRegistry map[string]*centralReconciler.CentralReconciler +var reconciledCentralCountCache int32 + var backoff = wait.Backoff{ Duration: 1 * time.Second, Factor: 1.5, @@ -119,7 +122,8 @@ func (r *Runtime) Start() error { } // Start for each Central its own reconciler which can be triggered by sending a central to the receive channel. - glog.Infof("Received %d centrals", len(list.Items)) + reconciledCentralCountCache = int32(len(list.Items)) + logger.InfoChangedInt32(&reconciledCentralCountCache, "Received central count changed: received %d centrals", reconciledCentralCountCache) for _, central := range list.Items { if _, ok := r.reconcilers[central.Id]; !ok { r.reconcilers[central.Id] = centralReconciler.NewCentralReconciler(r.k8sClient, central, @@ -135,7 +139,6 @@ func (r *Runtime) Start() error { ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) defer cancel() - glog.Infof("Start reconcile central %s/%s", central.Metadata.Namespace, central.Metadata.Name) status, err := reconciler.Reconcile(ctx, central) fleetshardmetrics.MetricsInstance().IncCentralReconcilations() r.handleReconcileResult(central, status, err) diff --git a/internal/dinosaur/pkg/workers/clusters_mgr.go b/internal/dinosaur/pkg/workers/clusters_mgr.go index 63379bd0a4..622bda58b3 100644 --- a/internal/dinosaur/pkg/workers/clusters_mgr.go +++ b/internal/dinosaur/pkg/workers/clusters_mgr.go @@ -3,6 +3,8 @@ package workers import ( "fmt" + "strings" + "sync" dinosaurConstants "github.com/stackrox/acs-fleet-manager/internal/dinosaur/constants" "github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/clusters/types" @@ -12,9 +14,6 @@ import ( "github.com/stackrox/acs-fleet-manager/pkg/constants" "github.com/stackrox/acs-fleet-manager/pkg/logger" - "strings" - "sync" - "github.com/goava/di" "github.com/google/go-cmp/cmp" "github.com/google/uuid" @@ -59,6 +58,10 @@ const ( clusterAdminRoleName = "cluster-admin" ) +var ( + readyClusterCount int32 +) + var clusterMetricsStatuses = []api.ClusterStatus{ api.ClusterAccepted, api.ClusterProvisioning, @@ -322,9 +325,9 @@ func (c *ClusterManager) processReadyClusters() []error { errs = append(errs, errors.Wrap(listErr, "failed to list ready clusters")) return errs } - if len(readyClusters) > 0 { - glog.V(10).Infof("ready clusters count = %d", len(readyClusters)) - } + + readyClusterCount = int32(len(readyClusters)) + logger.InfoChangedInt32(&readyClusterCount, "ready clusters count = %d", readyClusterCount) for _, readyCluster := range readyClusters { emptyClusterReconciled := false @@ -472,9 +475,9 @@ func (c *ClusterManager) reconcileClusterInstanceType(cluster api.Cluster) error if err != nil { return errors.Wrapf(err, "failed to update instance type in database for cluster %s", cluster.ClusterID) } + logger.Logger.V(10).Infof("supported instance type for cluster = %s successful updated", cluster.ClusterID) } - logger.Logger.V(10).Infof("supported instance type for cluster = %s successful updated", cluster.ClusterID) return nil } diff --git a/internal/dinosaur/pkg/workers/dinosaurmgrs/accepted_centrals_mgr.go b/internal/dinosaur/pkg/workers/dinosaurmgrs/accepted_centrals_mgr.go index ca6dbefe39..6923d48e79 100644 --- a/internal/dinosaur/pkg/workers/dinosaurmgrs/accepted_centrals_mgr.go +++ b/internal/dinosaur/pkg/workers/dinosaurmgrs/accepted_centrals_mgr.go @@ -59,6 +59,8 @@ func (k *AcceptedCentralManager) Stop() { k.StopWorker(k) } +var acceptedCentralCount int32 + // Reconcile ... func (k *AcceptedCentralManager) Reconcile() []error { var encounteredErrors []error @@ -68,9 +70,9 @@ func (k *AcceptedCentralManager) Reconcile() []error { if serviceErr != nil { encounteredErrors = append(encounteredErrors, errors.Wrap(serviceErr, "failed to list accepted centrals")) } - if len(acceptedCentralRequests) > 0 { - glog.Infof("accepted centrals count = %d", len(acceptedCentralRequests)) - } + + acceptedCentralCount = int32(len(acceptedCentralRequests)) + logger.InfoChangedInt32(&acceptedCentralCount, "accepted centrals count = %d", acceptedCentralCount) for _, centralRequest := range acceptedCentralRequests { glog.V(10).Infof("accepted central id = %s", centralRequest.ID) diff --git a/internal/dinosaur/pkg/workers/dinosaurmgrs/ready_dinosaurs_mgr.go b/internal/dinosaur/pkg/workers/dinosaurmgrs/ready_dinosaurs_mgr.go index 543f45198a..243131a9b3 100644 --- a/internal/dinosaur/pkg/workers/dinosaurmgrs/ready_dinosaurs_mgr.go +++ b/internal/dinosaur/pkg/workers/dinosaurmgrs/ready_dinosaurs_mgr.go @@ -1,12 +1,12 @@ package dinosaurmgrs import ( - "github.com/golang/glog" "github.com/google/uuid" "github.com/pkg/errors" constants2 "github.com/stackrox/acs-fleet-manager/internal/dinosaur/constants" "github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/services" "github.com/stackrox/acs-fleet-manager/pkg/client/iam" + "github.com/stackrox/acs-fleet-manager/pkg/logger" "github.com/stackrox/acs-fleet-manager/pkg/metrics" "github.com/stackrox/acs-fleet-manager/pkg/services/sso" "github.com/stackrox/acs-fleet-manager/pkg/workers" @@ -14,6 +14,8 @@ import ( const readyCentralWorkerType = "ready_dinosaur" +var readyCentralCountCache int32 + // ReadyDinosaurManager represents a dinosaur manager that periodically reconciles dinosaur requests type ReadyDinosaurManager struct { workers.BaseWorker @@ -49,21 +51,14 @@ func (k *ReadyDinosaurManager) Stop() { // Reconcile ... func (k *ReadyDinosaurManager) Reconcile() []error { - var encounteredErrors []error - readyDinosaurs, serviceErr := k.dinosaurService.ListByStatus(constants2.CentralRequestStatusReady) + readyCentrals, serviceErr := k.dinosaurService.ListByStatus(constants2.CentralRequestStatusReady) if serviceErr != nil { encounteredErrors = append(encounteredErrors, errors.Wrap(serviceErr, "failed to list ready centrals")) } - if len(readyDinosaurs) > 0 { - glog.Infof("ready centrals count = %d", len(readyDinosaurs)) - } - - for _, dinosaur := range readyDinosaurs { - glog.V(10).Infof("ready central id = %s", dinosaur.ID) - // TODO implement reconciliation logic for ready dinosaurs - } + readyCentralCountCache = int32(len(readyCentrals)) + logger.InfoChangedInt32(&readyCentralCountCache, "ready centrals count = %d", readyCentralCountCache) return encounteredErrors } diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index ad2b24519e..210c4f20df 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "strings" + "sync" sentry "github.com/getsentry/sentry-go" @@ -189,6 +190,18 @@ func (l *logger) InfoDepth(depth int, args ...interface{}) { glog.InfoDepth(logDepth+depth, args...) } +var cache = sync.Map{} + +// InfoChangedInt32 logs whenever the value of the passed counter reference changes when it is called. +func InfoChangedInt32(counter *int32, msg string, args ...interface{}) { + val, loaded := cache.LoadOrStore(counter, *counter) + if loaded && val == *counter { + return // counter not changed, do not log + } + + glog.InfoDepth(logDepth, fmt.Sprintf(msg, args...)) +} + // Warningf ... func (l *logger) Warningf(format string, args ...interface{}) { prefixed := l.prepareLogPrefix(format, args...)