From 246e7871770f84c4dc3a6feb5900c273cd0d0bb7 Mon Sep 17 00:00:00 2001 From: ivan-degtiarenko Date: Wed, 11 Jan 2023 20:07:06 +0100 Subject: [PATCH 1/4] Initialize counter metrics with 0 value --- .../dinosaurmgrs/accepted_centrals_mgr.go | 5 +++- .../dinosaurmgrs/deleting_dinosaurs_mgr.go | 7 +++++- .../dinosaurmgrs/dinosaurs_auth_config_mgr.go | 7 ++++-- .../dinosaurs_routes_cname_mgr.go | 6 ++++- .../dinosaurmgrs/preparing_dinosaurs_mgr.go | 5 +++- .../provisioning_dinosaurs_mgr.go | 5 +++- .../dinosaurmgrs/ready_dinosaurs_mgr.go | 6 ++++- pkg/metrics/metrics.go | 24 +++++++++++++++++++ 8 files changed, 57 insertions(+), 8 deletions(-) diff --git a/internal/dinosaur/pkg/workers/dinosaurmgrs/accepted_centrals_mgr.go b/internal/dinosaur/pkg/workers/dinosaurmgrs/accepted_centrals_mgr.go index e3fe1acda3..8e41a0fdc6 100644 --- a/internal/dinosaur/pkg/workers/dinosaurmgrs/accepted_centrals_mgr.go +++ b/internal/dinosaur/pkg/workers/dinosaurmgrs/accepted_centrals_mgr.go @@ -20,6 +20,8 @@ import ( "github.com/golang/glog" ) +const acceptedCentralWorkerType = "accepted_dinosaur" + // AcceptedCentralManager represents a manager that periodically reconciles central requests type AcceptedCentralManager struct { workers.BaseWorker @@ -32,10 +34,11 @@ type AcceptedCentralManager struct { // NewAcceptedCentralManager creates a new manager func NewAcceptedCentralManager(centralService services.DinosaurService, quotaServiceFactory services.QuotaServiceFactory, clusterPlmtStrategy services.ClusterPlacementStrategy, dataPlaneClusterConfig *config.DataplaneClusterConfig, centralConfig *config.CentralConfig) *AcceptedCentralManager { + metrics.InitReconcilerMetricsForType(acceptedCentralWorkerType) return &AcceptedCentralManager{ BaseWorker: workers.BaseWorker{ ID: uuid.New().String(), - WorkerType: "accepted_dinosaur", + WorkerType: acceptedCentralWorkerType, Reconciler: workers.Reconciler{}, }, centralService: centralService, diff --git a/internal/dinosaur/pkg/workers/dinosaurmgrs/deleting_dinosaurs_mgr.go b/internal/dinosaur/pkg/workers/dinosaurmgrs/deleting_dinosaurs_mgr.go index 4be405b049..81d3dba14b 100644 --- a/internal/dinosaur/pkg/workers/dinosaurmgrs/deleting_dinosaurs_mgr.go +++ b/internal/dinosaur/pkg/workers/dinosaurmgrs/deleting_dinosaurs_mgr.go @@ -4,6 +4,8 @@ import ( "context" "net/http" + "github.com/stackrox/acs-fleet-manager/pkg/metrics" + "github.com/google/uuid" "github.com/pkg/errors" "github.com/stackrox/acs-fleet-manager/internal/dinosaur/constants" @@ -19,6 +21,8 @@ import ( "github.com/golang/glog" ) +const deletingCentralWorkerType = "deleting_dinosaur" + // DeletingDinosaurManager represents a dinosaur manager that periodically reconciles dinosaur requests. type DeletingDinosaurManager struct { workers.BaseWorker @@ -31,10 +35,11 @@ type DeletingDinosaurManager struct { // NewDeletingDinosaurManager creates a new dinosaur manager. func NewDeletingDinosaurManager(dinosaurService services.DinosaurService, iamConfig *iam.IAMConfig, quotaServiceFactory services.QuotaServiceFactory) *DeletingDinosaurManager { + metrics.InitReconcilerMetricsForType(deletingCentralWorkerType) return &DeletingDinosaurManager{ BaseWorker: workers.BaseWorker{ ID: uuid.New().String(), - WorkerType: "deleting_dinosaur", + WorkerType: deletingCentralWorkerType, Reconciler: workers.Reconciler{}, }, dinosaurService: dinosaurService, diff --git a/internal/dinosaur/pkg/workers/dinosaurmgrs/dinosaurs_auth_config_mgr.go b/internal/dinosaur/pkg/workers/dinosaurmgrs/dinosaurs_auth_config_mgr.go index 4cafef56b6..3368821264 100644 --- a/internal/dinosaur/pkg/workers/dinosaurmgrs/dinosaurs_auth_config_mgr.go +++ b/internal/dinosaur/pkg/workers/dinosaurmgrs/dinosaurs_auth_config_mgr.go @@ -4,6 +4,8 @@ import ( "context" "fmt" + "github.com/stackrox/acs-fleet-manager/pkg/metrics" + "github.com/stackrox/rox/pkg/stringutils" "github.com/golang/glog" @@ -41,12 +43,13 @@ var _ workers.Worker = (*CentralAuthConfigManager)(nil) // In case this function fails, fleet-manager will fail on the startup. func NewCentralAuthConfigManager(centralService services.DinosaurService, iamConfig *iam.IAMConfig, centralConfig *config.CentralConfig) (*CentralAuthConfigManager, error) { realmConfig := iamConfig.RedhatSSORealm - if !centralConfig.HasStaticAuth() && !realmConfig.IsConfigured() { return nil, errors.Errorf("failed to create %s worker: neither static nor dynamic auth configuration was provided", centralAuthConfigManagerWorkerType) } - dynamicClientsAPI := dynamicclients.NewDynamicClientsAPI(realmConfig) + + metrics.InitReconcilerMetricsForType(centralAuthConfigManagerWorkerType) + return &CentralAuthConfigManager{ BaseWorker: workers.BaseWorker{ ID: uuid.New().String(), diff --git a/internal/dinosaur/pkg/workers/dinosaurmgrs/dinosaurs_routes_cname_mgr.go b/internal/dinosaur/pkg/workers/dinosaurmgrs/dinosaurs_routes_cname_mgr.go index af4e0e3e02..111705f690 100644 --- a/internal/dinosaur/pkg/workers/dinosaurmgrs/dinosaurs_routes_cname_mgr.go +++ b/internal/dinosaur/pkg/workers/dinosaurmgrs/dinosaurs_routes_cname_mgr.go @@ -6,9 +6,12 @@ import ( "github.com/pkg/errors" "github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/config" "github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/services" + "github.com/stackrox/acs-fleet-manager/pkg/metrics" "github.com/stackrox/acs-fleet-manager/pkg/workers" ) +const centralDNSWorkerType = "dinosaur_dns" + // DinosaurRoutesCNAMEManager ... type DinosaurRoutesCNAMEManager struct { workers.BaseWorker @@ -20,10 +23,11 @@ var _ workers.Worker = &DinosaurRoutesCNAMEManager{} // NewDinosaurCNAMEManager ... func NewDinosaurCNAMEManager(dinosaurService services.DinosaurService, kafkfConfig *config.CentralConfig) *DinosaurRoutesCNAMEManager { + metrics.InitReconcilerMetricsForType(centralDNSWorkerType) return &DinosaurRoutesCNAMEManager{ BaseWorker: workers.BaseWorker{ ID: uuid.New().String(), - WorkerType: "dinosaur_dns", + WorkerType: centralDNSWorkerType, Reconciler: workers.Reconciler{}, }, dinosaurService: dinosaurService, diff --git a/internal/dinosaur/pkg/workers/dinosaurmgrs/preparing_dinosaurs_mgr.go b/internal/dinosaur/pkg/workers/dinosaurmgrs/preparing_dinosaurs_mgr.go index 8f0d8a4ba5..eb64004ba6 100644 --- a/internal/dinosaur/pkg/workers/dinosaurmgrs/preparing_dinosaurs_mgr.go +++ b/internal/dinosaur/pkg/workers/dinosaurmgrs/preparing_dinosaurs_mgr.go @@ -19,6 +19,8 @@ import ( serviceErr "github.com/stackrox/acs-fleet-manager/pkg/errors" ) +const preparingCentralWorkerType = "preparing_dinosaur" + // PreparingDinosaurManager represents a dinosaur manager that periodically reconciles dinosaur requests type PreparingDinosaurManager struct { workers.BaseWorker @@ -28,10 +30,11 @@ type PreparingDinosaurManager struct { // NewPreparingDinosaurManager creates a new dinosaur manager func NewPreparingDinosaurManager(dinosaurService services.DinosaurService, centralConfig *config.CentralConfig) *PreparingDinosaurManager { + metrics.InitReconcilerMetricsForType(preparingCentralWorkerType) return &PreparingDinosaurManager{ BaseWorker: workers.BaseWorker{ ID: uuid.New().String(), - WorkerType: "preparing_dinosaur", + WorkerType: preparingCentralWorkerType, Reconciler: workers.Reconciler{}, }, dinosaurService: dinosaurService, diff --git a/internal/dinosaur/pkg/workers/dinosaurmgrs/provisioning_dinosaurs_mgr.go b/internal/dinosaur/pkg/workers/dinosaurmgrs/provisioning_dinosaurs_mgr.go index de3d0a3743..1b3007ccc1 100644 --- a/internal/dinosaur/pkg/workers/dinosaurmgrs/provisioning_dinosaurs_mgr.go +++ b/internal/dinosaur/pkg/workers/dinosaurmgrs/provisioning_dinosaurs_mgr.go @@ -16,6 +16,8 @@ import ( "github.com/golang/glog" ) +const provisioningCentralWorkerType = "provisioning_dinosaur" + // ProvisioningDinosaurManager represents a dinosaur manager that periodically reconciles dinosaur requests type ProvisioningDinosaurManager struct { workers.BaseWorker @@ -26,10 +28,11 @@ type ProvisioningDinosaurManager struct { // NewProvisioningDinosaurManager creates a new dinosaur manager func NewProvisioningDinosaurManager(dinosaurService services.DinosaurService, observatoriumService services.ObservatoriumService, centralConfig *config.CentralConfig) *ProvisioningDinosaurManager { + metrics.InitReconcilerMetricsForType(provisioningCentralWorkerType) return &ProvisioningDinosaurManager{ BaseWorker: workers.BaseWorker{ ID: uuid.New().String(), - WorkerType: "provisioning_dinosaur", + WorkerType: provisioningCentralWorkerType, Reconciler: workers.Reconciler{}, }, dinosaurService: dinosaurService, diff --git a/internal/dinosaur/pkg/workers/dinosaurmgrs/ready_dinosaurs_mgr.go b/internal/dinosaur/pkg/workers/dinosaurmgrs/ready_dinosaurs_mgr.go index 269c8feda6..b4014c7789 100644 --- a/internal/dinosaur/pkg/workers/dinosaurmgrs/ready_dinosaurs_mgr.go +++ b/internal/dinosaur/pkg/workers/dinosaurmgrs/ready_dinosaurs_mgr.go @@ -7,10 +7,13 @@ import ( constants2 "github.com/stackrox/acs-fleet-manager/internal/dinosaur/constants" "github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/services" "github.com/stackrox/acs-fleet-manager/pkg/client/iam" + "github.com/stackrox/acs-fleet-manager/pkg/metrics" "github.com/stackrox/acs-fleet-manager/pkg/services/sso" "github.com/stackrox/acs-fleet-manager/pkg/workers" ) +const readyCentralWorkerType = "ready_dinosaur" + // ReadyDinosaurManager represents a dinosaur manager that periodically reconciles dinosaur requests type ReadyDinosaurManager struct { workers.BaseWorker @@ -21,10 +24,11 @@ type ReadyDinosaurManager struct { // NewReadyDinosaurManager creates a new dinosaur manager func NewReadyDinosaurManager(dinosaurService services.DinosaurService, iamService sso.IAMService, iamConfig *iam.IAMConfig) *ReadyDinosaurManager { + metrics.InitReconcilerMetricsForType(readyCentralWorkerType) return &ReadyDinosaurManager{ BaseWorker: workers.BaseWorker{ ID: uuid.New().String(), - WorkerType: "ready_dinosaur", + WorkerType: readyCentralWorkerType, Reconciler: workers.Reconciler{}, }, dinosaurService: dinosaurService, diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 6877338c75..2cd89e6e6e 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -787,4 +787,28 @@ func Reset() { databaseRequestCountMetric.Reset() databaseQueryDurationMetric.Reset() + + InitCounterMetricsWithZeroEmptyLabels() +} + +// InitReconcilerMetricsForType initalizes reconciler metrics for specific reconciler type. +func InitReconcilerMetricsForType(reconcilerType string) { + labels := prometheus.Labels{ + labelWorkerType: reconcilerType, + } + reconcilerFailureCountMetric.With(labels).Add(0) + reconcilerSuccessCountMetric.With(labels).Add(0) + reconcilerErrorsCountMetric.With(labels).Add(0) +} + +// InitCounterMetricsWithZeroEmptyLabels initializes counter metrics with 0 value and empty labels. +func InitCounterMetricsWithZeroEmptyLabels() { + emptyLabels := prometheus.Labels{} + observatoriumRequestCountMetric.With(emptyLabels).Add(0) + databaseRequestCountMetric.With(emptyLabels).Add(0) + clusterOperationsSuccessCountMetric.With(emptyLabels).Add(0) + clusterOperationsTotalCountMetric.With(emptyLabels).Add(0) + centralOperationsTotalCountMetric.With(emptyLabels).Add(0) + centralOperationsSuccessCountMetric.With(emptyLabels).Add(0) + centralTimeoutCountMetric.With(emptyLabels).Add(0) } From a7c3046b63d5e33a6af50d07e36ed4dc329526ad Mon Sep 17 00:00:00 2001 From: ivan-degtiarenko Date: Thu, 12 Jan 2023 18:25:34 +0100 Subject: [PATCH 2/4] Add necessary dimensions to metric initialization --- pkg/constants/operation.go | 13 +++++++++++++ pkg/metrics/metrics.go | 36 ++++++++++++++++++++++++------------ 2 files changed, 37 insertions(+), 12 deletions(-) create mode 100644 pkg/constants/operation.go diff --git a/pkg/constants/operation.go b/pkg/constants/operation.go new file mode 100644 index 0000000000..265afcc4ba --- /dev/null +++ b/pkg/constants/operation.go @@ -0,0 +1,13 @@ +package constants + +const ( + // OperationCreate is a general name for create operation for centrals/clusters. + OperationCreate = "create" + + // OperationDelete is a general name for delete operation for centrals/clusters. + OperationDelete = "delete" +) + +// Operations is a list of possible operations values. +// It is used to initialize corresponding metrics with zero value. +var Operations = []string{OperationCreate, OperationDelete} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 2cd89e6e6e..d56e843c79 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -5,6 +5,8 @@ import ( "strconv" "time" + "github.com/stackrox/acs-fleet-manager/pkg/constants" + constants2 "github.com/stackrox/acs-fleet-manager/internal/dinosaur/constants" "github.com/stackrox/acs-fleet-manager/pkg/api" @@ -724,6 +726,8 @@ func init() { // metrics for database prometheus.MustRegister(databaseRequestCountMetric) prometheus.MustRegister(databaseQueryDurationMetric) + + InitOperationMetricsWithZero() } // ResetMetricsForCentralManagers will reset the metrics for the CentralManager background reconciler @@ -787,8 +791,6 @@ func Reset() { databaseRequestCountMetric.Reset() databaseQueryDurationMetric.Reset() - - InitCounterMetricsWithZeroEmptyLabels() } // InitReconcilerMetricsForType initalizes reconciler metrics for specific reconciler type. @@ -801,14 +803,24 @@ func InitReconcilerMetricsForType(reconcilerType string) { reconcilerErrorsCountMetric.With(labels).Add(0) } -// InitCounterMetricsWithZeroEmptyLabels initializes counter metrics with 0 value and empty labels. -func InitCounterMetricsWithZeroEmptyLabels() { - emptyLabels := prometheus.Labels{} - observatoriumRequestCountMetric.With(emptyLabels).Add(0) - databaseRequestCountMetric.With(emptyLabels).Add(0) - clusterOperationsSuccessCountMetric.With(emptyLabels).Add(0) - clusterOperationsTotalCountMetric.With(emptyLabels).Add(0) - centralOperationsTotalCountMetric.With(emptyLabels).Add(0) - centralOperationsSuccessCountMetric.With(emptyLabels).Add(0) - centralTimeoutCountMetric.With(emptyLabels).Add(0) +// InitOperationMetricsWithZero initializes counter "operation" metrics with 0 value and all possible labels. +// We do not initialize central timeout count metric as it has dynamic labels. +// We do not initialize observatorium request count metric as it is unused. +// We do not initialize database request count metric as it grows quickly and would not suffer from being undefined. +// We initialize reconciler metrics in InitReconcilerMetricsForType. +func InitOperationMetricsWithZero() { + for _, operation := range constants.Operations { + clusterOperationsSuccessCountMetric.With(prometheus.Labels{ + labelOperation: operation, + }).Add(0) + clusterOperationsTotalCountMetric.With(prometheus.Labels{ + labelOperation: operation, + }).Add(0) + centralOperationsTotalCountMetric.With(prometheus.Labels{ + labelOperation: operation, + }).Add(0) + centralOperationsSuccessCountMetric.With(prometheus.Labels{ + labelOperation: operation, + }).Add(0) + } } From 13dafb1b20188e6008083181c342d9e7b33a4b0a Mon Sep 17 00:00:00 2001 From: ivan-degtiarenko Date: Thu, 12 Jan 2023 18:35:04 +0100 Subject: [PATCH 3/4] Extract labels variable --- pkg/metrics/metrics.go | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index d56e843c79..7f141ed806 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -810,17 +810,12 @@ func InitReconcilerMetricsForType(reconcilerType string) { // We initialize reconciler metrics in InitReconcilerMetricsForType. func InitOperationMetricsWithZero() { for _, operation := range constants.Operations { - clusterOperationsSuccessCountMetric.With(prometheus.Labels{ + labels := prometheus.Labels{ labelOperation: operation, - }).Add(0) - clusterOperationsTotalCountMetric.With(prometheus.Labels{ - labelOperation: operation, - }).Add(0) - centralOperationsTotalCountMetric.With(prometheus.Labels{ - labelOperation: operation, - }).Add(0) - centralOperationsSuccessCountMetric.With(prometheus.Labels{ - labelOperation: operation, - }).Add(0) + } + clusterOperationsSuccessCountMetric.With(labels).Add(0) + clusterOperationsTotalCountMetric.With(labels).Add(0) + centralOperationsTotalCountMetric.With(labels).Add(0) + centralOperationsSuccessCountMetric.With(labels).Add(0) } } From d8eb8ddc6920704f5bff8a48ebdce70c223d153e Mon Sep 17 00:00:00 2001 From: ivan-degtiarenko Date: Fri, 13 Jan 2023 12:36:08 +0100 Subject: [PATCH 4/4] Remove metrics for create/delete operations --- pkg/constants/operation.go | 13 ------------- pkg/metrics/metrics.go | 21 --------------------- 2 files changed, 34 deletions(-) delete mode 100644 pkg/constants/operation.go diff --git a/pkg/constants/operation.go b/pkg/constants/operation.go deleted file mode 100644 index 265afcc4ba..0000000000 --- a/pkg/constants/operation.go +++ /dev/null @@ -1,13 +0,0 @@ -package constants - -const ( - // OperationCreate is a general name for create operation for centrals/clusters. - OperationCreate = "create" - - // OperationDelete is a general name for delete operation for centrals/clusters. - OperationDelete = "delete" -) - -// Operations is a list of possible operations values. -// It is used to initialize corresponding metrics with zero value. -var Operations = []string{OperationCreate, OperationDelete} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 7f141ed806..5bc7c3df6f 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -5,8 +5,6 @@ import ( "strconv" "time" - "github.com/stackrox/acs-fleet-manager/pkg/constants" - constants2 "github.com/stackrox/acs-fleet-manager/internal/dinosaur/constants" "github.com/stackrox/acs-fleet-manager/pkg/api" @@ -726,8 +724,6 @@ func init() { // metrics for database prometheus.MustRegister(databaseRequestCountMetric) prometheus.MustRegister(databaseQueryDurationMetric) - - InitOperationMetricsWithZero() } // ResetMetricsForCentralManagers will reset the metrics for the CentralManager background reconciler @@ -802,20 +798,3 @@ func InitReconcilerMetricsForType(reconcilerType string) { reconcilerSuccessCountMetric.With(labels).Add(0) reconcilerErrorsCountMetric.With(labels).Add(0) } - -// InitOperationMetricsWithZero initializes counter "operation" metrics with 0 value and all possible labels. -// We do not initialize central timeout count metric as it has dynamic labels. -// We do not initialize observatorium request count metric as it is unused. -// We do not initialize database request count metric as it grows quickly and would not suffer from being undefined. -// We initialize reconciler metrics in InitReconcilerMetricsForType. -func InitOperationMetricsWithZero() { - for _, operation := range constants.Operations { - labels := prometheus.Labels{ - labelOperation: operation, - } - clusterOperationsSuccessCountMetric.With(labels).Add(0) - clusterOperationsTotalCountMetric.With(labels).Add(0) - centralOperationsTotalCountMetric.With(labels).Add(0) - centralOperationsSuccessCountMetric.With(labels).Add(0) - } -}