diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 47569d23c..d59152b9f 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -32,8 +32,9 @@ const ( // LabelKind is a metrics label indicates kind of snapshot associated with metric. LabelKind = "kind" - namespaceEtcdBR = "etcdbr" - subsystemSnapshot = "snapshot" + namespaceEtcdBR = "etcdbr" + subsystemSnapshot = "snapshot" + subsystemSnapstore = "snapstore" ) var ( @@ -131,6 +132,27 @@ var ( }, []string{LabelSucceeded}, ) + + // SnapstoreLatestDeltasTotal is metric to expose total number of delta snapshots taken since the latest full snapshot. + SnapstoreLatestDeltasTotal = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespaceEtcdBR, + Subsystem: subsystemSnapstore, + Name: "latest_deltas_total", + Help: "Total number of delta snapshots taken since the latest full snapshot.", + }, + []string{}, + ) + // SnapstoreLatestDeltasRevisionsTotal is metric to expose total number of revisions stored in delta snapshots taken since the latest full snapshot. + SnapstoreLatestDeltasRevisionsTotal = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespaceEtcdBR, + Subsystem: subsystemSnapstore, + Name: "latest_deltas_revisions_total", + Help: "Total number of revisions stored in delta snapshots taken since the latest full snapshot.", + }, + []string{}, + ) ) // generateLabelCombinations generates combinations of label values for metrics @@ -291,6 +313,12 @@ func init() { DefragmentationDurationSeconds.With(prometheus.Labels(combination)) } + // SnapstoreLatestDeltasTotal + SnapstoreLatestDeltasTotal.With(prometheus.Labels(map[string]string{})) + + // SnapstoreLatestDeltasSize + SnapstoreLatestDeltasRevisionsTotal.With(prometheus.Labels(map[string]string{})) + // Metrics have to be registered to be exposed: prometheus.MustRegister(GCSnapshotCounter) @@ -302,4 +330,7 @@ func init() { prometheus.MustRegister(RestorationDurationSeconds) prometheus.MustRegister(ValidationDurationSeconds) prometheus.MustRegister(DefragmentationDurationSeconds) + + prometheus.MustRegister(SnapstoreLatestDeltasTotal) + prometheus.MustRegister(SnapstoreLatestDeltasRevisionsTotal) } diff --git a/pkg/miscellaneous/miscellaneous.go b/pkg/miscellaneous/miscellaneous.go index 72a7288ff..1512307d4 100644 --- a/pkg/miscellaneous/miscellaneous.go +++ b/pkg/miscellaneous/miscellaneous.go @@ -17,7 +17,9 @@ package miscellaneous import ( "sort" + "github.com/gardener/etcd-backup-restore/pkg/metrics" "github.com/gardener/etcd-backup-restore/pkg/snapstore" + "github.com/prometheus/client_golang/prometheus" ) // GetLatestFullSnapshotAndDeltaSnapList returns the latest snapshot @@ -28,16 +30,25 @@ func GetLatestFullSnapshotAndDeltaSnapList(store snapstore.SnapStore) (*snapstor return nil, nil, err } + metrics.SnapstoreLatestDeltasTotal.With(prometheus.Labels{}).Set(0) + metrics.SnapstoreLatestDeltasRevisionsTotal.With(prometheus.Labels{}).Set(0) + for index := len(snapList); index > 0; index-- { if snapList[index-1].IsChunk { continue } if snapList[index-1].Kind == snapstore.SnapshotKindFull { sort.Sort(deltaSnapList) + metrics.SnapstoreLatestDeltasTotal.With(prometheus.Labels{}).Set(float64(len(deltaSnapList))) + revisionDiff := deltaSnapList[len(deltaSnapList)-1].LastRevision - snapList[index-1].LastRevision + metrics.SnapstoreLatestDeltasRevisionsTotal.With(prometheus.Labels{}).Set(float64(revisionDiff)) return snapList[index-1], deltaSnapList, nil } deltaSnapList = append(deltaSnapList, snapList[index-1]) } sort.Sort(deltaSnapList) //added to ensure the list is well formed for only deltasnapshots scenarios as well + metrics.SnapstoreLatestDeltasTotal.With(prometheus.Labels{}).Set(float64(len(deltaSnapList))) + revisionDiff := deltaSnapList[len(deltaSnapList)-1].LastRevision + metrics.SnapstoreLatestDeltasRevisionsTotal.With(prometheus.Labels{}).Set(float64(revisionDiff)) return nil, deltaSnapList, nil } diff --git a/pkg/snapshot/snapshotter/snapshotter.go b/pkg/snapshot/snapshotter/snapshotter.go index 634472357..b7ff48f43 100644 --- a/pkg/snapshot/snapshotter/snapshotter.go +++ b/pkg/snapshot/snapshotter/snapshotter.go @@ -273,6 +273,8 @@ func (ssr *Snapshotter) takeFullSnapshot() error { metrics.LatestSnapshotRevision.With(prometheus.Labels{metrics.LabelKind: ssr.prevSnapshot.Kind}).Set(float64(ssr.prevSnapshot.LastRevision)) metrics.LatestSnapshotTimestamp.With(prometheus.Labels{metrics.LabelKind: ssr.prevSnapshot.Kind}).Set(float64(ssr.prevSnapshot.CreatedOn.Unix())) + metrics.SnapstoreLatestDeltasTotal.With(prometheus.Labels{}).Set(0) + metrics.SnapstoreLatestDeltasRevisionsTotal.With(prometheus.Labels{}).Set(0) ssr.logger.Infof("Successfully saved full snapshot at: %s", path.Join(s.SnapDir, s.SnapName)) } @@ -356,6 +358,8 @@ func (ssr *Snapshotter) TakeDeltaSnapshot() error { metrics.LatestSnapshotRevision.With(prometheus.Labels{metrics.LabelKind: ssr.prevSnapshot.Kind}).Set(float64(ssr.prevSnapshot.LastRevision)) metrics.LatestSnapshotTimestamp.With(prometheus.Labels{metrics.LabelKind: ssr.prevSnapshot.Kind}).Set(float64(ssr.prevSnapshot.CreatedOn.Unix())) metrics.SnapshotRequired.With(prometheus.Labels{metrics.LabelKind: snapstore.SnapshotKindDelta}).Set(0) + metrics.SnapstoreLatestDeltasTotal.With(prometheus.Labels{}).Inc() + metrics.SnapstoreLatestDeltasRevisionsTotal.With(prometheus.Labels{}).Add(float64(snap.LastRevision - snap.StartRevision)) ssr.logger.Infof("Successfully saved delta snapshot at: %s", path.Join(snap.SnapDir, snap.SnapName)) return nil }