Skip to content

Commit

Permalink
Add lease list metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
carreter committed Aug 23, 2024
1 parent 175cb5e commit 768a3b2
Show file tree
Hide file tree
Showing 2 changed files with 46 additions and 0 deletions.
14 changes: 14 additions & 0 deletions pkg/server/leases/gc_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,11 +54,25 @@ func (c *GarbageCollectionController) Run(ctx context.Context) {
}

func (c *GarbageCollectionController) gc(ctx context.Context) {
start := time.Now()
leases, err := c.leaseInterface.List(ctx, metav1.ListOptions{LabelSelector: c.labelSelector})
latency := time.Now().Sub(start)
if err != nil {
klog.Errorf("Could not list leases to garbage collect: %v", err)

var apiStatus apierrors.APIStatus
if errors.As(err, &apiStatus) {
status := apiStatus.Status()
metrics.Metrics.ObserveLeaseList(int(status.Code), string(status.Reason))
metrics.Metrics.ObserveLeaseListLatency(int(status.Code), latency)
} else {
klog.Errorf("Lease list error could not be logged to metrics as it is not an APIStatus: %v", err)
}

return
}
metrics.Metrics.ObserveLeaseList(200, "")
metrics.Metrics.ObserveLeaseListLatency(200, latency)

for _, lease := range leases.Items {
if util.IsLeaseValid(c.pc, lease) {
Expand Down
32 changes: 32 additions & 0 deletions pkg/server/metrics/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,8 @@ type ServerMetrics struct {
culledLeases prometheus.Counter
leaseDeleteLatencies *prometheus.HistogramVec
leaseDeletes *prometheus.CounterVec
leaseListLatencies *prometheus.HistogramVec
leaseLists *prometheus.CounterVec
}

// newServerMetrics create a new ServerMetrics, configured with default metric names.
Expand Down Expand Up @@ -176,6 +178,24 @@ func newServerMetrics() *ServerMetrics {
},
[]string{"http_status_code", "reason"},
)
leaseListLatencies := prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: Namespace,
Subsystem: Subsystem,
Name: "lease_list_latency_seconds",
Help: "Latency of lease list calls by the garbage collection controller in seconds.",
},
[]string{"http_status_code"},
)
leaseLists := prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: Namespace,
Subsystem: Subsystem,
Name: "lease_list_total",
Help: "Count of lease deletion calls by the garbage collection controller. Labeled by HTTP status code and reason.",
},
[]string{"http_status_code", "reason"},
)
streamPackets := commonmetrics.MakeStreamPacketsTotalMetric(Namespace, Subsystem)
streamErrors := commonmetrics.MakeStreamErrorsTotalMetric(Namespace, Subsystem)
prometheus.MustRegister(endpointLatencies)
Expand All @@ -192,6 +212,8 @@ func newServerMetrics() *ServerMetrics {
prometheus.MustRegister(culledLeases)
prometheus.MustRegister(leaseDeleteLatencies)
prometheus.MustRegister(leaseDeletes)
prometheus.MustRegister(leaseListLatencies)
prometheus.MustRegister(leaseLists)
return &ServerMetrics{
endpointLatencies: endpointLatencies,
frontendLatencies: frontendLatencies,
Expand All @@ -207,6 +229,8 @@ func newServerMetrics() *ServerMetrics {
culledLeases: culledLeases,
leaseDeleteLatencies: leaseDeleteLatencies,
leaseDeletes: leaseDeletes,
leaseListLatencies: leaseListLatencies,
leaseLists: leaseLists,
}
}

Expand Down Expand Up @@ -309,3 +333,11 @@ func (s *ServerMetrics) ObserveLeaseDeleteLatency(httpCode int, latency time.Dur
func (s *ServerMetrics) ObserveLeaseDelete(httpCode int, reason string) {
s.leaseDeletes.WithLabelValues(strconv.Itoa(httpCode), reason).Inc()
}

func (s *ServerMetrics) ObserveLeaseListLatency(httpCode int, latency time.Duration) {
s.leaseListLatencies.WithLabelValues(strconv.Itoa(httpCode)).Observe(latency.Seconds())
}

func (s *ServerMetrics) ObserveLeaseList(httpCode int, reason string) {
s.leaseLists.WithLabelValues(strconv.Itoa(httpCode), reason).Inc()
}

0 comments on commit 768a3b2

Please sign in to comment.