From 73799a9f3314ebfcd2a51330ae8b932c3e566253 Mon Sep 17 00:00:00 2001 From: jmdeal Date: Mon, 18 Jul 2022 12:21:48 -0700 Subject: [PATCH] feat: add pod startup summary metric --- designs/metrics.md | 14 ++++---- pkg/controllers/metrics/pod/controller.go | 40 +++++++++++++++++++++-- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/designs/metrics.md b/designs/metrics.md index d30dd778b27b..a76a7c15bf18 100644 --- a/designs/metrics.md +++ b/designs/metrics.md @@ -72,10 +72,10 @@ Scheduling statistics consist of actions not representable by a single controlle termination and pod startup. The visualizations will be the same as those for individual controller performance. The following metrics will be instrumented to implement these visualizations: -| Name | Type | Labels | Description | -|--------------------------------------------------------|-----------|-----------------------------------------------------------------|----------------------------------------------------| -| `karpenter_nodes_termination_time_seconds` | Summary | None | [Measurement Definitions](#measurment-definitions) | -| `karpenter_pods_startup_time_seconds` | Summary | `provisioner`, `zone`, `arch`, `capacity_type`, `instance_type` | [Measurement Definitions](#measurment-definitions) | +| Name | Type | Labels | Description | +|--------------------------------------------------------|-----------|--------|----------------------------------------------------| +| `karpenter_nodes_termination_time_seconds` | Summary | None | [Measurement Definitions](#measurment-definitions) | +| `karpenter_pods_startup_time_seconds` | Summary | None | [Measurement Definitions](#measurment-definitions) | API statistics will consist of API call latency, call rate, call method, return code, and payload size. These statistics will be separated into Kubernetes API and cloudprovider API statistics. Call latency and call rate will be represented the same @@ -188,7 +188,7 @@ file](https://github.com/aws/karpenter/blob/main/website/content/en/v0.12.0/gett ### Measurement Definitions -**Pod Startup Latency:** Measures the time in seconds from the pod being created until the pod is running. +**Pod Startup Latency:** Measures the time in seconds from the pod being created until the pod is ready. **Node Termination Latency:** Measures the time from an initial node termination request until the node object is deleted. This differs from the termination controller latency since multiple reconciles may be required to terminate the @@ -210,8 +210,8 @@ node. | `controller_runtime_reconcile_errors_total` | Counter | `controller` | Total number of reconciliation errors per controller | `controller_runtime_reconcile_time_seconds` | Histogram | `controller` | Length of time per reconciliation per controller | `controller_runtime_reconcile_total` | Counter | `controller` | Total number of reconciliations per controller -| `karpenter_nodes_termination_time_seconds` | Summary | `provisioner`, `zone`, `arch`, `capacity_type`, `instance_type` | [Measurement Definitions](#measurment-definitions) -| `karpenter_pods_startup_time_seconds` | Summary | `provisioner`, `zone`, `arch`, `capacity_type`, `instance_type` | [Measurement Definitions](#measurment-definitions) +| `karpenter_nodes_termination_time_seconds` | Summary | None | [Measurement Definitions](#measurment-definitions) +| `karpenter_pods_startup_time_seconds` | Summary | None | [Measurement Definitions](#measurment-definitions) | `karpenter_kube_api_time_seconds` | Summary | `object`, `verb`, `code` | The duration of a call the API server | `karpenter_kube_api_payload_total_bytes` | Gauge | `object`, `verb`, `code` | The total payload size transferred to/from the API server | `karpenter_cloudprovider_api_time_seconds` | Summary | `verb`, `code` | The duration of an API call to a cloudprovider diff --git a/pkg/controllers/metrics/pod/controller.go b/pkg/controllers/metrics/pod/controller.go index cd1c7d0515c2..4b486516961b 100644 --- a/pkg/controllers/metrics/pod/controller.go +++ b/pkg/controllers/metrics/pod/controller.go @@ -26,6 +26,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -33,6 +34,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5" + "github.com/aws/karpenter/pkg/metrics" ) const ( @@ -46,6 +48,8 @@ const ( podHostCapacityType = "capacity_type" podHostInstanceType = "instance_type" podPhase = "phase" + + phasePending = "Pending" ) var ( @@ -58,16 +62,28 @@ var ( }, labelNames(), ) + + podStartupTimeSummary = prometheus.NewSummary( + prometheus.SummaryOpts{ + Namespace: "karpenter", + Subsystem: "pods", + Name: "startup_time_seconds", + Help: "The time from pod creation until the pod is running.", + Objectives: metrics.SummaryObjectives(), + }, + ) ) // Controller for the resource type Controller struct { - kubeClient client.Client - labelsMap sync.Map + kubeClient client.Client + labelsMap sync.Map + pendingPods sets.String } func init() { crmetrics.Registry.MustRegister(podGaugeVec) + crmetrics.Registry.MustRegister(podStartupTimeSummary) } func labelNames() []string { @@ -88,7 +104,8 @@ func labelNames() []string { // NewController constructs a controller instance func NewController(kubeClient client.Client) *Controller { return &Controller{ - kubeClient: kubeClient, + kubeClient: kubeClient, + pendingPods: sets.NewString(), } } @@ -112,9 +129,26 @@ func (c *Controller) Reconcile(ctx context.Context, req reconcile.Request) (reco } func (c *Controller) record(ctx context.Context, pod *v1.Pod) { + // Record pods state metric labels := c.labels(ctx, pod) podGaugeVec.With(labels).Set(float64(1)) c.labelsMap.Store(client.ObjectKeyFromObject(pod), labels) + + // Record pods startup time metric + var condition *v1.PodCondition + for i := range pod.Status.Conditions { + if pod.Status.Conditions[i].Type == v1.PodReady { + condition = &pod.Status.Conditions[i] + } + } + + podKey := client.ObjectKeyFromObject(pod).String() + if pod.Status.Phase == phasePending { + c.pendingPods.Insert(podKey) + } else if c.pendingPods.Has(podKey) && condition != nil { + podStartupTimeSummary.Observe(condition.LastTransitionTime.Sub(pod.CreationTimestamp.Time).Seconds()) + c.pendingPods.Delete(podKey) + } } func (c *Controller) Register(ctx context.Context, m manager.Manager) error {