feat: add pod startup summary metric (aws#2146)

o11n · Jul 20, 2022 · 28c2e3d · 28c2e3d
1 parent 6c0eead
commit 28c2e3d
Show file tree

Hide file tree

Showing 2 changed files with 44 additions and 10 deletions.
diff --git a/designs/metrics.md b/designs/metrics.md
@@ -72,10 +72,10 @@ Scheduling statistics consist of actions not representable by a single controlle
 termination and pod startup. The visualizations will be the same as those for individual controller performance. The
 following metrics will be instrumented to implement these visualizations:
 
-| Name                                                   | Type      | Labels                                                          | Description                                        |
-|--------------------------------------------------------|-----------|-----------------------------------------------------------------|----------------------------------------------------|
-| `karpenter_nodes_termination_time_seconds`             | Summary   | None                                                            | [Measurement Definitions](#measurment-definitions) | 
-| `karpenter_pods_startup_time_seconds`                  | Summary   | `provisioner`, `zone`, `arch`, `capacity_type`, `instance_type` | [Measurement Definitions](#measurment-definitions) |
+| Name                                                   | Type      | Labels | Description                                        |
+|--------------------------------------------------------|-----------|--------|----------------------------------------------------|
+| `karpenter_nodes_termination_time_seconds`             | Summary   | None   | [Measurement Definitions](#measurment-definitions) | 
+| `karpenter_pods_startup_time_seconds`                  | Summary   | None   | [Measurement Definitions](#measurment-definitions) |
 
 API statistics will consist of API call latency, call rate, call method, return code, and payload size. These statistics will be
 separated into Kubernetes API and cloudprovider API statistics. Call latency and call rate will be represented the same
@@ -188,7 +188,7 @@ file](https://github.com/aws/karpenter/blob/main/website/content/en/v0.12.0/gett
 
 ### Measurement Definitions
 
-**Pod Startup Latency:** Measures the time in seconds from the pod being created until the pod is running.
+**Pod Startup Latency:** Measures the time in seconds from the pod being created until the pod is ready.
 
 **Node Termination Latency:** Measures the time from an initial node termination request until the node object is
 deleted. This differs from the termination controller latency since multiple reconciles may be required to terminate the
@@ -210,8 +210,8 @@ node.
 | `controller_runtime_reconcile_errors_total`       | Counter   | `controller`                                                                                           | Total number of reconciliation errors per controller
 | `controller_runtime_reconcile_time_seconds`       | Histogram | `controller`                                                                                           | Length of time per reconciliation per controller
 | `controller_runtime_reconcile_total`              | Counter   | `controller`                                                                                           | Total number of reconciliations per controller
-| `karpenter_nodes_termination_time_seconds`        | Summary   | `provisioner`, `zone`, `arch`, `capacity_type`, `instance_type`                                        | [Measurement Definitions](#measurment-definitions)
-| `karpenter_pods_startup_time_seconds`             | Summary   | `provisioner`, `zone`, `arch`, `capacity_type`, `instance_type`                                        | [Measurement Definitions](#measurment-definitions)
+| `karpenter_nodes_termination_time_seconds`        | Summary   | None                                                                                                   | [Measurement Definitions](#measurment-definitions)
+| `karpenter_pods_startup_time_seconds`             | Summary   | None                                                                                                   | [Measurement Definitions](#measurment-definitions)
 | `karpenter_kube_api_time_seconds`                 | Summary   | `object`, `verb`, `code`                                                                               | The duration of a call the API server
 | `karpenter_kube_api_payload_total_bytes`          | Gauge     | `object`, `verb`, `code`                                                                               | The total payload size transferred to/from the API server
 | `karpenter_cloudprovider_api_time_seconds`        | Summary   | `verb`, `code`                                                                                         | The duration of an API call to a cloudprovider

diff --git a/pkg/controllers/metrics/pod/controller.go b/pkg/controllers/metrics/pod/controller.go
@@ -26,13 +26,15 @@ import (
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/sets"
 	controllerruntime "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	crmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	"github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5"
+	"github.com/aws/karpenter/pkg/metrics"
 )
 
 const (
@@ -46,6 +48,8 @@ const (
 	podHostCapacityType = "capacity_type"
 	podHostInstanceType = "instance_type"
 	podPhase            = "phase"
+
+	phasePending = "Pending"
 )
 
 var (
@@ -58,16 +62,28 @@ var (
 		},
 		labelNames(),
 	)
+
+	podStartupTimeSummary = prometheus.NewSummary(
+		prometheus.SummaryOpts{
+			Namespace:  "karpenter",
+			Subsystem:  "pods",
+			Name:       "startup_time_seconds",
+			Help:       "The time from pod creation until the pod is running.",
+			Objectives: metrics.SummaryObjectives(),
+		},
+	)
 )
 
 // Controller for the resource
 type Controller struct {
-	kubeClient client.Client
-	labelsMap  sync.Map
+	kubeClient  client.Client
+	labelsMap   sync.Map
+	pendingPods sets.String
 }
 
 func init() {
 	crmetrics.Registry.MustRegister(podGaugeVec)
+	crmetrics.Registry.MustRegister(podStartupTimeSummary)
 }
 
 func labelNames() []string {
@@ -88,7 +104,8 @@ func labelNames() []string {
 // NewController constructs a controller instance
 func NewController(kubeClient client.Client) *Controller {
 	return &Controller{
-		kubeClient: kubeClient,
+		kubeClient:  kubeClient,
+		pendingPods: sets.NewString(),
 	}
 }
 
@@ -112,9 +129,26 @@ func (c *Controller) Reconcile(ctx context.Context, req reconcile.Request) (reco
 }
 
 func (c *Controller) record(ctx context.Context, pod *v1.Pod) {
+	// Record pods state metric
 	labels := c.labels(ctx, pod)
 	podGaugeVec.With(labels).Set(float64(1))
 	c.labelsMap.Store(client.ObjectKeyFromObject(pod), labels)
+
+	// Record pods startup time metric
+	var condition *v1.PodCondition
+	for i := range pod.Status.Conditions {
+		if pod.Status.Conditions[i].Type == v1.PodReady {
+			condition = &pod.Status.Conditions[i]
+		}
+	}
+
+	podKey := client.ObjectKeyFromObject(pod).String()
+	if pod.Status.Phase == phasePending {
+		c.pendingPods.Insert(podKey)
+	} else if c.pendingPods.Has(podKey) && condition != nil {
+		podStartupTimeSummary.Observe(condition.LastTransitionTime.Sub(pod.CreationTimestamp.Time).Seconds())
+		c.pendingPods.Delete(podKey)
+	}
 }
 
 func (c *Controller) Register(ctx context.Context, m manager.Manager) error {