feat: add pod startup summary metric

aws · Jul 19, 2022 · 5dbac87 · 5dbac87
1 parent 7a818e0
commit 5dbac87
Show file tree

Hide file tree

Showing 2 changed files with 50 additions and 12 deletions.
diff --git a/designs/metrics.md b/designs/metrics.md
@@ -72,10 +72,10 @@ Scheduling statistics consist of actions not representable by a single controlle
 termination and pod startup. The visualizations will be the same as those for individual controller performance. The
 following metrics will be instrumented to implement these visualizations:
 
-| Name                                                   | Type      | Labels                                                          | Description                                        |
-|--------------------------------------------------------|-----------|-----------------------------------------------------------------|----------------------------------------------------|
-| `karpenter_nodes_termination_time_seconds`             | Summary   | None                                                            | [Measurement Definitions](#measurment-definitions) | 
-| `karpenter_pods_startup_time_seconds`                  | Summary   | `provisioner`, `zone`, `arch`, `capacity_type`, `instance_type` | [Measurement Definitions](#measurment-definitions) |
+| Name                                                   | Type      | Labels | Description                                        |
+|--------------------------------------------------------|-----------|--------|----------------------------------------------------|
+| `karpenter_nodes_termination_time_seconds`             | Summary   | None   | [Measurement Definitions](#measurment-definitions) | 
+| `karpenter_pods_startup_time_seconds`                  | Summary   | None   | [Measurement Definitions](#measurment-definitions) |
 
 API statistics will consist of API call latency, call rate, call method, return code, and payload size. These statistics will be
 separated into Kubernetes API and cloudprovider API statistics. Call latency and call rate will be represented the same
@@ -188,7 +188,7 @@ file](https://github.com/aws/karpenter/blob/main/website/content/en/v0.12.0/gett
 
 ### Measurement Definitions
 
-**Pod Startup Latency:** Measures the time in seconds from the pod being created until the pod is running.
+**Pod Startup Latency:** Measures the time in seconds from the pod being created until the pod is ready.
 
 **Node Termination Latency:** Measures the time from an initial node termination request until the node object is
 deleted. This differs from the termination controller latency since multiple reconciles may be required to terminate the
@@ -210,8 +210,8 @@ node.
 | `controller_runtime_reconcile_errors_total`       | Counter   | `controller`                                                                                           | Total number of reconciliation errors per controller
 | `controller_runtime_reconcile_time_seconds`       | Histogram | `controller`                                                                                           | Length of time per reconciliation per controller
 | `controller_runtime_reconcile_total`              | Counter   | `controller`                                                                                           | Total number of reconciliations per controller
-| `karpenter_nodes_termination_time_seconds`        | Summary   | `provisioner`, `zone`, `arch`, `capacity_type`, `instance_type`                                        | [Measurement Definitions](#measurment-definitions)
-| `karpenter_pods_startup_time_seconds`             | Summary   | `provisioner`, `zone`, `arch`, `capacity_type`, `instance_type`                                        | [Measurement Definitions](#measurment-definitions)
+| `karpenter_nodes_termination_time_seconds`        | Summary   | None                                                                                                   | [Measurement Definitions](#measurment-definitions)
+| `karpenter_pods_startup_time_seconds`             | Summary   | None                                                                                                   | [Measurement Definitions](#measurment-definitions)
 | `karpenter_kube_api_time_seconds`                 | Summary   | `object`, `verb`, `code`                                                                               | The duration of a call the API server
 | `karpenter_kube_api_payload_total_bytes`          | Gauge     | `object`, `verb`, `code`                                                                               | The total payload size transferred to/from the API server
 | `karpenter_cloudprovider_api_time_seconds`        | Summary   | `verb`, `code`                                                                                         | The duration of an API call to a cloudprovider

diff --git a/pkg/controllers/metrics/pod/controller.go b/pkg/controllers/metrics/pod/controller.go
@@ -19,20 +19,23 @@ import (
 	"fmt"
 	"strings"
 	"sync"
+	"time"
 
 	"knative.dev/pkg/logging"
 
 	"github.com/prometheus/client_golang/prometheus"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/sets"
 	controllerruntime "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	crmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	"github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5"
+	"github.com/aws/karpenter/pkg/metrics"
 )
 
 const (
@@ -46,6 +49,8 @@ const (
 	podHostCapacityType = "capacity_type"
 	podHostInstanceType = "instance_type"
 	podPhase            = "phase"
+
+	phasePending = "Pending"
 )
 
 var (
@@ -58,16 +63,28 @@ var (
 		},
 		labelNames(),
 	)
+
+	podStartupTimeSummary = prometheus.NewSummary(
+		prometheus.SummaryOpts{
+			Namespace:  "karpenter",
+			Subsystem:  "pods",
+			Name:       "startup_time_seconds",
+			Help:       "The time from pod creation until the pod is running.",
+			Objectives: metrics.SummaryObjectives(),
+		},
+	)
 )
 
 // Controller for the resource
 type Controller struct {
-	kubeClient client.Client
-	labelsMap  sync.Map
+	kubeClient  client.Client
+	labelsMap   sync.Map
+	pendingPods sets.String
 }
 
 func init() {
 	crmetrics.Registry.MustRegister(podGaugeVec)
+	crmetrics.Registry.MustRegister(podStartupTimeSummary)
 }
 
 func labelNames() []string {
@@ -88,13 +105,15 @@ func labelNames() []string {
 // NewController constructs a controller instance
 func NewController(kubeClient client.Client) *Controller {
 	return &Controller{
-		kubeClient: kubeClient,
+		kubeClient:  kubeClient,
+		pendingPods: sets.NewString(),
 	}
 }
 
 // Reconcile executes a termination control loop for the resource
 func (c *Controller) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
 	ctx = logging.WithLogger(ctx, logging.FromContext(ctx).Named("podmetrics").With("pod", req.Name))
+	ts := time.Now()
 	// Remove the previous gauge after pod labels are updated
 	if labels, ok := c.labelsMap.Load(req.NamespacedName); ok {
 		podGaugeVec.Delete(labels.(prometheus.Labels))
@@ -107,14 +126,33 @@ func (c *Controller) Reconcile(ctx context.Context, req reconcile.Request) (reco
 		}
 		return reconcile.Result{}, err
 	}
-	c.record(ctx, pod)
+	c.record(ctx, pod, ts)
 	return reconcile.Result{}, nil
 }
 
-func (c *Controller) record(ctx context.Context, pod *v1.Pod) {
+func (c *Controller) record(ctx context.Context, pod *v1.Pod, timestamp time.Time) {
+	// Record pods state metric
 	labels := c.labels(ctx, pod)
 	podGaugeVec.With(labels).Set(float64(1))
 	c.labelsMap.Store(client.ObjectKeyFromObject(pod), labels)
+
+	// Record pods startup time metric
+	getReadyCondition := func() *v1.PodCondition {
+		for _, condition := range pod.Status.Conditions {
+			if condition.Type == v1.PodReady {
+				return &condition
+			}
+		}
+		return nil
+	}
+
+	podKey := client.ObjectKeyFromObject(pod).String()
+	if pod.Status.Phase == phasePending {
+		c.pendingPods.Insert(podKey)
+	} else if condition := getReadyCondition(); c.pendingPods.Has(podKey) && condition != nil {
+		podStartupTimeSummary.Observe(condition.LastTransitionTime.Sub(pod.CreationTimestamp.Time).Seconds())
+		c.pendingPods.Delete(podKey)
+	}
 }
 
 func (c *Controller) Register(ctx context.Context, m manager.Manager) error {