Measure time from Application creation/modification to Chart Installation

cmd/shipper/main.go

@@ -38,6 +38,7 @@ import (
 	"github.com/bookingcom/shipper/pkg/controller/capacity"
 	"github.com/bookingcom/shipper/pkg/controller/installation"
 	"github.com/bookingcom/shipper/pkg/controller/janitor"
+	"github.com/bookingcom/shipper/pkg/controller/metrics"
 	"github.com/bookingcom/shipper/pkg/controller/release"
 	"github.com/bookingcom/shipper/pkg/controller/rolloutblock"
 	"github.com/bookingcom/shipper/pkg/controller/traffic"
@@ -55,6 +56,7 @@ var controllers = []string{
 	"rolloutblock",
 	"janitor",
 	"webhook",
+	"metrics",
 }
 
 const defaultRESTTimeout time.Duration = 10 * time.Second
@@ -82,10 +84,11 @@ var (
 type metricsCfg struct {
 	readyCh chan struct{}
 
-	wqMetrics   *shippermetrics.PrometheusWorkqueueProvider
-	restLatency *shippermetrics.RESTLatencyMetric
-	restResult  *shippermetrics.RESTResultMetric
-	certExpire  *shippermetrics.WebhookMetric
+	wqMetrics     *shippermetrics.PrometheusWorkqueueProvider
+	restLatency   *shippermetrics.RESTLatencyMetric
+	restResult    *shippermetrics.RESTResultMetric
+	certExpire    *shippermetrics.WebhookMetric
+	metricsBundle *metrics.MetricsBundle
 }
 
 type cfg struct {
@@ -210,11 +213,12 @@ func main() {
 		stopCh: stopCh,
 
 		metrics: &metricsCfg{
-			readyCh:     metricsReadyCh,
-			wqMetrics:   shippermetrics.NewProvider(),
-			restLatency: shippermetrics.NewRESTLatencyMetric(),
-			restResult:  shippermetrics.NewRESTResultMetric(),
-			certExpire:   shippermetrics.NewTLSCertExpireMetric(),
+			readyCh:       metricsReadyCh,
+			wqMetrics:     shippermetrics.NewProvider(),
+			restLatency:   shippermetrics.NewRESTLatencyMetric(),
+			restResult:    shippermetrics.NewRESTResultMetric(),
+			certExpire:    shippermetrics.NewTLSCertExpireMetric(),
+			metricsBundle: metrics.NewMetricsBundle(),
 		},
 	}
 
@@ -244,6 +248,7 @@ func runMetrics(cfg *metricsCfg) {
 	prometheus.MustRegister(cfg.restLatency.Summary, cfg.restResult.Counter)
 	prometheus.MustRegister(cfg.certExpire.GetMetrics()...)
 	prometheus.MustRegister(instrumentedclient.GetMetrics()...)
+	prometheus.MustRegister(cfg.metricsBundle.TimeToInstallation)
 
 	srv := http.Server{
 		Addr: *metricsAddr,
@@ -363,6 +368,7 @@ func buildInitializers() map[string]initFunc {
 	controllers["rolloutblock"] = startRolloutBlockController
 	controllers["janitor"] = startJanitorController
 	controllers["webhook"] = startWebhook
+	controllers["metrics"] = startMetricsController
 	return controllers
 }
 
@@ -512,6 +518,27 @@ func startRolloutBlockController(cfg *cfg) (bool, error) {
 	return true, nil
 }
 
+func startMetricsController(cfg *cfg) (bool, error) {
+	enabled := cfg.enabledControllers["metrics"]
+	if !enabled {
+		return false, nil
+	}
+
+	c := metrics.NewController(
+		client.NewShipperClientOrDie(cfg.restCfg, metrics.AgentName, cfg.restTimeout),
+		cfg.shipperInformerFactory,
+		cfg.metrics.metricsBundle,
+	)
+
+	cfg.wg.Add(1)
+	go func() {
+		c.Run(cfg.stopCh)
+		cfg.wg.Done()
+	}()
+
+	return true, nil
+}
+
 func startWebhook(cfg *cfg) (bool, error) {
 	enabled := cfg.enabledControllers["webhook"]
 	if !enabled {

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/pmezard/go-difflib v1.0.0
 	github.com/prometheus/client_golang v1.7.1
+	github.com/prometheus/client_model v0.2.0
 	github.com/rodaine/table v1.0.1
 	github.com/satori/go.uuid v1.2.0 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect

pkg/controller/metrics/metrics_bundle.go

@@ -0,0 +1,21 @@
+package metrics
+
+import "github.com/prometheus/client_golang/prometheus"
+
+// MetricsBundle encapsulates all the metrics that the Metrics Controller reports on.
+type MetricsBundle struct {
+	TimeToInstallation prometheus.Histogram
+}
+
+func NewMetricsBundle() *MetricsBundle {
+	timeToInstallation := prometheus.NewHistogram(prometheus.HistogramOpts{
+		Namespace: "shipper",
+		Subsystem: "metrics_controller",
+		Name:      "time_to_installation",
+		Help:      "The time it takes for a Release to be installed on all of the clusters it's scheduled on",
+	})
+
+	return &MetricsBundle{
+		TimeToInstallation: timeToInstallation,
+	}
+}