feature: optimise the logic of using prometheus

1. use the custom registry to replace the default.
2. upgrade the package called client-go of prometheus
3. add the new dependent package called promhttp

Related to:  AliyunContainerService#2395
Signed-off-by:  fengzixu <[email protected]>

apis/metrics/metrics.go

@@ -4,8 +4,6 @@ import (
 	"sync"
 
 	util_metrics "github.com/alibaba/pouch/pkg/utils/metrics"
-
-	"github.com/prometheus/client_golang/prometheus"
 )
 
 func init() {
@@ -47,15 +45,17 @@ var registerMetrics sync.Once
 
 // Register all metrics.
 func Register() {
-	// Register the metrics.
+	// Get a custom prometheus registry.
+	registry := util_metrics.GetCustomPrometheusRegistry()
 	registerMetrics.Do(func() {
-		prometheus.MustRegister(ImagePullSummary)
-		prometheus.MustRegister(EngineVersion)
-		prometheus.MustRegister(ContainerActionsCounter)
-		prometheus.MustRegister(ContainerSuccessActionsCounter)
-		prometheus.MustRegister(ImageActionsCounter)
-		prometheus.MustRegister(ImageSuccessActionsCounter)
-		prometheus.MustRegister(ContainerActionsTimer)
-		prometheus.MustRegister(ImageActionsTimer)
+		// Register the custom metrics.
+		registry.MustRegister(ImagePullSummary)
+		registry.MustRegister(EngineVersion)
+		registry.MustRegister(ContainerActionsCounter)
+		registry.MustRegister(ContainerSuccessActionsCounter)
+		registry.MustRegister(ImageActionsCounter)
+		registry.MustRegister(ImageSuccessActionsCounter)
+		registry.MustRegister(ContainerActionsTimer)
+		registry.MustRegister(ImageActionsTimer)
 	})
 }

apis/server/router.go

@@ -11,9 +11,9 @@ import (
 	"github.com/alibaba/pouch/apis/types"
 	"github.com/alibaba/pouch/pkg/errtypes"
 	"github.com/alibaba/pouch/pkg/httputils"
+	util_metrics "github.com/alibaba/pouch/pkg/utils/metrics"
 
 	"github.com/gorilla/mux"
-	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 )
 
@@ -88,8 +88,8 @@ func initRoute(s *Server) http.Handler {
 	s.addRoute(r, http.MethodPost, "/networks/{id:.*}/disconnect", s.disconnectNetwork)
 
 	// metrics
-	r.Path(versionMatcher + "/metrics").Methods(http.MethodGet).Handler(prometheus.Handler())
-	r.Path("/metrics").Methods(http.MethodGet).Handler(prometheus.Handler())
+	r.Path(versionMatcher + "/metrics").Methods(http.MethodGet).Handler(util_metrics.GetPrometheusHandler())
+	r.Path("/metrics").Methods(http.MethodGet).Handler(util_metrics.GetPrometheusHandler())
 
 	// CRI stream server related handlers
 	if s.StreamRouter != nil {

cri/metrics/metrics.go

@@ -6,7 +6,6 @@ import (
 	util_metrics "github.com/alibaba/pouch/pkg/utils/metrics"
 
 	"github.com/grpc-ecosystem/go-grpc-prometheus"
-	"github.com/prometheus/client_golang/prometheus"
 )
 
 func init() {
@@ -75,22 +74,26 @@ var registerMetrics sync.Once
 
 // Register all metrics.
 func Register() {
+	// Get a custom prometheus registry.
+	registry := util_metrics.GetCustomPrometheusRegistry()
 	registerMetrics.Do(func() {
-		prometheus.MustRegister(PodActionsCounter)
-		prometheus.MustRegister(PodSuccessActionsCounter)
-		prometheus.MustRegister(PodActionsTimer)
-		prometheus.MustRegister(ContainerActionsCounter)
-		prometheus.MustRegister(ContainerSuccessActionsCounter)
-		prometheus.MustRegister(ContainerActionsTimer)
-		prometheus.MustRegister(ImagePullSummary)
-		prometheus.MustRegister(ImageActionsCounter)
-		prometheus.MustRegister(ImageSuccessActionsCounter)
-		prometheus.MustRegister(ImageActionsTimer)
-		prometheus.MustRegister(VolumeActionsCounter)
-		prometheus.MustRegister(VolumeSuccessActionsCounter)
-		prometheus.MustRegister(VolumeActionsTimer)
-		prometheus.MustRegister(RuntimeActionsCounter)
-		prometheus.MustRegister(RuntimeSuccessActionsCounter)
-		prometheus.MustRegister(RuntimeActionsTimer)
+		// Register the custom metrics.
+		registry.MustRegister(PodActionsCounter)
+		registry.MustRegister(PodSuccessActionsCounter)
+		registry.MustRegister(PodActionsTimer)
+		registry.MustRegister(ContainerActionsCounter)
+		registry.MustRegister(ContainerSuccessActionsCounter)
+		registry.MustRegister(ContainerActionsTimer)
+		registry.MustRegister(ImagePullSummary)
+		registry.MustRegister(ImageActionsCounter)
+		registry.MustRegister(ImageSuccessActionsCounter)
+		registry.MustRegister(ImageActionsTimer)
+		registry.MustRegister(VolumeActionsCounter)
+		registry.MustRegister(VolumeSuccessActionsCounter)
+		registry.MustRegister(VolumeActionsTimer)
+		registry.MustRegister(RuntimeActionsCounter)
+		registry.MustRegister(RuntimeSuccessActionsCounter)
+		registry.MustRegister(RuntimeActionsTimer)
+		registry.MustRegister(GRPCMetrics)
 	})
 }

pkg/utils/metrics/metrics.go

@@ -2,15 +2,32 @@ package metrics
 
 import (
 	"fmt"
+	"net/http"
+	"os"
+	"sync"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 const (
 	namespace = "engine"
 )
 
+// CustomPrometheusRegistry creates a custom prometheus registry.
+var (
+	customPrometheusRegistry *prometheus.Registry
+	prometheusHandler        http.Handler
+	registerMetrics          sync.Once
+)
+
+func init() {
+	customPrometheusRegistry = prometheus.NewRegistry()
+	prometheusHandler = promhttp.HandlerFor(customPrometheusRegistry, promhttp.HandlerOpts{})
+	registerDefaultMetrics(customPrometheusRegistry)
+}
+
 // SinceInMicroseconds gets the time since the specified start in microseconds.
 func SinceInMicroseconds(start time.Time) float64 {
 	return float64(time.Since(start).Nanoseconds() / time.Microsecond.Nanoseconds())
@@ -63,3 +80,21 @@ func NewLabelTimer(subsystem, name, help string, labels ...string) *prometheus.H
 			ConstLabels: nil,
 		}, labels)
 }
+
+// GetCustomPrometheusRegistry create a custom resigtry of Prometheus.
+func GetCustomPrometheusRegistry() *prometheus.Registry {
+	return customPrometheusRegistry
+}
+
+// GetPrometheusHandler return the prometheus handler.
+func GetPrometheusHandler() http.Handler {
+	return prometheusHandler
+}
+
+func registerDefaultMetrics(registry *prometheus.Registry) {
+	//Register the default metrics to the custom registry in prometheus.
+	registerMetrics.Do(func() {
+		registry.MustRegister(prometheus.NewProcessCollector(os.Getpid(), ""))
+		registry.MustRegister(prometheus.NewGoCollector())
+	})
+}