design: control plane metrics monitoring

Signed-off-by: bitliu <[email protected]>

api/v1alpha1/envoygateway_helpers.go

@@ -6,6 +6,8 @@
 package v1alpha1
 
 import (
+	"fmt"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -17,10 +19,12 @@ func DefaultEnvoyGateway() *EnvoyGateway {
 			APIVersion: GroupVersion.String(),
 		},
 		EnvoyGatewaySpec{
-			Gateway:  DefaultGateway(),
-			Provider: DefaultEnvoyGatewayProvider(),
-			Logging:  DefaultEnvoyGatewayLogging(),
-			Admin:    DefaultEnvoyGatewayAdmin(),
+			Gateway:   DefaultGateway(),
+			Provider:  DefaultEnvoyGatewayProvider(),
+			Logging:   DefaultEnvoyGatewayLogging(),
+			Admin:     DefaultEnvoyGatewayAdmin(),
+			Debug:     DefaultEnvoyGatewayDebug(),
+			Telemetry: DefaultEnvoyGatewayTelemetry(),
 		},
 	}
 }
@@ -45,6 +49,12 @@ func (e *EnvoyGateway) SetEnvoyGatewayDefaults() {
 	if e.Admin == nil {
 		e.Admin = DefaultEnvoyGatewayAdmin()
 	}
+	if e.Telemetry == nil {
+		e.Telemetry = DefaultEnvoyGatewayTelemetry()
+	}
+	if e.Debug == nil {
+		e.Debug = DefaultEnvoyGatewayDebug()
+	}
 }
 
 // GetEnvoyGatewayAdmin returns the EnvoyGatewayAdmin of EnvoyGateway or a default EnvoyGatewayAdmin if unspecified.
@@ -60,6 +70,22 @@ func (e *EnvoyGateway) GetEnvoyGatewayAdmin() *EnvoyGatewayAdmin {
 	return e.Admin
 }
 
+// GetEnvoyGatewayDebug returns the EnvoyGatewayDebug of EnvoyGateway or a default EnvoyGatewayDebug if unspecified.
+func (e *EnvoyGateway) GetEnvoyGatewayDebug() *EnvoyGatewayDebug {
+	if e.Debug != nil {
+		return e.Debug
+	}
+	e.Debug = DefaultEnvoyGatewayDebug()
+
+	return e.Debug
+}
+
+// GetEnvoyGatewayAdminAddress returns the EnvoyGateway Admin Address.
+func (e *EnvoyGateway) GetEnvoyGatewayAdminAddress() string {
+	address := e.GetEnvoyGatewayAdmin().Address
+	return fmt.Sprintf("%s:%d", address.Host, address.Port)
+}
+
 // DefaultGateway returns a new Gateway with default configuration parameters.
 func DefaultGateway() *Gateway {
 	return &Gateway{
@@ -76,6 +102,51 @@ func DefaultEnvoyGatewayLogging() *EnvoyGatewayLogging {
 	}
 }
 
+// GetEnvoyGatewayAdmin returns the EnvoyGatewayAdmin of EnvoyGateway or a default EnvoyGatewayAdmin if unspecified.
+func (e *EnvoyGateway) GetEnvoyGatewayTelemetry() *EnvoyGatewayTelemetry {
+	if e.Telemetry != nil {
+		if e.Telemetry.Metrics.Prometheus == nil {
+			e.Telemetry.Metrics.Prometheus = DefaultEnvoyGatewayPrometheus()
+		}
+		if e.Telemetry.Metrics == nil {
+			e.Telemetry.Metrics = DefaultEnvoyGatewayMetrics()
+		}
+		return e.Telemetry
+	}
+	e.Telemetry = DefaultEnvoyGatewayTelemetry()
+
+	return e.Telemetry
+}
+
+func (e *EnvoyGateway) IfEnablePrometheus() bool {
+	return e.GetEnvoyGatewayTelemetry().Metrics.Prometheus.Enable
+}
+
+// DefaultEnvoyGatewayTelemetry returns a new EnvoyGatewayTelemetry with default configuration parameters.
+func DefaultEnvoyGatewayTelemetry() *EnvoyGatewayTelemetry {
+	return &EnvoyGatewayTelemetry{
+		Metrics: DefaultEnvoyGatewayMetrics(),
+	}
+}
+
+// DefaultEnvoyGatewayMetrics returns a new EnvoyGatewayMetrics with default configuration parameters.
+func DefaultEnvoyGatewayMetrics() *EnvoyGatewayMetrics {
+	return &EnvoyGatewayMetrics{
+		// Enable prometheus pull by default.
+		Prometheus: &EnvoyGatewayPrometheusProvider{
+			Enable: true,
+		},
+	}
+}
+
+// DefaultEnvoyGatewayPrometheus returns a new EnvoyGatewayMetrics with default configuration parameters.
+func DefaultEnvoyGatewayPrometheus() *EnvoyGatewayPrometheusProvider {
+	return &EnvoyGatewayPrometheusProvider{
+		// Enable prometheus pull by default.
+		Enable: true,
+	}
+}
+
 // DefaultEnvoyGatewayProvider returns a new EnvoyGatewayProvider with default configuration parameters.
 func DefaultEnvoyGatewayProvider() *EnvoyGatewayProvider {
 	return &EnvoyGatewayProvider{
@@ -103,11 +174,17 @@ func DefaultEnvoyGatewayKubeProvider() *EnvoyGatewayKubernetesProvider {
 // DefaultEnvoyGatewayAdmin returns a new EnvoyGatewayAdmin with default configuration parameters.
 func DefaultEnvoyGatewayAdmin() *EnvoyGatewayAdmin {
 	return &EnvoyGatewayAdmin{
-		Debug:   false,
 		Address: DefaultEnvoyGatewayAdminAddress(),
 	}
 }
 
+// DefaultEnvoyGatewayDebug returns a new EnvoyGatewayDebug with default configuration parameters.
+func DefaultEnvoyGatewayDebug() *EnvoyGatewayDebug {
+	return &EnvoyGatewayDebug{
+		DumpConfig: false,
+	}
+}
+
 // DefaultEnvoyGatewayAdminAddress returns a new EnvoyGatewayAdminAddress with default configuration parameters.
 func DefaultEnvoyGatewayAdminAddress() *EnvoyGatewayAdminAddress {
 	return &EnvoyGatewayAdminAddress{

api/v1alpha1/envoygateway_metric_types.go

@@ -0,0 +1,43 @@
+// Copyright Envoy Gateway Authors
+// SPDX-License-Identifier: Apache-2.0
+// The full text of the Apache license is available in the LICENSE file at
+// the root of the repo.
+
+package v1alpha1
+
+// EnvoyGatewayMetrics defines control plane push/pull metrics configurations.
+type EnvoyGatewayMetrics struct {
+	// Sinks defines the metric sinks where metrics are sent to.
+	Sinks []EnvoyGatewayMetricSink `json:"sinks,omitempty"`
+	// Prometheus defines the configuration for prometheus endpoint.
+	Prometheus *EnvoyGatewayPrometheusProvider `json:"prometheus,omitempty"`
+}
+
+// EnvoyGatewayMetricSink defines control plane
+// metric sinks where metrics are sent to.
+type EnvoyGatewayMetricSink struct {
+	// Type defines the metric sink type.
+	// EG control plane currently supports OpenTelemetry.
+	// +kubebuilder:validation:Enum=OpenTelemetry
+	// +kubebuilder:default=OpenTelemetry
+	Type MetricSinkType `json:"type"`
+	// Host define the sink service hostname.
+	Host string `json:"host"`
+	// Protocol define the sink service protocol.
+	Protocol string `json:"protocol"`
+	// Port defines the port the sink service is exposed on.
+	//
+	// +optional
+	// +kubebuilder:validation:Minimum=0
+	// +kubebuilder:default=4317
+	Port int32 `json:"port,omitempty"`
+}
+
+// EnvoyGatewayPrometheusProvider will expose prometheus endpoint in pull mode.
+type EnvoyGatewayPrometheusProvider struct {
+	// Enable defines if enables the prometheus metrics in pull mode. Default is true.
+	//
+	// +optional
+	// +kubebuilder:default=true
+	Enable bool `json:"enable,omitempty"`
+}

api/v1alpha1/envoygateway_types.go

@@ -51,13 +51,25 @@ type EnvoyGatewaySpec struct {
 	// +optional
 	// +kubebuilder:default={default: info}
 	Logging *EnvoyGatewayLogging `json:"logging,omitempty"`
+
+	// Telemetry defines telemetry related configurations for envoy gateway.
+	//
+	// +optional
+	Telemetry *EnvoyGatewayTelemetry `json:"telemetry,omitempty"`
+
 	// Admin defines the desired admin related abilities.
 	// If unspecified, the Admin is used with default configuration
 	// parameters.
 	//
 	// +optional
 	Admin *EnvoyGatewayAdmin `json:"admin,omitempty"`
 
+	// Debug defines the desired debug related abilities.
+	// If unspecified, the debug will not be running, including pprof, dump config etc.
+	//
+	// +optional
+	Debug *EnvoyGatewayDebug `json:"debug,omitempty"`
+
 	// RateLimit defines the configuration associated with the Rate Limit service
 	// deployed by Envoy Gateway required to implement the Global Rate limiting
 	// functionality. The specific rate limit service used here is the reference
@@ -79,6 +91,13 @@ type EnvoyGatewaySpec struct {
 	ExtensionAPIs *ExtensionAPISettings `json:"extensionApis,omitempty"`
 }
 
+// EnvoyGatewayTelemetry defines telemetry configurations for envoy gateway control plane.
+// Control plane will focus on metrics observability telemetry and tracing telemetry later.
+type EnvoyGatewayTelemetry struct {
+	// Metrics defines metrics configuration for envoy gateway.
+	Metrics *EnvoyGatewayMetrics `json:"metrics,omitempty"`
+}
+
 // EnvoyGatewayLogging defines logging for Envoy Gateway.
 type EnvoyGatewayLogging struct {
 	// Level is the logging level. If unspecified, defaults to "info".
@@ -424,11 +443,21 @@ type EnvoyGatewayAdmin struct {
 	//
 	// +optional
 	Address *EnvoyGatewayAdminAddress `json:"address,omitempty"`
+}
+
+// EnvoyGatewayDebug defines the Envoy Gateway Debug configuration.
+type EnvoyGatewayDebug struct {
+
+	// DumpConfig defines if dump the Envoy Gateway config in logs.
+	//
+	// +optional
+	DumpConfig bool `json:"dumpConfig,omitempty"`
 
-	// Debug defines if enable the /debug endpoint of Envoy Gateway.
+	// Address defines the address of Envoy Gateway Debug Server.
+	// Pprof will use the debug address, if you set it to non-nil.
 	//
 	// +optional
-	Debug bool `json:"debug,omitempty"`
+	Address *EnvoyGatewayDebugAddress `json:"address,omitempty"`
 }
 
 // EnvoyGatewayAdminAddress defines the Envoy Gateway Admin Address configuration.
@@ -446,6 +475,21 @@ type EnvoyGatewayAdminAddress struct {
 	Host string `json:"host,omitempty"`
 }
 
+// EnvoyGatewayDebugAddress defines the Envoy Gateway Debug Address configuration.
+type EnvoyGatewayDebugAddress struct {
+	// Port defines the port the debug server is exposed on.
+	//
+	// +optional
+	// +kubebuilder:validation:Minimum=0
+	// +kubebuilder:default=19010
+	Port int `json:"port,omitempty"`
+	// Host defines the debug server hostname.
+	//
+	// +optional
+	// +kubebuilder:default="127.0.0.1"
+	Host string `json:"host,omitempty"`
+}
+
 func init() {
 	SchemeBuilder.Register(&EnvoyGateway{})
 }

api/v1alpha1/envoyproxy_types.go

@@ -129,44 +129,44 @@ type ProxyLogging struct {
 	// and the log level is the value. If unspecified, defaults to "default: warn".
 	//
 	// +kubebuilder:default={default: warn}
-	Level map[LogComponent]LogLevel `json:"level,omitempty"`
+	Level map[ProxyLogComponent]LogLevel `json:"level,omitempty"`
 }
 
-// LogComponent defines a component that supports a configured logging level.
+// ProxyLogComponent defines a component that supports a configured logging level.
 // +kubebuilder:validation:Enum=system;upstream;http;connection;admin;client;filter;main;router;runtime
-type LogComponent string
+type ProxyLogComponent string
 
 const (
 	// LogComponentDefault defines the default logging component.
 	// See more details: https://www.envoyproxy.io/docs/envoy/latest/operations/cli#cmdoption-l
-	LogComponentDefault LogComponent = "default"
+	LogComponentDefault ProxyLogComponent = "default"
 
 	// LogComponentUpstream defines the "upstream" logging component.
-	LogComponentUpstream LogComponent = "upstream"
+	LogComponentUpstream ProxyLogComponent = "upstream"
 
 	// LogComponentHTTP defines the "http" logging component.
-	LogComponentHTTP LogComponent = "http"
+	LogComponentHTTP ProxyLogComponent = "http"
 
 	// LogComponentConnection defines the "connection" logging component.
-	LogComponentConnection LogComponent = "connection"
+	LogComponentConnection ProxyLogComponent = "connection"
 
 	// LogComponentAdmin defines the "admin" logging component.
-	LogComponentAdmin LogComponent = "admin"
+	LogComponentAdmin ProxyLogComponent = "admin"
 
 	// LogComponentClient defines the "client" logging component.
-	LogComponentClient LogComponent = "client"
+	LogComponentClient ProxyLogComponent = "client"
 
 	// LogComponentFilter defines the "filter" logging component.
-	LogComponentFilter LogComponent = "filter"
+	LogComponentFilter ProxyLogComponent = "filter"
 
 	// LogComponentMain defines the "main" logging component.
-	LogComponentMain LogComponent = "main"
+	LogComponentMain ProxyLogComponent = "main"
 
 	// LogComponentRouter defines the "router" logging component.
-	LogComponentRouter LogComponent = "router"
+	LogComponentRouter ProxyLogComponent = "router"
 
 	// LogComponentRuntime defines the "runtime" logging component.
-	LogComponentRuntime LogComponent = "runtime"
+	LogComponentRuntime ProxyLogComponent = "runtime"
 )
 
 // ProxyBootstrap defines Envoy Bootstrap configuration.

api/v1alpha1/metric_types.go

@@ -5,9 +5,15 @@
 
 package v1alpha1
 
+type MetricSinkType string
+
+const (
+	MetricSinkTypeOpenTelemetry MetricSinkType = "OpenTelemetry"
+)
+
 type ProxyMetrics struct {
 	// Prometheus defines the configuration for Admin endpoint `/stats/prometheus`.
-	Prometheus *PrometheusProvider `json:"prometheus,omitempty"`
+	Prometheus *ProxyPrometheusProvider `json:"prometheus,omitempty"`
 	// Sinks defines the metric sinks where metrics are sent to.
 	Sinks []MetricSink `json:"sinks,omitempty"`
 	// Matches defines configuration for selecting specific metrics instead of generating all metrics stats
@@ -23,12 +29,6 @@ type ProxyMetrics struct {
 	EnableVirtualHostStats bool `json:"enableVirtualHostStats,omitempty"`
 }
 
-type MetricSinkType string
-
-const (
-	MetricSinkTypeOpenTelemetry MetricSinkType = "OpenTelemetry"
-)
-
 type MetricSink struct {
 	// Type defines the metric sink type.
 	// EG currently only supports OpenTelemetry.
@@ -71,5 +71,5 @@ type OpenTelemetrySink struct {
 	// TODO: add support for customizing OpenTelemetry sink in https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/stat_sinks/open_telemetry/v3/open_telemetry.proto#envoy-v3-api-msg-extensions-stat-sinks-open-telemetry-v3-sinkconfig
 }
 
-type PrometheusProvider struct {
+type ProxyPrometheusProvider struct {
 }