design(docs/api): control plane metrics monitoring

Signed-off-by: bitliu <[email protected]>

api/v1alpha1/envoygateway_helpers.go

@@ -19,10 +19,11 @@ func DefaultEnvoyGateway() *EnvoyGateway {
 			APIVersion: GroupVersion.String(),
 		},
 		EnvoyGatewaySpec{
-			Gateway:  DefaultGateway(),
-			Provider: DefaultEnvoyGatewayProvider(),
-			Logging:  DefaultEnvoyGatewayLogging(),
-			Admin:    DefaultEnvoyGatewayAdmin(),
+			Gateway:   DefaultGateway(),
+			Provider:  DefaultEnvoyGatewayProvider(),
+			Logging:   DefaultEnvoyGatewayLogging(),
+			Admin:     DefaultEnvoyGatewayAdmin(),
+			Telemetry: DefaultEnvoyGatewayTelemetry(),
 		},
 	}
 }
@@ -47,6 +48,9 @@ func (e *EnvoyGateway) SetEnvoyGatewayDefaults() {
 	if e.Admin == nil {
 		e.Admin = DefaultEnvoyGatewayAdmin()
 	}
+	if e.Telemetry == nil {
+		e.Telemetry = DefaultEnvoyGatewayTelemetry()
+	}
 }
 
 // GetEnvoyGatewayAdmin returns the EnvoyGatewayAdmin of EnvoyGateway or a default EnvoyGatewayAdmin if unspecified.
@@ -88,6 +92,71 @@ func DefaultEnvoyGatewayLogging() *EnvoyGatewayLogging {
 	}
 }
 
+// GetEnvoyGatewayAdmin returns the EnvoyGatewayAdmin of EnvoyGateway or a default EnvoyGatewayAdmin if unspecified.
+func (e *EnvoyGateway) GetEnvoyGatewayTelemetry() *EnvoyGatewayTelemetry {
+	if e.Telemetry != nil {
+		if e.Telemetry.Metrics.Prometheus == nil {
+			e.Telemetry.Metrics.Prometheus = DefaultEnvoyGatewayPrometheus()
+		}
+		if e.Telemetry.Metrics.Address == nil {
+			e.Telemetry.Metrics.Address = DefaultEnvoyGatewayMetricsAddress()
+		}
+		if e.Telemetry.Metrics == nil {
+			e.Telemetry.Metrics = DefaultEnvoyGatewayMetrics()
+		}
+		return e.Telemetry
+	}
+	e.Telemetry = DefaultEnvoyGatewayTelemetry()
+
+	return e.Telemetry
+}
+
+// GetEnvoyGatewayMetricsAddress returns the EnvoyGateway Metrics Address.
+func (e *EnvoyGateway) GetEnvoyGatewayMetricsAddress() string {
+	address := e.GetEnvoyGatewayTelemetry().Metrics.Address
+	if address != nil {
+		return fmt.Sprintf("%s:%d", address.Host, address.Port)
+	}
+
+	return ""
+}
+
+// IfDisablePrometheus returns if disable prometheus.
+func (e *EnvoyGateway) IfDisablePrometheus() bool {
+	return e.GetEnvoyGatewayTelemetry().Metrics.Prometheus.Disable
+}
+
+// DefaultEnvoyGatewayTelemetry returns a new EnvoyGatewayTelemetry with default configuration parameters.
+func DefaultEnvoyGatewayTelemetry() *EnvoyGatewayTelemetry {
+	return &EnvoyGatewayTelemetry{
+		Metrics: DefaultEnvoyGatewayMetrics(),
+	}
+}
+
+// DefaultEnvoyGatewayMetrics returns a new EnvoyGatewayMetrics with default configuration parameters.
+func DefaultEnvoyGatewayMetrics() *EnvoyGatewayMetrics {
+	return &EnvoyGatewayMetrics{
+		Address:    DefaultEnvoyGatewayMetricsAddress(),
+		Prometheus: DefaultEnvoyGatewayPrometheus(),
+	}
+}
+
+// DefaultEnvoyGatewayMetricsAddress returns a new EnvoyGatewayMetrics with default configuration parameters.
+func DefaultEnvoyGatewayMetricsAddress() *EnvoyGatewayMetricsAddress {
+	return &EnvoyGatewayMetricsAddress{
+		Host: GatewayMetricsHost,
+		Port: GatewayMetricsPort,
+	}
+}
+
+// DefaultEnvoyGatewayPrometheus returns a new EnvoyGatewayMetrics with default configuration parameters.
+func DefaultEnvoyGatewayPrometheus() *EnvoyGatewayPrometheusProvider {
+	return &EnvoyGatewayPrometheusProvider{
+		// Enable prometheus pull by default.
+		Disable: false,
+	}
+}
+
 // DefaultEnvoyGatewayProvider returns a new EnvoyGatewayProvider with default configuration parameters.
 func DefaultEnvoyGatewayProvider() *EnvoyGatewayProvider {
 	return &EnvoyGatewayProvider{

api/v1alpha1/envoygateway_metrics_types.go

@@ -0,0 +1,65 @@
+// Copyright Envoy Gateway Authors
+// SPDX-License-Identifier: Apache-2.0
+// The full text of the Apache license is available in the LICENSE file at
+// the root of the repo.
+
+package v1alpha1
+
+// EnvoyGatewayMetrics defines control plane push/pull metrics configurations.
+type EnvoyGatewayMetrics struct {
+	// Address defines the address of Envoy Gateway Metrics Server.
+	Address *EnvoyGatewayMetricsAddress
+	// Sinks defines the metric sinks where metrics are sent to.
+	Sinks []EnvoyGatewayMetricSink `json:"sinks,omitempty"`
+	// Prometheus defines the configuration for prometheus endpoint.
+	Prometheus *EnvoyGatewayPrometheusProvider `json:"prometheus,omitempty"`
+}
+
+// EnvoyGatewayMetricSink defines control plane
+// metric sinks where metrics are sent to.
+type EnvoyGatewayMetricSink struct {
+	// Type defines the metric sink type.
+	// EG control plane currently supports OpenTelemetry.
+	// +kubebuilder:validation:Enum=OpenTelemetry
+	// +kubebuilder:default=OpenTelemetry
+	Type MetricSinkType `json:"type"`
+	// OpenTelemetry defines the configuration for OpenTelemetry sink.
+	// It's required if the sink type is OpenTelemetry.
+	OpenTelemetry *EnvoyGatewayOpenTelemetrySink `json:"openTelemetry,omitempty"`
+}
+
+type EnvoyGatewayOpenTelemetrySink struct {
+	// Host define the sink service hostname.
+	Host string `json:"host"`
+	// Protocol define the sink service protocol.
+	// +kubebuilder:validation:Enum=grpc;http
+	Protocol string `json:"protocol"`
+	// Port defines the port the sink service is exposed on.
+	//
+	// +optional
+	// +kubebuilder:validation:Minimum=0
+	// +kubebuilder:default=4317
+	Port int32 `json:"port,omitempty"`
+}
+
+// EnvoyGatewayPrometheusProvider will expose prometheus endpoint in pull mode.
+type EnvoyGatewayPrometheusProvider struct {
+	// Disable defines if disables the prometheus metrics in pull mode.
+	//
+	Disable bool `json:"disable,omitempty"`
+}
+
+// EnvoyGatewayMetricsAddress defines the Envoy Gateway Metrics Address configuration.
+type EnvoyGatewayMetricsAddress struct {
+	// Port defines the port the metrics server is exposed on.
+	//
+	// +optional
+	// +kubebuilder:validation:Minimum=0
+	// +kubebuilder:default=19001
+	Port int `json:"port,omitempty"`
+	// Host defines the metrics server hostname.
+	//
+	// +optional
+	// +kubebuilder:default="0.0.0.0"
+	Host string `json:"host,omitempty"`
+}

api/v1alpha1/envoygateway_types.go

@@ -19,6 +19,10 @@ const (
 	GatewayAdminPort = 19000
 	// GatewayAdminHost is the host of envoy gateway admin server.
 	GatewayAdminHost = "127.0.0.1"
+	// GatewayMetricsPort is the port which envoy gateway metrics server is listening on.
+	GatewayMetricsPort = 19001
+	// GatewayMetricsHost is the host of envoy gateway metrics server.
+	GatewayMetricsHost = "0.0.0.0"
 )
 
 // +kubebuilder:object:root=true
@@ -59,6 +63,12 @@ type EnvoyGatewaySpec struct {
 	// +optional
 	Admin *EnvoyGatewayAdmin `json:"admin,omitempty"`
 
+	// Telemetry defines the desired control plane telemetry related abilities.
+	// If unspecified, the telemetry is used with default configuration.
+	//
+	// +optional
+	Telemetry *EnvoyGatewayTelemetry `json:"telemetry,omitempty"`
+
 	// RateLimit defines the configuration associated with the Rate Limit service
 	// deployed by Envoy Gateway required to implement the Global Rate limiting
 	// functionality. The specific rate limit service used here is the reference
@@ -80,6 +90,13 @@ type EnvoyGatewaySpec struct {
 	ExtensionAPIs *ExtensionAPISettings `json:"extensionApis,omitempty"`
 }
 
+// EnvoyGatewayTelemetry defines telemetry configurations for envoy gateway control plane.
+// Control plane will focus on metrics observability telemetry and tracing telemetry later.
+type EnvoyGatewayTelemetry struct {
+	// Metrics defines metrics configuration for envoy gateway.
+	Metrics *EnvoyGatewayMetrics `json:"metrics,omitempty"`
+}
+
 // EnvoyGatewayLogging defines logging for Envoy Gateway.
 type EnvoyGatewayLogging struct {
 	// Level is the logging level. If unspecified, defaults to "info".

api/v1alpha1/metric_types.go → api/v1alpha1/envoyproxy_metric_types.go

@@ -5,11 +5,17 @@
 
 package v1alpha1
 
+type MetricSinkType string
+
+const (
+	MetricSinkTypeOpenTelemetry MetricSinkType = "OpenTelemetry"
+)
+
 type ProxyMetrics struct {
 	// Prometheus defines the configuration for Admin endpoint `/stats/prometheus`.
-	Prometheus *PrometheusProvider `json:"prometheus,omitempty"`
+	Prometheus *ProxyPrometheusProvider `json:"prometheus,omitempty"`
 	// Sinks defines the metric sinks where metrics are sent to.
-	Sinks []MetricSink `json:"sinks,omitempty"`
+	Sinks []ProxyMetricSink `json:"sinks,omitempty"`
 	// Matches defines configuration for selecting specific metrics instead of generating all metrics stats
 	// that are enabled by default. This helps reduce CPU and memory overhead in Envoy, but eliminating some stats
 	// may after critical functionality. Here are the stats that we strongly recommend not disabling:
@@ -23,21 +29,32 @@ type ProxyMetrics struct {
 	EnableVirtualHostStats bool `json:"enableVirtualHostStats,omitempty"`
 }
 
-type MetricSinkType string
-
-const (
-	MetricSinkTypeOpenTelemetry MetricSinkType = "OpenTelemetry"
-)
-
-type MetricSink struct {
+type ProxyMetricSink struct {
 	// Type defines the metric sink type.
 	// EG currently only supports OpenTelemetry.
 	// +kubebuilder:validation:Enum=OpenTelemetry
 	// +kubebuilder:default=OpenTelemetry
 	Type MetricSinkType `json:"type"`
 	// OpenTelemetry defines the configuration for OpenTelemetry sink.
 	// It's required if the sink type is OpenTelemetry.
-	OpenTelemetry *OpenTelemetrySink `json:"openTelemetry,omitempty"`
+	OpenTelemetry *ProxyOpenTelemetrySink `json:"openTelemetry,omitempty"`
+}
+
+type ProxyOpenTelemetrySink struct {
+	// Host define the service hostname.
+	Host string `json:"host"`
+	// Port defines the port the service is exposed on.
+	//
+	// +optional
+	// +kubebuilder:validation:Minimum=0
+	// +kubebuilder:validation:Maximum=65535
+	// +kubebuilder:default=4317
+	Port int32 `json:"port,omitempty"`
+
+	// TODO: add support for customizing OpenTelemetry sink in https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/stat_sinks/open_telemetry/v3/open_telemetry.proto#envoy-v3-api-msg-extensions-stat-sinks-open-telemetry-v3-sinkconfig
+}
+
+type ProxyPrometheusProvider struct {
 }
 
 // Match defines the stats match configuration.
@@ -56,20 +73,3 @@ const (
 	RegularExpression MatcherType = "RegularExpression"
 	Suffix            MatcherType = "Suffix"
 )
-
-type OpenTelemetrySink struct {
-	// Host define the service hostname.
-	Host string `json:"host"`
-	// Port defines the port the service is exposed on.
-	//
-	// +optional
-	// +kubebuilder:validation:Minimum=0
-	// +kubebuilder:validation:Maximum=65535
-	// +kubebuilder:default=4317
-	Port int32 `json:"port,omitempty"`
-
-	// TODO: add support for customizing OpenTelemetry sink in https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/stat_sinks/open_telemetry/v3/open_telemetry.proto#envoy-v3-api-msg-extensions-stat-sinks-open-telemetry-v3-sinkconfig
-}
-
-type PrometheusProvider struct {
-}

api/v1alpha1/validation/validate.go

@@ -179,6 +179,17 @@ func validateProxyTelemetry(spec *egv1a1.EnvoyProxySpec) []error {
 		}
 	}
 
+	if spec != nil && spec.Telemetry.Metrics != nil {
+		for _, sink := range spec.Telemetry.Metrics.Sinks {
+			if sink.Type == egv1a1.MetricSinkTypeOpenTelemetry {
+				if sink.OpenTelemetry == nil {
+					err := fmt.Errorf("opentelemetry is required if the sink type is OpenTelemetry")
+					errs = append(errs, err)
+				}
+			}
+		}
+	}
+
 	return errs
 }