feat: support statsmatch from proxy metric

api/config/v1alpha1/metric_types.go

@@ -11,7 +11,12 @@ type ProxyMetrics struct {
 	// Sinks defines the metric sinks where metrics are sent to.
 	Sinks []MetricSink `json:"sinks,omitempty"`
 	// Matches defines configuration for selecting specific metrics instead of generating all metrics stats
-	// that are enabled by default. This helps reduce CPU and memory overhead in Envoy.
+	// that are enabled by default. This helps reduce CPU and memory overhead in Envoy, but eliminating some stats
+	// may after critical functionality. Here are the stats that we strongly recommend not disabling:
+	// `cluster_manager.warming_clusters`, `cluster.<cluster_name>.membership_total`,`cluster.<cluster_name>.membership_healthy`,
+	// `cluster.<cluster_name>.membership_degraded`，reference  https://github.com/envoyproxy/envoy/issues/9856,
+	// https://github.com/envoyproxy/envoy/issues/14610
+	//
 	Matches []Match `json:"matches,omitempty"`
 
 	// EnableVirtualHostStats enables envoy stat metrics for virtual hosts.

charts/gateway-helm/crds/generated/config.gateway.envoyproxy.io_envoyproxies.yaml

@@ -3943,10 +3943,15 @@ spec:
                           for virtual hosts.
                         type: boolean
                       matches:
-                        description: Matches defines configuration for selecting specific
-                          metrics instead of generating all metrics stats that are
-                          enabled by default. This helps reduce CPU and memory overhead
-                          in Envoy.
+                        description: 'Matches defines configuration for selecting
+                          specific metrics instead of generating all metrics stats
+                          that are enabled by default. This helps reduce CPU and memory
+                          overhead in Envoy, but eliminating some stats may after
+                          critical functionality. Here are the stats that we strongly
+                          recommend not disabling: `cluster_manager.warming_clusters`,
+                          `cluster.<cluster_name>.membership_total`,`cluster.<cluster_name>.membership_healthy`,
+                          `cluster.<cluster_name>.membership_degraded`，reference  https://github.com/envoyproxy/envoy/issues/9856,
+                          https://github.com/envoyproxy/envoy/issues/14610'
                         items:
                           description: Match defines the stats match configuration.
                           properties:

docs/latest/api/config_types.md

@@ -849,7 +849,7 @@ _Appears in:_
 | --- | --- |
 | `prometheus` _[PrometheusProvider](#prometheusprovider)_ | Prometheus defines the configuration for Admin endpoint `/stats/prometheus`. |
 | `sinks` _[MetricSink](#metricsink) array_ | Sinks defines the metric sinks where metrics are sent to. |
-| `matches` _[Match](#match) array_ | Matches defines configuration for selecting specific metrics instead of generating all metrics stats that are enabled by default. This helps reduce CPU and memory overhead in Envoy. |
+| `matches` _[Match](#match) array_ | Matches defines configuration for selecting specific metrics instead of generating all metrics stats that are enabled by default. This helps reduce CPU and memory overhead in Envoy, but eliminating some stats may after critical functionality. Here are the stats that we strongly recommend not disabling: `cluster_manager.warming_clusters`, `cluster.<cluster_name>.membership_total`,`cluster.<cluster_name>.membership_healthy`, `cluster.<cluster_name>.membership_degraded`，reference  https://github.com/envoyproxy/envoy/issues/9856, https://github.com/envoyproxy/envoy/issues/14610 |
 | `enableVirtualHostStats` _boolean_ | EnableVirtualHostStats enables envoy stat metrics for virtual hosts. |
 
 

internal/xds/bootstrap/bootstrap.go

@@ -63,6 +63,11 @@ type bootstrapParameters struct {
 	EnablePrometheus bool
 	// OtelMetricSinks defines the configuration of the OpenTelemetry sinks.
 	OtelMetricSinks []metricSink
+	// EnableStatConfig defines whether to to customize the Envoy proxy stats.
+	EnableStatConfig bool
+	// StatsMatcher is to control creation of custom Envoy stats with prefix,
+	// suffix, and regex expressions match on the name of the stats.
+	StatsMatcher *StatsMatcherParameters
 }
 
 type xdsServerParameters struct {
@@ -97,6 +102,12 @@ type readyServerParameters struct {
 	ReadinessPath string
 }
 
+type StatsMatcherParameters struct {
+	Prefixs            []string
+	Suffixs            []string
+	RegularExpressions []string
+}
+
 // render the stringified bootstrap config in yaml format.
 func (b *bootstrapConfig) render() error {
 	buf := new(strings.Builder)
@@ -113,6 +124,7 @@ func GetRenderedBootstrapConfig(proxyMetrics *egcfgv1a1.ProxyMetrics) (string, e
 	var (
 		enablePrometheus bool
 		metricSinks      []metricSink
+		StatsMatcher     StatsMatcherParameters
 	)
 
 	if proxyMetrics != nil {
@@ -138,6 +150,22 @@ func GetRenderedBootstrapConfig(proxyMetrics *egcfgv1a1.ProxyMetrics) (string, e
 				Port:    sink.OpenTelemetry.Port,
 			})
 		}
+
+		if proxyMetrics.Matches != nil {
+
+			// Add custom envoy proxy stats
+			for _, match := range proxyMetrics.Matches {
+				switch match.Type {
+				case egcfgv1a1.Prefix:
+					StatsMatcher.Prefixs = append(StatsMatcher.Prefixs, match.Value)
+				case egcfgv1a1.Suffix:
+					StatsMatcher.Suffixs = append(StatsMatcher.Suffixs, match.Value)
+				case egcfgv1a1.RegularExpression:
+					StatsMatcher.RegularExpressions = append(StatsMatcher.RegularExpressions, match.Value)
+				}
+			}
+		}
+
 	}
 
 	cfg := &bootstrapConfig{
@@ -160,6 +188,9 @@ func GetRenderedBootstrapConfig(proxyMetrics *egcfgv1a1.ProxyMetrics) (string, e
 			OtelMetricSinks:  metricSinks,
 		},
 	}
+	if proxyMetrics != nil && proxyMetrics.Matches != nil {
+		cfg.parameters.StatsMatcher = &StatsMatcher
+	}
 
 	if err := cfg.render(); err != nil {
 		return "", err

internal/xds/bootstrap/bootstrap.yaml.tpl

@@ -8,6 +8,23 @@ admin:
     socket_address:
       address: {{ .AdminServer.Address }}
       port_value: {{ .AdminServer.Port }}
+{{- if .StatsMatcher  }}
+stats_config:
+  stats_matcher:
+    inclusion_list:
+      patterns:
+      {{- range $_, $item := .StatsMatcher.Prefixs }}
+      - prefix: {{$item}}
+      {{- end}}
+      {{- range $_, $item := .StatsMatcher.Suffixs }}
+      - suffix: {{$item}}
+      {{- end}}
+      {{- range $_, $item := .StatsMatcher.RegularExpressions }}
+      - safe_regex:
+          google_re2: {}
+          regex: {{js $item}}
+      {{- end}}
+{{- end }}
 dynamic_resources:
   ads_config:
     api_type: DELTA_GRPC

internal/xds/bootstrap/bootstrap_test.go

@@ -44,6 +44,30 @@ func TestGetRenderedBootstrapConfig(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "custom-stats-matcher",
+			proxyMetrics: &egcfgv1a1.ProxyMetrics{
+				Matches: []egcfgv1a1.Match{
+					{
+						Type:  egcfgv1a1.Prefix,
+						Value: "http",
+					},
+					{
+						Type:  egcfgv1a1.Suffix,
+						Value: "upstream_rq",
+					},
+					{
+						Type:  egcfgv1a1.RegularExpression,
+						Value: "virtual.*",
+					},
+					{
+						Type:  egcfgv1a1.Prefix,
+						Value: "cluster",
+					},
+				},
+				Prometheus: &egcfgv1a1.PrometheusProvider{},
+			},
+		},
 	}
 
 	for _, tc := range cases {

internal/xds/bootstrap/testdata/custom-stats-matcher.yaml

@@ -0,0 +1,133 @@
+admin:
+  access_log:
+  - name: envoy.access_loggers.file
+    typed_config:
+      "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
+      path: /dev/null
+  address:
+    socket_address:
+      address: 127.0.0.1
+      port_value: 19000
+stats_config:
+  stats_matcher:
+    inclusion_list:
+      patterns:
+      - prefix: http
+      - prefix: cluster
+      - suffix: upstream_rq
+      - safe_regex:
+          google_re2: {}
+          regex: virtual.*
+dynamic_resources:
+  ads_config:
+    api_type: DELTA_GRPC
+    transport_api_version: V3
+    grpc_services:
+    - envoy_grpc:
+        cluster_name: xds_cluster
+    set_node_on_first_message_only: true
+  lds_config:
+    ads: {}
+    resource_api_version: V3
+  cds_config:
+    ads: {}
+    resource_api_version: V3
+static_resources:
+  listeners:
+  - name: envoy-gateway-proxy-ready-0.0.0.0-19001
+    address:
+      socket_address:
+        address: 0.0.0.0
+        port_value: 19001
+        protocol: TCP
+    filter_chains:
+    - filters:
+      - name: envoy.filters.network.http_connection_manager
+        typed_config:
+          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+          stat_prefix: eg-ready-http
+          route_config:
+            name: local_route
+            virtual_hosts:
+            - name: prometheus_stats
+              domains:
+              - "*"
+              routes:
+              - match:
+                  prefix: /stats/prometheus
+                route:
+                  cluster: prometheus_stats
+          http_filters:
+          - name: envoy.filters.http.health_check
+            typed_config:
+              "@type": type.googleapis.com/envoy.extensions.filters.http.health_check.v3.HealthCheck
+              pass_through_mode: false
+              headers:
+              - name: ":path"
+                string_match:
+                  exact: /ready
+          - name: envoy.filters.http.router
+            typed_config:
+              "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+  clusters:
+  - name: prometheus_stats
+    connect_timeout: 0.250s
+    type: STATIC
+    lb_policy: ROUND_ROBIN
+    load_assignment:
+      cluster_name: prometheus_stats
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: 127.0.0.1
+                port_value: 19000
+  - connect_timeout: 10s
+    load_assignment:
+      cluster_name: xds_cluster
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: envoy-gateway
+                port_value: 18000
+    typed_extension_protocol_options:
+      envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
+        "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions"
+        explicit_http_config:
+          http2_protocol_options: {}
+    name: xds_cluster
+    type: STRICT_DNS
+    http2_protocol_options:
+      connection_keepalive:
+        interval: 30s
+        timeout: 5s
+    transport_socket:
+      name: envoy.transport_sockets.tls
+      typed_config:
+        "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
+        common_tls_context:
+          tls_params:
+            tls_maximum_protocol_version: TLSv1_3
+          tls_certificate_sds_secret_configs:
+          - name: xds_certificate
+            sds_config:
+              path_config_source:
+                path: "/sds/xds-certificate.json"
+              resource_api_version: V3
+          validation_context_sds_secret_config:
+            name: xds_trusted_ca
+            sds_config:
+              path_config_source:
+                path: "/sds/xds-trusted-ca.json"
+              resource_api_version: V3
+layered_runtime:
+  layers:
+  - name: runtime-0
+    rtds_layer:
+      rtds_config:
+        ads: {}
+        resource_api_version: V3
+      name: runtime-0