Add XDS keepalives (#1747)

* Add XDS keepalives Fixes #1746 Signed-off-by: Jack Kleeman <[email protected]> * Use instead http2 keepalives Signed-off-by: Jack Kleeman <[email protected]> * Permit keepalives on server side Signed-off-by: Jack Kleeman <[email protected]> --------- Signed-off-by: Jack Kleeman <[email protected]>
envoyproxy · Aug 9, 2023 · 6b8268f · 6b8268f
1 parent 85a2db8
commit 6b8268f
Show file tree

Hide file tree

Showing 22 changed files with 101 additions and 1 deletion.
diff --git a/internal/cmd/egctl/testdata/translate/out/authn-single-route-single-match-to-xds.all.json b/internal/cmd/egctl/testdata/translate/out/authn-single-route-single-match-to-xds.all.json
@@ -62,6 +62,12 @@
               "clusters": [
                 {
                   "connectTimeout": "10s",
+                  "http2ProtocolOptions": {
+                    "connectionKeepalive": {
+                      "interval": "30s",
+                      "timeout": "5s"
+                    }
+                  },
                   "loadAssignment": {
                     "clusterName": "xds_cluster",
                     "endpoints": [

diff --git a/internal/cmd/egctl/testdata/translate/out/authn-single-route-single-match-to-xds.all.yaml b/internal/cmd/egctl/testdata/translate/out/authn-single-route-single-match-to-xds.all.yaml
@@ -38,6 +38,10 @@ xds:
         staticResources:
           clusters:
           - connectTimeout: 10s
+            http2ProtocolOptions:
+              connectionKeepalive:
+                interval: 30s
+                timeout: 5s
             loadAssignment:
               clusterName: xds_cluster
               endpoints:

diff --git a/...al/cmd/egctl/testdata/translate/out/authn-single-route-single-match-to-xds.bootstrap.yaml b/...al/cmd/egctl/testdata/translate/out/authn-single-route-single-match-to-xds.bootstrap.yaml
@@ -37,6 +37,10 @@ xds:
       staticResources:
         clusters:
         - connectTimeout: 10s
+          http2ProtocolOptions:
+            connectionKeepalive:
+              interval: 30s
+              timeout: 5s
           loadAssignment:
             clusterName: xds_cluster
             endpoints:

diff --git a/internal/cmd/egctl/testdata/translate/out/default-resources.all.yaml b/internal/cmd/egctl/testdata/translate/out/default-resources.all.yaml
@@ -75,6 +75,10 @@ envoyProxy:
                 http2_protocol_options: {}
           name: xds_cluster
           type: STRICT_DNS
+          http2_protocol_options:
+            connection_keepalive:
+              interval: 30s
+              timeout: 5s
           transport_socket:
             name: envoy.transport_sockets.tls
             typed_config:
@@ -460,6 +464,10 @@ xds:
         staticResources:
           clusters:
           - connectTimeout: 10s
+            http2ProtocolOptions:
+              connectionKeepalive:
+                interval: 30s
+                timeout: 5s
             loadAssignment:
               clusterName: xds_cluster
               endpoints:

diff --git a/internal/cmd/egctl/testdata/translate/out/envoy-patch-policy.all.yaml b/internal/cmd/egctl/testdata/translate/out/envoy-patch-policy.all.yaml
@@ -38,6 +38,10 @@ xds:
         staticResources:
           clusters:
           - connectTimeout: 10s
+            http2ProtocolOptions:
+              connectionKeepalive:
+                interval: 30s
+                timeout: 5s
             loadAssignment:
               clusterName: xds_cluster
               endpoints:

diff --git a/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.json b/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.json
@@ -62,6 +62,12 @@
               "clusters": [
                 {
                   "connectTimeout": "10s",
+                  "http2ProtocolOptions": {
+                    "connectionKeepalive": {
+                      "interval": "30s",
+                      "timeout": "5s"
+                    }
+                  },
                   "loadAssignment": {
                     "clusterName": "xds_cluster",
                     "endpoints": [

diff --git a/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.yaml b/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.all.yaml
@@ -38,6 +38,10 @@ xds:
         staticResources:
           clusters:
           - connectTimeout: 10s
+            http2ProtocolOptions:
+              connectionKeepalive:
+                interval: 30s
+                timeout: 5s
             loadAssignment:
               clusterName: xds_cluster
               endpoints:

diff --git a/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.bootstrap.yaml b/internal/cmd/egctl/testdata/translate/out/from-gateway-api-to-xds.bootstrap.yaml
@@ -37,6 +37,10 @@ xds:
       staticResources:
         clusters:
         - connectTimeout: 10s
+          http2ProtocolOptions:
+            connectionKeepalive:
+              interval: 30s
+              timeout: 5s
           loadAssignment:
             clusterName: xds_cluster
             endpoints:

diff --git a/.../egctl/testdata/translate/out/rate-limit-filter-single-route-single-match-to-xds.all.json b/.../egctl/testdata/translate/out/rate-limit-filter-single-route-single-match-to-xds.all.json
@@ -62,6 +62,12 @@
               "clusters": [
                 {
                   "connectTimeout": "10s",
+                  "http2ProtocolOptions": {
+                    "connectionKeepalive": {
+                      "interval": "30s",
+                      "timeout": "5s"
+                    }
+                  },
                   "loadAssignment": {
                     "clusterName": "xds_cluster",
                     "endpoints": [

diff --git a/.../egctl/testdata/translate/out/rate-limit-filter-single-route-single-match-to-xds.all.yaml b/.../egctl/testdata/translate/out/rate-limit-filter-single-route-single-match-to-xds.all.yaml
@@ -38,6 +38,10 @@ xds:
         staticResources:
           clusters:
           - connectTimeout: 10s
+            http2ProtocolOptions:
+              connectionKeepalive:
+                interval: 30s
+                timeout: 5s
             loadAssignment:
               clusterName: xds_cluster
               endpoints:

diff --git a/.../testdata/translate/out/rate-limit-filter-single-route-single-match-to-xds.bootstrap.yaml b/.../testdata/translate/out/rate-limit-filter-single-route-single-match-to-xds.bootstrap.yaml
@@ -37,6 +37,10 @@ xds:
       staticResources:
         clusters:
         - connectTimeout: 10s
+          http2ProtocolOptions:
+            connectionKeepalive:
+              interval: 30s
+              timeout: 5s
           loadAssignment:
             clusterName: xds_cluster
             endpoints:

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom.yaml
@@ -109,6 +109,10 @@ spec:
                         http2_protocol_options: {}
                   name: xds_cluster
                   type: STRICT_DNS
+                  http2_protocol_options:
+                    connection_keepalive:
+                      interval: 30s
+                      timeout: 5s
                   transport_socket:
                     name: envoy.transport_sockets.tls
                     typed_config:

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/default-env.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/default-env.yaml
@@ -107,6 +107,10 @@ spec:
                         http2_protocol_options: {}
                   name: xds_cluster
                   type: STRICT_DNS
+                  http2_protocol_options:
+                    connection_keepalive:
+                      interval: 30s
+                      timeout: 5s
                   transport_socket:
                     name: envoy.transport_sockets.tls
                     typed_config:

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/default.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/default.yaml
@@ -105,6 +105,10 @@ spec:
                         http2_protocol_options: {}
                   name: xds_cluster
                   type: STRICT_DNS
+                  http2_protocol_options:
+                    connection_keepalive:
+                      interval: 30s
+                      timeout: 5s
                   transport_socket:
                     name: envoy.transport_sockets.tls
                     typed_config:

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/enable-prometheus.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/enable-prometheus.yaml
@@ -131,6 +131,10 @@ spec:
                         http2_protocol_options: {}
                   name: xds_cluster
                   type: STRICT_DNS
+                  http2_protocol_options:
+                    connection_keepalive:
+                      interval: 30s
+                      timeout: 5s
                   transport_socket:
                     name: envoy.transport_sockets.tls
                     typed_config:

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/extension-env.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/extension-env.yaml
@@ -107,6 +107,10 @@ spec:
                         http2_protocol_options: {}
                   name: xds_cluster
                   type: STRICT_DNS
+                  http2_protocol_options:
+                    connection_keepalive:
+                      interval: 30s
+                      timeout: 5s
                   transport_socket:
                     name: envoy.transport_sockets.tls
                     typed_config:

diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/volumes.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/volumes.yaml
@@ -107,6 +107,10 @@ spec:
                         http2_protocol_options: {}
                   name: xds_cluster
                   type: STRICT_DNS
+                  http2_protocol_options:
+                    connection_keepalive:
+                      interval: 30s
+                      timeout: 5s
                   transport_socket:
                     name: envoy.transport_sockets.tls
                     typed_config:

diff --git a/internal/xds/bootstrap/bootstrap.yaml.tpl b/internal/xds/bootstrap/bootstrap.yaml.tpl
@@ -125,6 +125,10 @@ static_resources:
           http2_protocol_options: {}
     name: xds_cluster
     type: STRICT_DNS
+    http2_protocol_options:
+      connection_keepalive:
+        interval: 30s
+        timeout: 5s
     transport_socket:
       name: envoy.transport_sockets.tls
       typed_config:

diff --git a/internal/xds/bootstrap/testdata/default.yaml b/internal/xds/bootstrap/testdata/default.yaml
@@ -68,6 +68,10 @@ static_resources:
           http2_protocol_options: {}
     name: xds_cluster
     type: STRICT_DNS
+    http2_protocol_options:
+      connection_keepalive:
+        interval: 30s
+        timeout: 5s
     transport_socket:
       name: envoy.transport_sockets.tls
       typed_config:

diff --git a/internal/xds/bootstrap/testdata/enable-prometheus.yaml b/internal/xds/bootstrap/testdata/enable-prometheus.yaml
@@ -90,6 +90,10 @@ static_resources:
           http2_protocol_options: {}
     name: xds_cluster
     type: STRICT_DNS
+    http2_protocol_options:
+      connection_keepalive:
+        interval: 30s
+        timeout: 5s
     transport_socket:
       name: envoy.transport_sockets.tls
       typed_config:

diff --git a/internal/xds/bootstrap/testdata/otel-metrics.yaml b/internal/xds/bootstrap/testdata/otel-metrics.yaml
@@ -93,6 +93,10 @@ static_resources:
           http2_protocol_options: {}
     name: xds_cluster
     type: STRICT_DNS
+    http2_protocol_options:
+      connection_keepalive:
+        interval: 30s
+        timeout: 5s
     transport_socket:
       name: envoy.transport_sockets.tls
       typed_config:

diff --git a/internal/xds/server/runner/runner.go b/internal/xds/server/runner/runner.go
@@ -14,6 +14,9 @@ import (
 	"net"
 	"os"
 	"strconv"
+	"time"
+
+	"google.golang.org/grpc/keepalive"
 
 	clusterv3 "github.com/envoyproxy/go-control-plane/envoy/service/cluster/v3"
 	discoveryv3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
@@ -75,7 +78,10 @@ func (r *Runner) Start(ctx context.Context) error {
 	// Create SnapshotCache before start subscribeAndTranslate,
 	// prevent panics in case cache is nil.
 	cfg := r.tlsConfig(xdsTLSCertFilename, xdsTLSKeyFilename, xdsTLSCaFilename)
-	r.grpc = grpc.NewServer(grpc.Creds(credentials.NewTLS(cfg)))
+	r.grpc = grpc.NewServer(grpc.Creds(credentials.NewTLS(cfg)), grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
+		MinTime:             15 * time.Second,
+		PermitWithoutStream: true,
+	}))
 
 	r.cache = cache.NewSnapshotCache(false, r.Logger)
 	registerServer(serverv3.NewServer(ctx, r.cache, r.cache), r.grpc)