kuma-dp: make drain time of Envoy Listeners configurable

app/kuma-dp/pkg/dataplane/envoy/envoy.go

@@ -2,9 +2,11 @@ package envoy
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"os"
 	"os/exec"
+	"time"
 
 	"path/filepath"
 
@@ -105,6 +107,8 @@ func (e *Envoy) Run(stop <-chan struct{}) error {
 
 	args := []string{
 		"-c", configFile,
+		"--drain-time-s",
+		fmt.Sprintf("%d", e.opts.Config.Dataplane.DrainTime/time.Second),
 		// "hot restart" (enabled by default) requires each Envoy instance to have
 		// `--base-id <uint32_t>` argument.
 		// it is not possible to start multiple Envoy instances on the same Linux machine

app/kuma-dp/pkg/dataplane/envoy/envoy_test.go

@@ -10,6 +10,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/gogo/protobuf/proto"
 
@@ -63,6 +64,9 @@ var _ = Describe("Envoy", func() {
 		It("should generate bootstrap config file and start Envoy", func(done Done) {
 			// given
 			cfg := kuma_dp.Config{
+				Dataplane: kuma_dp.Dataplane{
+					DrainTime: 15 * time.Second,
+				},
 				DataplaneRuntime: kuma_dp.DataplaneRuntime{
 					BinaryPath: filepath.Join("testdata", "envoy-mock.exit-0.sh"),
 					ConfigDir:  configDir,
@@ -115,7 +119,7 @@ var _ = Describe("Envoy", func() {
 			// then
 			Expect(err).ToNot(HaveOccurred())
 			// and
-			Expect(strings.TrimSpace(buf.String())).To(Equal(fmt.Sprintf("-c %s --disable-hot-restart", expectedConfigFile)))
+			Expect(strings.TrimSpace(buf.String())).To(Equal(fmt.Sprintf("-c %s --drain-time-s 15 --disable-hot-restart", expectedConfigFile)))
 
 			By("verifying the contents Envoy config file")
 			// when

app/kuma-injector/pkg/injector/injector.go

@@ -104,6 +104,10 @@ func (i *KumaInjector) NewSidecarContainer(pod *kube_core.Pod) kube_core.Contain
 				Name:  "KUMA_DATAPLANE_ADMIN_PORT",
 				Value: fmt.Sprintf("%d", i.cfg.SidecarContainer.AdminPort),
 			},
+			{
+				Name:  "KUMA_DATAPLANE_DRAIN_TIME",
+				Value: fmt.Sprintf("%s", i.cfg.SidecarContainer.DrainTime),
+			},
 		},
 		SecurityContext: &kube_core.SecurityContext{
 			RunAsUser:  &i.cfg.SidecarContainer.UID,

app/kuma-injector/pkg/injector/testdata/inject.01.golden.yaml

@@ -46,6 +46,8 @@ spec:
       value: $(POD_NAME).$(POD_NAMESPACE)
     - name: KUMA_DATAPLANE_ADMIN_PORT
       value: "9901"
+    - name: KUMA_DATAPLANE_DRAIN_TIME
+      value: 31s
     image: kuma/kuma-sidecar:latest
     imagePullPolicy: IfNotPresent
     livenessProbe:

app/kuma-injector/pkg/injector/testdata/inject.02.golden.yaml

@@ -47,6 +47,8 @@ spec:
       value: $(POD_NAME).$(POD_NAMESPACE)
     - name: KUMA_DATAPLANE_ADMIN_PORT
       value: "9901"
+    - name: KUMA_DATAPLANE_DRAIN_TIME
+      value: 31s
     image: kuma/kuma-sidecar:latest
     imagePullPolicy: IfNotPresent
     livenessProbe:

app/kuma-injector/pkg/injector/testdata/inject.03.golden.yaml

@@ -106,6 +106,8 @@ spec:
       value: $(POD_NAME).$(POD_NAMESPACE)
     - name: KUMA_DATAPLANE_ADMIN_PORT
       value: "9901"
+    - name: KUMA_DATAPLANE_DRAIN_TIME
+      value: 31s
     image: kuma/kuma-sidecar:latest
     imagePullPolicy: IfNotPresent
     livenessProbe:

app/kuma-injector/pkg/injector/testdata/inject.04.golden.yaml

@@ -46,6 +46,8 @@ spec:
       value: $(POD_NAME).$(POD_NAMESPACE)
     - name: KUMA_DATAPLANE_ADMIN_PORT
       value: "9901"
+    - name: KUMA_DATAPLANE_DRAIN_TIME
+      value: 31s
     image: kuma/kuma-sidecar:latest
     imagePullPolicy: IfNotPresent
     livenessProbe:

app/kuma-injector/pkg/injector/testdata/inject.config.yaml

@@ -9,6 +9,7 @@ sidecarContainer:
   uid: 5678
   gid: 5678
   adminPort: 9901
+  drainTime: 31s
 
   readinessProbe:
     initialDelaySeconds: 11

app/kumactl/cmd/install/testdata/install-control-plane.defaults.golden.yaml

@@ -47,6 +47,7 @@ data:
         uid: 5678
         gid: 5678
         adminPort: 9901
+        drainTime: 30s
 
         readinessProbe:
           initialDelaySeconds: 1

app/kumactl/cmd/install/testdata/install-control-plane.overrides.golden.yaml

@@ -47,6 +47,7 @@ data:
         uid: 5678
         gid: 5678
         adminPort: 9901
+        drainTime: 30s
 
         readinessProbe:
           initialDelaySeconds: 1

app/kumactl/data/install/k8s/control-plane/kuma-injector/app.yaml

@@ -36,6 +36,7 @@ data:
         uid: 5678
         gid: 5678
         adminPort: 9901
+        drainTime: 30s
 
         readinessProbe:
           initialDelaySeconds: 1

pkg/config/app/kuma-dp/config.go

@@ -2,6 +2,7 @@ package kumadp
 
 import (
 	"net/url"
+	"time"
 
 	"github.com/Kong/kuma/pkg/config"
 
@@ -20,6 +21,7 @@ func DefaultConfig() Config {
 			Mesh:      "default",
 			Name:      "", // Dataplane name must be set explicitly
 			AdminPort: 0,  // by default, turn off Admin interface of Envoy
+			DrainTime: 30 * time.Second,
 		},
 		DataplaneRuntime: DataplaneRuntime{
 			BinaryPath: "envoy",
@@ -57,6 +59,8 @@ type Dataplane struct {
 	Name string `yaml:"name,omitempty" envconfig:"kuma_dataplane_name"`
 	// Envoy Admin port.
 	AdminPort uint32 `yaml:"adminPort,omitempty" envconfig:"kuma_dataplane_admin_port"`
+	// Drain time for listeners.
+	DrainTime time.Duration `yaml:"drainTime,omitempty" envconfig:"kuma_dataplane_drain_time"`
 }
 
 // DataplaneRuntime defines the context in which dataplane (Envoy) runs.
@@ -103,6 +107,9 @@ func (d *Dataplane) Validate() (errs error) {
 	if 65535 < d.AdminPort {
 		errs = multierr.Append(errs, errors.Errorf(".AdminPort must be in the range [0, 65535]"))
 	}
+	if d.DrainTime <= 0 {
+		errs = multierr.Append(errs, errors.Errorf(".DrainTime must be positive"))
+	}
 	return
 }
 

pkg/config/app/kuma-dp/config_test.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -27,6 +28,7 @@ var _ = Describe("Config", func() {
 		// and
 		Expect(cfg.ControlPlane.BootstrapServer.URL).To(Equal("https://kuma-control-plane.internal:5682"))
 		Expect(cfg.Dataplane.AdminPort).To(Equal(uint32(2345)))
+		Expect(cfg.Dataplane.DrainTime).To(Equal(60 * time.Second))
 	})
 
 	Context("with modified environment variables", func() {
@@ -52,6 +54,7 @@ var _ = Describe("Config", func() {
 				"KUMA_DATAPLANE_MESH":                     "pilot",
 				"KUMA_DATAPLANE_NAME":                     "example",
 				"KUMA_DATAPLANE_ADMIN_PORT":               "2345",
+				"KUMA_DATAPLANE_DRAIN_TIME":               "60s",
 				"KUMA_DATAPLANE_RUNTIME_BINARY_PATH":      "envoy.sh",
 				"KUMA_DATAPLANE_RUNTIME_CONFIG_DIR":       "/var/run/envoy",
 			}
@@ -73,6 +76,7 @@ var _ = Describe("Config", func() {
 			Expect(cfg.Dataplane.Mesh).To(Equal("pilot"))
 			Expect(cfg.Dataplane.Name).To(Equal("example"))
 			Expect(cfg.Dataplane.AdminPort).To(Equal(uint32(2345)))
+			Expect(cfg.Dataplane.DrainTime).To(Equal(60 * time.Second))
 			Expect(cfg.DataplaneRuntime.BinaryPath).To(Equal("envoy.sh"))
 			Expect(cfg.DataplaneRuntime.ConfigDir).To(Equal("/var/run/envoy"))
 		})
@@ -103,6 +107,6 @@ var _ = Describe("Config", func() {
 		err := config.Load(filepath.Join("testdata", "invalid-config.input.yaml"), &cfg)
 
 		// then
-		Expect(err).To(MatchError(`Invalid configuration: .ControlPlane is not valid: .BootstrapServer is not valid: .URL must be a valid absolute URI; .Dataplane is not valid: .Mesh must be non-empty; .Name must be non-empty; .AdminPort must be in the range [0, 65535]; .DataplaneRuntime is not valid: .BinaryPath must be non-empty`))
+		Expect(err).To(MatchError(`Invalid configuration: .ControlPlane is not valid: .BootstrapServer is not valid: .URL must be a valid absolute URI; .Dataplane is not valid: .Mesh must be non-empty; .Name must be non-empty; .AdminPort must be in the range [0, 65535]; .DrainTime must be positive; .DataplaneRuntime is not valid: .BinaryPath must be non-empty`))
 	})
 })

pkg/config/app/kuma-dp/testdata/default-config.golden.yaml

@@ -1,4 +1,5 @@
 dataplane:
   mesh: default
+  drainTime: 30s
 dataplaneRuntime:
   binaryPath: envoy

pkg/config/app/kuma-dp/testdata/invalid-config.input.yaml

@@ -5,5 +5,6 @@ dataplane:
   mesh:
   name:
   adminPort: 82345
+  drainTime: 0
 dataplaneRuntime:
   binaryPath:

pkg/config/app/kuma-dp/testdata/valid-config.input.yaml

@@ -5,6 +5,7 @@ dataplane:
   mesh: pilot
   name: example
   adminPort: 2345
+  drainTime: 60s
 dataplaneRuntime:
   binaryPath: envoy.sh
   configDir: /var/run/envoy

pkg/config/app/kuma-injector/config.go

@@ -3,6 +3,7 @@ package kumainjector
 import (
 	"net"
 	"net/url"
+	"time"
 
 	"github.com/Kong/kuma/pkg/config"
 
@@ -37,6 +38,7 @@ func DefaultConfig() Config {
 				UID:          5678,
 				GID:          5678,
 				AdminPort:    9901,
+				DrainTime:    30 * time.Second,
 
 				ReadinessProbe: SidecarReadinessProbe{
 					InitialDelaySeconds: 1,
@@ -130,6 +132,8 @@ type SidecarContainer struct {
 	GID int64 `yaml:"gid,omitempty" envconfig:"kuma_injector_sidecar_container_gui"`
 	// Admin port.
 	AdminPort uint32 `yaml:"adminPort,omitempty" envconfig:"kuma_injector_sidecar_container_admin_port"`
+	// Drain time for listeners.
+	DrainTime time.Duration `yaml:"drainTime,omitempty" envconfig:"kuma_injector_sidecar_container_drain_time"`
 	// Readiness probe.
 	ReadinessProbe SidecarReadinessProbe `yaml:"readinessProbe,omitempty"`
 	// Liveness probe.
@@ -288,6 +292,9 @@ func (c *SidecarContainer) Validate() (errs error) {
 	if 65535 < c.AdminPort {
 		errs = multierr.Append(errs, errors.Errorf(".AdminPort must be in the range [0, 65535]"))
 	}
+	if c.DrainTime <= 0 {
+		errs = multierr.Append(errs, errors.Errorf(".DrainTime must be positive"))
+	}
 	if err := c.ReadinessProbe.Validate(); err != nil {
 		errs = multierr.Append(errs, errors.Wrapf(err, ".ReadinessProbe is not valid"))
 	}